diff options
Diffstat (limited to 'zjit/src')
77 files changed, 74596 insertions, 0 deletions
diff --git a/zjit/src/asm/arm64/README.md b/zjit/src/asm/arm64/README.md new file mode 100644 index 0000000000..6adfad804d --- /dev/null +++ b/zjit/src/asm/arm64/README.md @@ -0,0 +1,16 @@ +# Arm64 + +This module is responsible for encoding ZJIT operands into an appropriate Arm64 encoding. + +## Architecture + +Every instruction in the Arm64 instruction set is 32 bits wide and is represented in little-endian order. Because they're all going to the same size, we represent each instruction by a struct that implements `From<T> for u32`, which contains the mechanism for encoding each instruction. The encoding for each instruction is shown in the documentation for the struct that ends up being created. + +In general each set of bytes inside of the struct has either a direct value (usually a `u8`/`u16`) or some kind of `enum` that can be converted directly into a `u32`. For more complicated pieces of encoding (e.g., bitmask immediates) a corresponding module under the `arg` namespace is available. + +## Helpful links + +* [Arm A64 Instruction Set Architecture](https://developer.arm.com/documentation/ddi0596/2021-12?lang=en) Official documentation +* [armconverter.com](https://armconverter.com/) A website that encodes Arm assembly syntax +* [hatstone](https://github.com/tenderlove/hatstone) A wrapper around the Capstone disassembler written in Ruby +* [onlinedisassembler.com](https://onlinedisassembler.com/odaweb/) A web-based disassembler diff --git a/zjit/src/asm/arm64/arg/bitmask_imm.rs b/zjit/src/asm/arm64/arg/bitmask_imm.rs new file mode 100644 index 0000000000..70a439afd5 --- /dev/null +++ b/zjit/src/asm/arm64/arg/bitmask_imm.rs @@ -0,0 +1,255 @@ +/// Immediates used by the logical immediate instructions are not actually the +/// immediate value, but instead are encoded into a 13-bit wide mask of 3 +/// elements. This allows many more values to be represented than 13 bits would +/// normally allow, at the expense of not being able to represent every possible +/// value. +/// +/// In order for a number to be encodeable in this form, the binary +/// representation must consist of a single set of contiguous 1s. That pattern +/// must then be replicatable across all of the bits either 1, 2, 4, 8, 16, or +/// 32 times (rotated or not). +/// +/// For example, 1 (0b1), 2 (0b10), 3 (0b11), and 4 (0b100) are all valid. +/// However, 5 (0b101) is invalid, because it contains 2 sets of 1s and cannot +/// be replicated across 64 bits. +/// +/// Some more examples to illustrate the idea of replication: +/// * 0x5555555555555555 is a valid value (0b0101...) because it consists of a +/// single set of 1s which can be replicated across all of the bits 32 times. +/// * 0xf0f0f0f0f0f0f0f0 is a valid value (0b1111000011110000...) because it +/// consists of a single set of 1s which can be replicated across all of the +/// bits 8 times (rotated by 4 bits). +/// * 0x0ff00ff00ff00ff0 is a valid value (0000111111110000...) because it +/// consists of a single set of 1s which can be replicated across all of the +/// bits 4 times (rotated by 12 bits). +/// +/// To encode the values, there are 3 elements: +/// * n = 1 if the pattern is 64-bits wide, 0 otherwise +/// * imms = the size of the pattern, a 0, and then one less than the number of +/// sequential 1s +/// * immr = the number of right rotations to apply to the pattern to get the +/// target value +/// +pub struct BitmaskImmediate { + n: u8, + imms: u8, + immr: u8 +} + +impl TryFrom<u64> for BitmaskImmediate { + type Error = (); + + /// Attempt to convert a u64 into a BitmaskImmediate. + /// + /// The implementation here is largely based on this blog post: + /// <https://dougallj.wordpress.com/2021/10/30/bit-twiddling-optimising-aarch64-logical-immediate-encoding-and-decoding/> + fn try_from(value: u64) -> Result<Self, Self::Error> { + if value == 0 || value == u64::MAX { + return Err(()); + } + + fn rotate_right(value: u64, rotations: u32) -> u64 { + (value >> (rotations & 0x3F)) | + (value << (rotations.wrapping_neg() & 0x3F)) + } + + let rotations = (value & (value + 1)).trailing_zeros(); + let normalized = rotate_right(value, rotations & 0x3F); + + let zeroes = normalized.leading_zeros(); + let ones = (!normalized).trailing_zeros(); + let size = zeroes + ones; + + if rotate_right(value, size & 0x3F) != value { + return Err(()); + } + + Ok(BitmaskImmediate { + n: ((size >> 6) & 1) as u8, + imms: (((size << 1).wrapping_neg() | (ones - 1)) & 0x3F) as u8, + immr: ((rotations.wrapping_neg() & (size - 1)) & 0x3F) as u8 + }) + } +} + +impl BitmaskImmediate { + /// Attempt to make a BitmaskImmediate for a 32 bit register. + /// The result has N==0, which is required for some 32-bit instructions. + /// Note that the exact same BitmaskImmediate produces different values + /// depending on the size of the target register. + pub fn new_32b_reg(value: u32) -> Result<Self, ()> { + // The same bit pattern replicated to u64 + let value = value as u64; + let replicated: u64 = (value << 32) | value; + let converted = Self::try_from(replicated); + if let Ok(ref imm) = converted { + assert_eq!(0, imm.n); + } + + converted + } +} + +impl BitmaskImmediate { + /// Encode a bitmask immediate into a 32-bit value. + pub fn encode(self) -> u32 { + 0 + | ((self.n as u32) << 12) + | ((self.immr as u32) << 6) + | (self.imms as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_failures() { + [5, 9, 10, 11, 13, 17, 18, 19].iter().for_each(|&imm| { + assert!(BitmaskImmediate::try_from(imm).is_err()); + }); + } + + #[test] + fn test_negative() { + let bitmask: BitmaskImmediate = (-9_i64 as u64).try_into().unwrap(); + let encoded: u32 = bitmask.encode(); + assert_eq!(7998, encoded); + } + + #[test] + fn test_size_2_minimum() { + let bitmask = BitmaskImmediate::try_from(0x5555555555555555); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b111100 }))); + } + + #[test] + fn test_size_2_maximum() { + let bitmask = BitmaskImmediate::try_from(0xaaaaaaaaaaaaaaaa); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000001, imms: 0b111100 }))); + } + + #[test] + fn test_size_4_minimum() { + let bitmask = BitmaskImmediate::try_from(0x1111111111111111); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b111000 }))); + } + + #[test] + fn test_size_4_rotated() { + let bitmask = BitmaskImmediate::try_from(0x6666666666666666); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000011, imms: 0b111001 }))); + } + + #[test] + fn test_size_4_maximum() { + let bitmask = BitmaskImmediate::try_from(0xeeeeeeeeeeeeeeee); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000011, imms: 0b111010 }))); + } + + #[test] + fn test_size_8_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0101010101010101); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b110000 }))); + } + + #[test] + fn test_size_8_rotated() { + let bitmask = BitmaskImmediate::try_from(0x1818181818181818); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000101, imms: 0b110001 }))); + } + + #[test] + fn test_size_8_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfefefefefefefefe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000111, imms: 0b110110 }))); + } + + #[test] + fn test_size_16_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0001000100010001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b100000 }))); + } + + #[test] + fn test_size_16_rotated() { + let bitmask = BitmaskImmediate::try_from(0xff8fff8fff8fff8f); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b001001, imms: 0b101100 }))); + } + + #[test] + fn test_size_16_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffefffefffefffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b001111, imms: 0b101110 }))); + } + + #[test] + fn test_size_32_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0000000100000001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b000000 }))); + } + + #[test] + fn test_size_32_rotated() { + let bitmask = BitmaskImmediate::try_from(0x3fffff003fffff00); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b011000, imms: 0b010101 }))); + } + + #[test] + fn test_size_32_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffffffefffffffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b011111, imms: 0b011110 }))); + } + + #[test] + fn test_size_64_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0000000000000001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b000000, imms: 0b000000 }))); + } + + #[test] + fn test_size_64_rotated() { + let bitmask = BitmaskImmediate::try_from(0x0000001fffff0000); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b110000, imms: 0b010100 }))); + } + + #[test] + fn test_size_64_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffffffffffffffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b111111, imms: 0b111110 }))); + } + + #[test] + fn test_size_64_invalid() { + let bitmask = BitmaskImmediate::try_from(u64::MAX); + assert!(matches!(bitmask, Err(()))); + } + + #[test] + fn test_all_valid_32b_pattern() { + let mut patterns = vec![]; + for pattern_size in [2, 4, 8, 16, 32_u64] { + for ones_count in 1..pattern_size { + for rotation in 0..pattern_size { + let ones = (1_u64 << ones_count) - 1; + let rotated = (ones >> rotation) | + ((ones & ((1 << rotation) - 1)) << (pattern_size - rotation)); + let mut replicated = rotated; + let mut shift = pattern_size; + while shift < 32 { + replicated |= replicated << shift; + shift *= 2; + } + let replicated: u32 = replicated.try_into().unwrap(); + assert!(BitmaskImmediate::new_32b_reg(replicated).is_ok()); + patterns.push(replicated); + } + } + } + patterns.sort(); + patterns.dedup(); + // Up to {size}-1 ones, and a total of {size} possible rotations. + assert_eq!(1*2 + 3*4 + 7*8 + 15*16 + 31*32, patterns.len()); + } +} diff --git a/zjit/src/asm/arm64/arg/condition.rs b/zjit/src/asm/arm64/arg/condition.rs new file mode 100644 index 0000000000..f711b8b0d8 --- /dev/null +++ b/zjit/src/asm/arm64/arg/condition.rs @@ -0,0 +1,52 @@ +/// Various instructions in A64 can have condition codes attached. This enum +/// includes all of the various kinds of conditions along with their respective +/// encodings. +pub struct Condition; + +impl Condition { + pub const EQ: u8 = 0b0000; // equal to + pub const NE: u8 = 0b0001; // not equal to + pub const CS: u8 = 0b0010; // carry set (alias for HS) + pub const CC: u8 = 0b0011; // carry clear (alias for LO) + pub const MI: u8 = 0b0100; // minus, negative + pub const PL: u8 = 0b0101; // positive or zero + pub const VS: u8 = 0b0110; // signed overflow + pub const VC: u8 = 0b0111; // no signed overflow + pub const HI: u8 = 0b1000; // greater than (unsigned) + pub const LS: u8 = 0b1001; // less than or equal to (unsigned) + pub const GE: u8 = 0b1010; // greater than or equal to (signed) + pub const LT: u8 = 0b1011; // less than (signed) + pub const GT: u8 = 0b1100; // greater than (signed) + pub const LE: u8 = 0b1101; // less than or equal to (signed) + pub const AL: u8 = 0b1110; // always + + pub const fn inverse(condition: u8) -> u8 { + match condition { + Condition::EQ => Condition::NE, + Condition::NE => Condition::EQ, + + Condition::CS => Condition::CC, + Condition::CC => Condition::CS, + + Condition::MI => Condition::PL, + Condition::PL => Condition::MI, + + Condition::VS => Condition::VC, + Condition::VC => Condition::VS, + + Condition::HI => Condition::LS, + Condition::LS => Condition::HI, + + Condition::LT => Condition::GE, + Condition::GE => Condition::LT, + + Condition::GT => Condition::LE, + Condition::LE => Condition::GT, + + Condition::AL => Condition::AL, + + _ => panic!("Unknown condition") + + } + } +} diff --git a/zjit/src/asm/arm64/arg/inst_offset.rs b/zjit/src/asm/arm64/arg/inst_offset.rs new file mode 100644 index 0000000000..f4a6bc73a0 --- /dev/null +++ b/zjit/src/asm/arm64/arg/inst_offset.rs @@ -0,0 +1,47 @@ +/// There are a lot of instructions in the AArch64 architectrue that take an +/// offset in terms of number of instructions. Usually they are jump +/// instructions or instructions that load a value relative to the current PC. +/// +/// This struct is used to mark those locations instead of a generic operand in +/// order to give better clarity to the developer when reading the AArch64 +/// backend code. It also helps to clarify that everything is in terms of a +/// number of instructions and not a number of bytes (i.e., the offset is the +/// number of bytes divided by 4). +#[derive(Copy, Clone)] +pub struct InstructionOffset(i32); + +impl InstructionOffset { + /// Create a new instruction offset. + pub fn from_insns(insns: i32) -> Self { + InstructionOffset(insns) + } + + /// Create a new instruction offset from a number of bytes. + pub fn from_bytes(bytes: i32) -> Self { + assert_eq!(bytes % 4, 0, "Byte offset must be a multiple of 4"); + InstructionOffset(bytes / 4) + } +} + +impl From<i32> for InstructionOffset { + /// Convert an i64 into an instruction offset. + fn from(value: i32) -> Self { + InstructionOffset(value) + } +} + +impl From<InstructionOffset> for i32 { + /// Convert an instruction offset into a number of instructions as an i32. + fn from(offset: InstructionOffset) -> Self { + offset.0 + } +} + +impl From<InstructionOffset> for i64 { + /// Convert an instruction offset into a number of instructions as an i64. + /// This is useful for when we're checking how many bits this offset fits + /// into. + fn from(offset: InstructionOffset) -> Self { + offset.0.into() + } +} diff --git a/zjit/src/asm/arm64/arg/mod.rs b/zjit/src/asm/arm64/arg/mod.rs new file mode 100644 index 0000000000..7eb37834f9 --- /dev/null +++ b/zjit/src/asm/arm64/arg/mod.rs @@ -0,0 +1,18 @@ +// This module contains various A64 instruction arguments and the logic +// necessary to encode them. + +mod bitmask_imm; +mod condition; +mod inst_offset; +mod sf; +mod shifted_imm; +mod sys_reg; +mod truncate; + +pub use bitmask_imm::BitmaskImmediate; +pub use condition::Condition; +pub use inst_offset::InstructionOffset; +pub use sf::Sf; +pub use shifted_imm::ShiftedImmediate; +pub use sys_reg::SystemRegister; +pub use truncate::{truncate_imm, truncate_uimm}; diff --git a/zjit/src/asm/arm64/arg/sf.rs b/zjit/src/asm/arm64/arg/sf.rs new file mode 100644 index 0000000000..b6091821e9 --- /dev/null +++ b/zjit/src/asm/arm64/arg/sf.rs @@ -0,0 +1,19 @@ +/// This is commonly the top-most bit in the encoding of the instruction, and +/// represents whether register operands should be treated as 64-bit registers +/// or 32-bit registers. +pub enum Sf { + Sf32 = 0b0, + Sf64 = 0b1 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Sf { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Sf::Sf64, + 32 => Sf::Sf32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} diff --git a/zjit/src/asm/arm64/arg/shifted_imm.rs b/zjit/src/asm/arm64/arg/shifted_imm.rs new file mode 100644 index 0000000000..06daefdef7 --- /dev/null +++ b/zjit/src/asm/arm64/arg/shifted_imm.rs @@ -0,0 +1,80 @@ +/// How much to shift the immediate by. +pub enum Shift { + LSL0 = 0b0, // no shift + LSL12 = 0b1 // logical shift left by 12 bits +} + +/// Some instructions accept a 12-bit immediate that has an optional shift +/// attached to it. This allows encoding larger values than just fit into 12 +/// bits. We attempt to encode those here. If the values are too large we have +/// to bail out. +pub struct ShiftedImmediate { + shift: Shift, + value: u16 +} + +impl TryFrom<u64> for ShiftedImmediate { + type Error = (); + + fn try_from(value: u64) -> Result<Self, Self::Error> { + let current = value; + if current < 2_u64.pow(12) { + return Ok(ShiftedImmediate { shift: Shift::LSL0, value: current as u16 }); + } + + if (current & (2_u64.pow(12) - 1) == 0) && ((current >> 12) < 2_u64.pow(12)) { + return Ok(ShiftedImmediate { shift: Shift::LSL12, value: (current >> 12) as u16 }); + } + + Err(()) + } +} + +impl From<ShiftedImmediate> for u32 { + /// Encode a bitmask immediate into a 32-bit value. + fn from(imm: ShiftedImmediate) -> Self { + 0 + | (((imm.shift as u32) & 1) << 12) + | (imm.value as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_no_shift() { + let expected_value = 256; + let result = ShiftedImmediate::try_from(expected_value); + + match result { + Ok(ShiftedImmediate { shift: Shift::LSL0, value }) => assert_eq!(value as u64, expected_value), + _ => panic!("Unexpected shift value") + } + } + + #[test] + fn test_maximum_no_shift() { + let expected_value = (1 << 12) - 1; + let result = ShiftedImmediate::try_from(expected_value); + + match result { + Ok(ShiftedImmediate { shift: Shift::LSL0, value }) => assert_eq!(value as u64, expected_value), + _ => panic!("Unexpected shift value") + } + } + + #[test] + fn test_with_shift() { + let result = ShiftedImmediate::try_from(256 << 12); + + assert!(matches!(result, Ok(ShiftedImmediate { shift: Shift::LSL12, value: 256 }))); + } + + #[test] + fn test_unencodable() { + let result = ShiftedImmediate::try_from((256 << 12) + 1); + assert!(matches!(result, Err(()))); + } +} diff --git a/zjit/src/asm/arm64/arg/sys_reg.rs b/zjit/src/asm/arm64/arg/sys_reg.rs new file mode 100644 index 0000000000..6229d5c1fd --- /dev/null +++ b/zjit/src/asm/arm64/arg/sys_reg.rs @@ -0,0 +1,6 @@ +/// The encoded representation of an A64 system register. +/// <https://developer.arm.com/documentation/ddi0601/2022-06/AArch64-Registers/> +pub enum SystemRegister { + /// <https://developer.arm.com/documentation/ddi0601/2022-06/AArch64-Registers/NZCV--Condition-Flags?lang=en> + NZCV = 0b1_011_0100_0010_000 +} diff --git a/zjit/src/asm/arm64/arg/truncate.rs b/zjit/src/asm/arm64/arg/truncate.rs new file mode 100644 index 0000000000..85d56ff202 --- /dev/null +++ b/zjit/src/asm/arm64/arg/truncate.rs @@ -0,0 +1,66 @@ +// There are many instances in AArch64 instruction encoding where you represent +// an integer value with a particular bit width that isn't a power of 2. These +// functions represent truncating those integer values down to the appropriate +// number of bits. + +/// Truncate a signed immediate to fit into a compile-time known width. It is +/// assumed before calling this function that the value fits into the correct +/// size. If it doesn't, then this function will panic. +/// +/// When the value is positive, this should effectively be a no-op since we're +/// just dropping leading zeroes. When the value is negative we should only be +/// dropping leading ones. +pub fn truncate_imm<T: Into<i32>, const WIDTH: usize>(imm: T) -> u32 { + let value: i32 = imm.into(); + let masked = (value as u32) & ((1 << WIDTH) - 1); + + // Assert that we didn't drop any bits by truncating. + if value >= 0 { + assert_eq!(value as u32, masked); + } else { + assert_eq!(value as u32, masked | (u32::MAX << WIDTH)); + } + + masked +} + +/// Truncate an unsigned immediate to fit into a compile-time known width. It is +/// assumed before calling this function that the value fits into the correct +/// size. If it doesn't, then this function will panic. +/// +/// This should effectively be a no-op since we're just dropping leading zeroes. +pub fn truncate_uimm<T: Into<u32>, const WIDTH: usize>(uimm: T) -> u32 { + let value: u32 = uimm.into(); + let masked = value & ((1 << WIDTH) - 1); + + // Assert that we didn't drop any bits by truncating. + assert_eq!(value, masked); + + masked +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_truncate_imm_positive() { + let inst = truncate_imm::<i32, 4>(5); + let result: u32 = inst; + assert_eq!(0b0101, result); + } + + #[test] + fn test_truncate_imm_negative() { + let inst = truncate_imm::<i32, 4>(-5); + let result: u32 = inst; + assert_eq!(0b1011, result); + } + + #[test] + fn test_truncate_uimm() { + let inst = truncate_uimm::<u32, 4>(5); + let result: u32 = inst; + assert_eq!(0b0101, result); + } +} diff --git a/zjit/src/asm/arm64/inst/atomic.rs b/zjit/src/asm/arm64/inst/atomic.rs new file mode 100644 index 0000000000..0917a4fd1c --- /dev/null +++ b/zjit/src/asm/arm64/inst/atomic.rs @@ -0,0 +1,86 @@ +/// The size of the register operands to this instruction. +enum Size { + /// Using 32-bit registers. + Size32 = 0b10, + + /// Using 64-bit registers. + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 atomic instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 1 1 1 0 0 0 0 0 0 | +/// | size rs.............. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Atomic { + /// The register holding the value to be loaded. + rt: u8, + + /// The base register. + rn: u8, + + /// The register holding the data value to be operated on. + rs: u8, + + /// The size of the registers used in this instruction. + size: Size +} + +impl Atomic { + /// LDADDAL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDADD--LDADDA--LDADDAL--LDADDL--Atomic-add-on-word-or-doubleword-in-memory-?lang=en> + pub fn ldaddal(rs: u8, rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<Atomic> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Atomic) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | (0b111 << 21) + | ((inst.rs as u32) << 16) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<Atomic> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Atomic) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldaddal() { + let result: u32 = Atomic::ldaddal(20, 21, 22, 64).into(); + assert_eq!(0xf8f402d5, result); + } +} diff --git a/zjit/src/asm/arm64/inst/branch.rs b/zjit/src/asm/arm64/inst/branch.rs new file mode 100644 index 0000000000..2db52e5d31 --- /dev/null +++ b/zjit/src/asm/arm64/inst/branch.rs @@ -0,0 +1,100 @@ +/// Which operation to perform. +enum Op { + /// Perform a BR instruction. + BR = 0b00, + + /// Perform a BLR instruction. + BLR = 0b01, + + /// Perform a RET instruction. + RET = 0b10 +} + +/// The struct that represents an A64 branch instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 1 0 0 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 | +/// | op... rn.............. rm.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Branch { + /// The register holding the address to be branched to. + rn: u8, + + /// The operation to perform. + op: Op +} + +impl Branch { + /// BR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/BR--Branch-to-Register-?lang=en> + pub fn br(rn: u8) -> Self { + Self { rn, op: Op::BR } + } + + /// BLR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/BLR--Branch-with-Link-to-Register-?lang=en> + pub fn blr(rn: u8) -> Self { + Self { rn, op: Op::BLR } + } + + /// RET + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/RET--Return-from-subroutine-?lang=en> + pub fn ret(rn: u8) -> Self { + Self { rn, op: Op::RET } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<Branch> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Branch) -> Self { + 0 + | (0b11 << 30) + | (FAMILY << 26) + | (1 << 25) + | ((inst.op as u32) << 21) + | (0b11111 << 16) + | ((inst.rn as u32) << 5) + } +} + +impl From<Branch> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Branch) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_br() { + let result: u32 = Branch::br(0).into(); + assert_eq!(0xd61f0000, result); + } + + #[test] + fn test_blr() { + let result: u32 = Branch::blr(0).into(); + assert_eq!(0xd63f0000, result); + } + + #[test] + fn test_ret() { + let result: u32 = Branch::ret(30).into(); + assert_eq!(0xd65f03c0, result); + } + + #[test] + fn test_ret_rn() { + let result: u32 = Branch::ret(20).into(); + assert_eq!(0xd65f0280, result); + } +} diff --git a/zjit/src/asm/arm64/inst/branch_cond.rs b/zjit/src/asm/arm64/inst/branch_cond.rs new file mode 100644 index 0000000000..266e9ccb31 --- /dev/null +++ b/zjit/src/asm/arm64/inst/branch_cond.rs @@ -0,0 +1,78 @@ +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The struct that represents an A64 conditional branch instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 1 0 0 0 | +/// | imm19........................................................... cond....... | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct BranchCond { + /// The kind of condition to check before branching. + cond: u8, + + /// The instruction offset from this instruction to branch to. + offset: InstructionOffset +} + +impl BranchCond { + /// B.cond + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/B-cond--Branch-conditionally-> + pub fn bcond(cond: u8, offset: InstructionOffset) -> Self { + Self { cond, offset } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<BranchCond> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: BranchCond) -> Self { + 0 + | (1 << 30) + | (FAMILY << 26) + | (truncate_imm::<_, 19>(inst.offset) << 5) + | (inst.cond as u32) + } +} + +impl From<BranchCond> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: BranchCond) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::super::arg::Condition; + + #[test] + fn test_b_eq() { + let result: u32 = BranchCond::bcond(Condition::EQ, 32.into()).into(); + assert_eq!(0x54000400, result); + } + + #[test] + fn test_b_vs() { + let result: u32 = BranchCond::bcond(Condition::VS, 32.into()).into(); + assert_eq!(0x54000406, result); + } + + #[test] + fn test_b_eq_max() { + let result: u32 = BranchCond::bcond(Condition::EQ, ((1 << 18) - 1).into()).into(); + assert_eq!(0x547fffe0, result); + } + + #[test] + fn test_b_eq_min() { + let result: u32 = BranchCond::bcond(Condition::EQ, (-(1 << 18)).into()).into(); + assert_eq!(0x54800000, result); + } +} diff --git a/zjit/src/asm/arm64/inst/breakpoint.rs b/zjit/src/asm/arm64/inst/breakpoint.rs new file mode 100644 index 0000000000..d66a35c4c6 --- /dev/null +++ b/zjit/src/asm/arm64/inst/breakpoint.rs @@ -0,0 +1,55 @@ +/// The struct that represents an A64 breakpoint instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 0 0 0 1 0 0 0 0 0 | +/// | imm16.................................................. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Breakpoint { + /// The value to be captured by ESR_ELx.ISS + imm16: u16 +} + +impl Breakpoint { + /// BRK + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/BRK--Breakpoint-instruction-> + pub fn brk(imm16: u16) -> Self { + Self { imm16 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en#control> +const FAMILY: u32 = 0b101; + +impl From<Breakpoint> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Breakpoint) -> Self { + let imm16 = inst.imm16 as u32; + + 0 + | (0b11 << 30) + | (FAMILY << 26) + | (1 << 21) + | (imm16 << 5) + } +} + +impl From<Breakpoint> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Breakpoint) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_brk() { + let result: u32 = Breakpoint::brk(7).into(); + assert_eq!(0xd42000e0, result); + } +} diff --git a/zjit/src/asm/arm64/inst/call.rs b/zjit/src/asm/arm64/inst/call.rs new file mode 100644 index 0000000000..fd26d09f8a --- /dev/null +++ b/zjit/src/asm/arm64/inst/call.rs @@ -0,0 +1,104 @@ +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The operation to perform for this instruction. +enum Op { + /// Branch directly, with a hint that this is not a subroutine call or + /// return. + Branch = 0, + + /// Branch directly, with a hint that this is a subroutine call or return. + BranchWithLink = 1 +} + +/// The struct that represents an A64 branch with our without link instruction +/// that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 0 1 | +/// | op imm26.................................................................................... | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Call { + /// The PC-relative offset to jump to in terms of number of instructions. + offset: InstructionOffset, + + /// The operation to perform for this instruction. + op: Op +} + +impl Call { + /// B + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B--Branch-> + pub fn b(offset: InstructionOffset) -> Self { + Self { offset, op: Op::Branch } + } + + /// BL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link-?lang=en> + pub fn bl(offset: InstructionOffset) -> Self { + Self { offset, op: Op::BranchWithLink } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<Call> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Call) -> Self { + 0 + | ((inst.op as u32) << 31) + | (FAMILY << 26) + | truncate_imm::<_, 26>(inst.offset) + } +} + +impl From<Call> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Call) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bl() { + let result: u32 = Call::bl(0.into()).into(); + assert_eq!(0x94000000, result); + } + + #[test] + fn test_bl_positive() { + let result: u32 = Call::bl(256.into()).into(); + assert_eq!(0x94000100, result); + } + + #[test] + fn test_bl_negative() { + let result: u32 = Call::bl((-256).into()).into(); + assert_eq!(0x97ffff00, result); + } + + #[test] + fn test_b() { + let result: u32 = Call::b(0.into()).into(); + assert_eq!(0x14000000, result); + } + + #[test] + fn test_b_positive() { + let result: u32 = Call::b(((1 << 25) - 1).into()).into(); + assert_eq!(0x15ffffff, result); + } + + #[test] + fn test_b_negative() { + let result: u32 = Call::b((-(1 << 25)).into()).into(); + assert_eq!(0x16000000, result); + } +} diff --git a/zjit/src/asm/arm64/inst/conditional.rs b/zjit/src/asm/arm64/inst/conditional.rs new file mode 100644 index 0000000000..1e26c7408b --- /dev/null +++ b/zjit/src/asm/arm64/inst/conditional.rs @@ -0,0 +1,73 @@ +use super::super::arg::Sf; + +/// The struct that represents an A64 conditional instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 1 0 1 0 1 0 0 0 0 | +/// | sf rm.............. cond....... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Conditional { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The condition to use for the conditional instruction. + cond: u8, + + /// The number of the second general-purpose source register. + rm: u8, + + /// The size of the registers of this instruction. + sf: Sf +} + +impl Conditional { + /// CSEL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CSEL--Conditional-Select-?lang=en> + pub fn csel(rd: u8, rn: u8, rm: u8, cond: u8, num_bits: u8) -> Self { + Self { rd, rn, cond, rm, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en#condsel> +const FAMILY: u32 = 0b101; + +impl From<Conditional> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Conditional) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (1 << 28) + | (FAMILY << 25) + | (1 << 23) + | ((inst.rm as u32) << 16) + | ((inst.cond as u32) << 12) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<Conditional> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Conditional) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::super::arg::Condition; + + #[test] + fn test_csel() { + let result: u32 = Conditional::csel(0, 1, 2, Condition::NE, 64).into(); + assert_eq!(0x9a821020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/data_imm.rs b/zjit/src/asm/arm64/inst/data_imm.rs new file mode 100644 index 0000000000..ea71705478 --- /dev/null +++ b/zjit/src/asm/arm64/inst/data_imm.rs @@ -0,0 +1,143 @@ +use super::super::arg::{Sf, ShiftedImmediate}; + +/// The operation being performed by this instruction. +enum Op { + Add = 0b0, + Sub = 0b1 +} + +// Whether or not to update the flags when this instruction is performed. +enum S { + LeaveFlags = 0b0, + UpdateFlags = 0b1 +} + +/// The struct that represents an A64 data processing -- immediate instruction +/// that can be encoded. +/// +/// Add/subtract (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 0 1 0 | +/// | sf op S sh imm12.................................... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct DataImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// How much to shift the immediate by. + imm: ShiftedImmediate, + + /// Whether or not to update the flags when this instruction is performed. + s: S, + + /// The opcode for this instruction. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl DataImm { + /// ADD (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADD--immediate---Add--immediate--?lang=en> + pub fn add(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::LeaveFlags, op: Op::Add, sf: num_bits.into() } + } + + /// ADDS (immediate, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADDS--immediate---Add--immediate---setting-flags-?lang=en> + pub fn adds(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::UpdateFlags, op: Op::Add, sf: num_bits.into() } + } + + /// CMP (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CMP--immediate---Compare--immediate---an-alias-of-SUBS--immediate--?lang=en> + pub fn cmp(rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self::subs(31, rn, imm, num_bits) + } + + /// SUB (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUB--immediate---Subtract--immediate--?lang=en> + pub fn sub(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::LeaveFlags, op: Op::Sub, sf: num_bits.into() } + } + + /// SUBS (immediate, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUBS--immediate---Subtract--immediate---setting-flags-?lang=en> + pub fn subs(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::UpdateFlags, op: Op::Sub, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<DataImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: DataImm) -> Self { + let imm: u32 = inst.imm.into(); + + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 30) + | ((inst.s as u32) << 29) + | (FAMILY << 25) + | (1 << 24) + | (imm << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<DataImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: DataImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + let inst = DataImm::add(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0x91001c20, result); + } + + #[test] + fn test_adds() { + let inst = DataImm::adds(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb1001c20, result); + } + + #[test] + fn test_cmp() { + let inst = DataImm::cmp(0, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf1001c1f, result); + } + + #[test] + fn test_sub() { + let inst = DataImm::sub(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xd1001c20, result); + } + + #[test] + fn test_subs() { + let inst = DataImm::subs(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf1001c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/data_reg.rs b/zjit/src/asm/arm64/inst/data_reg.rs new file mode 100644 index 0000000000..ed4afa956b --- /dev/null +++ b/zjit/src/asm/arm64/inst/data_reg.rs @@ -0,0 +1,192 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// The operation being performed by this instruction. +enum Op { + Add = 0b0, + Sub = 0b1 +} + +// Whether or not to update the flags when this instruction is performed. +enum S { + LeaveFlags = 0b0, + UpdateFlags = 0b1 +} + +/// The type of shift to perform on the second operand register. +enum Shift { + LSL = 0b00, // logical shift left (unsigned) + LSR = 0b01, // logical shift right (unsigned) + ASR = 0b10 // arithmetic shift right (signed) +} + +/// The struct that represents an A64 data processing -- register instruction +/// that can be encoded. +/// +/// Add/subtract (shifted register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 1 0 | +/// | sf op S shift rm.............. imm6............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct DataReg { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The amount to shift the second operand register by. + imm6: u8, + + /// The register number of the second operand register. + rm: u8, + + /// The type of shift to perform on the second operand register. + shift: Shift, + + /// Whether or not to update the flags when this instruction is performed. + s: S, + + /// The opcode for this instruction. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl DataReg { + /// ADD (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADD--shifted-register---Add--shifted-register--?lang=en> + pub fn add(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::LeaveFlags, + op: Op::Add, + sf: num_bits.into() + } + } + + /// ADDS (shifted register, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADDS--shifted-register---Add--shifted-register---setting-flags-?lang=en> + pub fn adds(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::UpdateFlags, + op: Op::Add, + sf: num_bits.into() + } + } + + /// CMP (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CMP--shifted-register---Compare--shifted-register---an-alias-of-SUBS--shifted-register--?lang=en> + pub fn cmp(rn: u8, rm: u8, num_bits: u8) -> Self { + Self::subs(31, rn, rm, num_bits) + } + + /// SUB (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUB--shifted-register---Subtract--shifted-register--?lang=en> + pub fn sub(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::LeaveFlags, + op: Op::Sub, + sf: num_bits.into() + } + } + + /// SUBS (shifted register, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUBS--shifted-register---Subtract--shifted-register---setting-flags-?lang=en> + pub fn subs(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::UpdateFlags, + op: Op::Sub, + sf: num_bits.into() + } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en> +const FAMILY: u32 = 0b0101; + +impl From<DataReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: DataReg) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 30) + | ((inst.s as u32) << 29) + | (FAMILY << 25) + | (1 << 24) + | ((inst.shift as u32) << 22) + | ((inst.rm as u32) << 16) + | (truncate_uimm::<_, 6>(inst.imm6) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<DataReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: DataReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + let inst = DataReg::add(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0x8b020020, result); + } + + #[test] + fn test_adds() { + let inst = DataReg::adds(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xab020020, result); + } + + #[test] + fn test_cmp() { + let inst = DataReg::cmp(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xeb01001f, result); + } + + #[test] + fn test_sub() { + let inst = DataReg::sub(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xcb020020, result); + } + + #[test] + fn test_subs() { + let inst = DataReg::subs(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xeb020020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/halfword_imm.rs b/zjit/src/asm/arm64/inst/halfword_imm.rs new file mode 100644 index 0000000000..863ac947dd --- /dev/null +++ b/zjit/src/asm/arm64/inst/halfword_imm.rs @@ -0,0 +1,179 @@ +use super::super::arg::truncate_imm; + +/// Whether this is a load or a store. +enum Op { + Load = 1, + Store = 0 +} + +/// The type of indexing to perform for this instruction. +enum Index { + /// No indexing. + None = 0b00, + + /// Mutate the register after the read. + PostIndex = 0b01, + + /// Mutate the register before the read. + PreIndex = 0b11 +} + +/// The struct that represents an A64 halfword instruction that can be encoded. +/// +/// LDRH/STRH +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 1 1 0 0 1 0 | +/// | op imm12.................................... rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +/// LDRH (pre-index/post-index) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 1 1 0 0 0 0 0 | +/// | op imm9.......................... index rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct HalfwordImm { + /// The number of the 32-bit register to be loaded. + rt: u8, + + /// The number of the 64-bit base register to calculate the memory address. + rn: u8, + + /// The type of indexing to perform for this instruction. + index: Index, + + /// The immediate offset from the base register. + imm: i16, + + /// The operation to perform. + op: Op +} + +impl HalfwordImm { + /// LDRH + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh(rt: u8, rn: u8, imm12: i16) -> Self { + Self { rt, rn, index: Index::None, imm: imm12, op: Op::Load } + } + + /// LDRH (pre-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh_pre(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PreIndex, imm: imm9, op: Op::Load } + } + + /// LDRH (post-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh_post(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PostIndex, imm: imm9, op: Op::Load } + } + + /// STRH + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh(rt: u8, rn: u8, imm12: i16) -> Self { + Self { rt, rn, index: Index::None, imm: imm12, op: Op::Store } + } + + /// STRH (pre-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh_pre(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PreIndex, imm: imm9, op: Op::Store } + } + + /// STRH (post-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh_post(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PostIndex, imm: imm9, op: Op::Store } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b111100; + +impl From<HalfwordImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: HalfwordImm) -> Self { + let (opc, imm) = match inst.index { + Index::None => { + assert_eq!(inst.imm & 1, 0, "immediate offset must be even"); + let imm12 = truncate_imm::<_, 12>(inst.imm / 2); + (0b100, imm12) + }, + Index::PreIndex | Index::PostIndex => { + let imm9 = truncate_imm::<_, 9>(inst.imm); + (0b000, (imm9 << 2) | (inst.index as u32)) + } + }; + + 0 + | (FAMILY << 25) + | ((opc | (inst.op as u32)) << 22) + | (imm << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<HalfwordImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: HalfwordImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldrh() { + let inst = HalfwordImm::ldrh(0, 1, 8); + let result: u32 = inst.into(); + assert_eq!(0x79401020, result); + } + + #[test] + fn test_ldrh_pre() { + let inst = HalfwordImm::ldrh_pre(0, 1, 16); + let result: u32 = inst.into(); + assert_eq!(0x78410c20, result); + } + + #[test] + fn test_ldrh_post() { + let inst = HalfwordImm::ldrh_post(0, 1, 24); + let result: u32 = inst.into(); + assert_eq!(0x78418420, result); + } + + #[test] + fn test_ldrh_post_negative() { + let inst = HalfwordImm::ldrh_post(0, 1, -24); + let result: u32 = inst.into(); + assert_eq!(0x785e8420, result); + } + + #[test] + fn test_strh() { + let inst = HalfwordImm::strh(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x79000020, result); + } + + #[test] + fn test_strh_pre() { + let inst = HalfwordImm::strh_pre(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78000c20, result); + } + + #[test] + fn test_strh_post() { + let inst = HalfwordImm::strh_post(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78000420, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_literal.rs b/zjit/src/asm/arm64/inst/load_literal.rs new file mode 100644 index 0000000000..37b5f3c7a7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_literal.rs @@ -0,0 +1,91 @@ +#![allow(clippy::identity_op)] + +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The size of the operands being operated on. +enum Opc { + Size32 = 0b00, + Size64 = 0b01, +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Opc { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Opc::Size64, + 32 => Opc::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load literal instruction that can be encoded. +/// +/// LDR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 0 0 0 | +/// | opc.. imm19........................................................... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadLiteral { + /// The number of the register to load the value into. + rt: u8, + + /// The PC-relative number of instructions to load the value from. + offset: InstructionOffset, + + /// The size of the operands being operated on. + opc: Opc +} + +impl LoadLiteral { + /// LDR (load literal) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--literal---Load-Register--literal--?lang=en> + pub fn ldr_literal(rt: u8, offset: InstructionOffset, num_bits: u8) -> Self { + Self { rt, offset, opc: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadLiteral> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadLiteral) -> Self { + 0 + | ((inst.opc as u32) << 30) + | (1 << 28) + | (FAMILY << 25) + | (truncate_imm::<_, 19>(inst.offset) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadLiteral> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadLiteral) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr_positive() { + let inst = LoadLiteral::ldr_literal(0, 5.into(), 64); + let result: u32 = inst.into(); + assert_eq!(0x580000a0, result); + } + + #[test] + fn test_ldr_negative() { + let inst = LoadLiteral::ldr_literal(0, (-5).into(), 64); + let result: u32 = inst.into(); + assert_eq!(0x58ffff60, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_register.rs b/zjit/src/asm/arm64/inst/load_register.rs new file mode 100644 index 0000000000..80813ffc87 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_register.rs @@ -0,0 +1,108 @@ +/// Whether or not to shift the register. +enum S { + Shift = 1, + NoShift = 0 +} + +/// The option for this instruction. +enum Option { + UXTW = 0b010, + LSL = 0b011, + SXTW = 0b110, + SXTX = 0b111 +} + +/// The size of the operands of this instruction. +enum Size { + Size32 = 0b10, + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load instruction that can be encoded. +/// +/// LDR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 0 1 1 1 0 | +/// | size. rm.............. option.. S rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadRegister { + /// The number of the register to load the value into. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// Whether or not to shift the value of the register. + s: S, + + /// The option associated with this instruction that controls the shift. + option: Option, + + /// The number of the offset register. + rm: u8, + + /// The size of the operands. + size: Size +} + +impl LoadRegister { + /// LDR + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--register---Load-Register--register--?lang=en> + pub fn ldr(rt: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rt, rn, s: S::NoShift, option: Option::LSL, rm, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadRegister> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadRegister) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | (0b11 << 21) + | ((inst.rm as u32) << 16) + | ((inst.option as u32) << 13) + | ((inst.s as u32) << 12) + | (0b10 << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadRegister> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadRegister) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr() { + let inst = LoadRegister::ldr(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8626820, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_store.rs b/zjit/src/asm/arm64/inst/load_store.rs new file mode 100644 index 0000000000..d38e851ed7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_store.rs @@ -0,0 +1,255 @@ +use super::super::arg::truncate_imm; + +/// The size of the operands being operated on. +enum Size { + Size8 = 0b00, + Size16 = 0b01, + Size32 = 0b10, + Size64 = 0b11, +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The operation to perform for this instruction. +enum Opc { + STR = 0b00, + LDR = 0b01, + LDURSW = 0b10 +} + +/// What kind of indexing to perform for this instruction. +enum Index { + None = 0b00, + PostIndex = 0b01, + PreIndex = 0b11 +} + +/// The struct that represents an A64 load or store instruction that can be +/// encoded. +/// +/// LDR/LDUR/LDURSW/STR/STUR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 0 | +/// | size. opc.. imm9.......................... idx.. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadStore { + /// The number of the register to load the value into. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// What kind of indexing to perform for this instruction. + idx: Index, + + /// The optional signed immediate byte offset from the base register. + imm9: i16, + + /// The operation to perform for this instruction. + opc: Opc, + + /// The size of the operands being operated on. + size: Size +} + +impl LoadStore { + /// LDR (immediate, post-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDR--immediate---Load-Register--immediate--> + pub fn ldr_post(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PostIndex, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDR (immediate, pre-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDR--immediate---Load-Register--immediate--> + pub fn ldr_pre(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PreIndex, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDUR (load register, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled--?lang=en> + pub fn ldur(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDURH Load Register Halfword (unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURH--Load-Register-Halfword--unscaled--?lang=en> + pub fn ldurh(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: Size::Size16 } + } + + /// LDURB (load register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURB--Load-Register-Byte--unscaled--?lang=en> + pub fn ldurb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: Size::Size8 } + } + + /// LDURSW (load register, unscaled, signed) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURSW--Load-Register-Signed-Word--unscaled--?lang=en> + pub fn ldursw(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDURSW, size: Size::Size32 } + } + + /// STR (immediate, post-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STR--immediate---Store-Register--immediate--> + pub fn str_post(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PostIndex, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STR (immediate, pre-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STR--immediate---Store-Register--immediate--> + pub fn str_pre(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PreIndex, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STUR (store register, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STUR--Store-Register--unscaled--?lang=en> + pub fn stur(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STURH (store register, halfword, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturh(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size16 } + } + + /// STURB (store register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size8 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadStore> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadStore) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | ((inst.opc as u32) << 22) + | (truncate_imm::<_, 9>(inst.imm9) << 12) + | ((inst.idx as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadStore> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadStore) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr_post() { + let inst = LoadStore::ldr_post(0, 1, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8410420, result); + } + + #[test] + fn test_ldr_pre() { + let inst = LoadStore::ldr_pre(0, 1, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8410c20, result); + } + + #[test] + fn test_ldur() { + let inst = LoadStore::ldur(0, 1, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8400020, result); + } + + #[test] + fn test_ldurb() { + let inst = LoadStore::ldurb(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x38400020, result); + } + + #[test] + fn test_ldurh() { + let inst = LoadStore::ldurh(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78400020, result); + } + + #[test] + fn test_ldur_with_imm() { + let inst = LoadStore::ldur(0, 1, 123, 64); + let result: u32 = inst.into(); + assert_eq!(0xf847b020, result); + } + + #[test] + fn test_ldursw() { + let inst = LoadStore::ldursw(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0xb8800020, result); + } + + #[test] + fn test_ldursw_with_imm() { + let inst = LoadStore::ldursw(0, 1, 123); + let result: u32 = inst.into(); + assert_eq!(0xb887b020, result); + } + + #[test] + fn test_str_post() { + let inst = LoadStore::str_post(0, 1, -16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81f0420, result); + } + + #[test] + fn test_str_pre() { + let inst = LoadStore::str_pre(0, 1, -16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81f0c20, result); + } + + #[test] + fn test_stur() { + let inst = LoadStore::stur(0, 1, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8000020, result); + } + + #[test] + fn test_stur_negative_offset() { + let inst = LoadStore::stur(0, 1, -1, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81ff020, result); + } + + #[test] + fn test_stur_positive_offset() { + let inst = LoadStore::stur(0, 1, 255, 64); + let result: u32 = inst.into(); + assert_eq!(0xf80ff020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_store_exclusive.rs b/zjit/src/asm/arm64/inst/load_store_exclusive.rs new file mode 100644 index 0000000000..30cb663bdb --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_store_exclusive.rs @@ -0,0 +1,109 @@ +/// The operation being performed for this instruction. +enum Op { + Store = 0, + Load = 1 +} + +/// The size of the registers being operated on. +enum Size { + Size32 = 0b10, + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load or store exclusive instruction that +/// can be encoded. +/// +/// LDAXR/STLXR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 0 0 0 0 1 1 1 1 1 1 | +/// | size. op rs.............. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadStoreExclusive { + /// The number of the register to be loaded. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// The register to be used for the status result if it applies to this + /// operation. Otherwise it's the zero register. + rs: u8, + + /// The operation being performed for this instruction. + op: Op, + + /// The size of the registers being operated on. + size: Size +} + +impl LoadStoreExclusive { + /// LDAXR + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/LDAXR--Load-Acquire-Exclusive-Register-> + pub fn ldaxr(rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs: 31, op: Op::Load, size: num_bits.into() } + } + + /// STLXR + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/STLXR--Store-Release-Exclusive-Register-> + pub fn stlxr(rs: u8, rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs, op: Op::Store, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadStoreExclusive> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadStoreExclusive) -> Self { + 0 + | ((inst.size as u32) << 30) + | (FAMILY << 25) + | ((inst.op as u32) << 22) + | ((inst.rs as u32) << 16) + | (0b111111 << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadStoreExclusive> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadStoreExclusive) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldaxr() { + let inst = LoadStoreExclusive::ldaxr(16, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xc85ffc10, result); + } + + #[test] + fn test_stlxr() { + let inst = LoadStoreExclusive::stlxr(17, 16, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xc811fc10, result); + } +} diff --git a/zjit/src/asm/arm64/inst/logical_imm.rs b/zjit/src/asm/arm64/inst/logical_imm.rs new file mode 100644 index 0000000000..d57ad5f5b7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/logical_imm.rs @@ -0,0 +1,154 @@ +use super::super::arg::{BitmaskImmediate, Sf}; + +// Which operation to perform. +enum Opc { + /// The AND operation. + And = 0b00, + + /// The ORR operation. + Orr = 0b01, + + /// The EOR operation. + Eor = 0b10, + + /// The ANDS operation. + Ands = 0b11 +} + +/// The struct that represents an A64 bitwise immediate instruction that can be +/// encoded. +/// +/// AND/ORR/ANDS (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 0 | +/// | sf opc.. N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LogicalImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The immediate value to test. + imm: BitmaskImmediate, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl LogicalImm { + /// AND (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/AND--immediate---Bitwise-AND--immediate--?lang=en> + pub fn and(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::And, sf: num_bits.into() } + } + + /// ANDS (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ANDS--immediate---Bitwise-AND--immediate---setting-flags-?lang=en> + pub fn ands(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Ands, sf: num_bits.into() } + } + + /// EOR (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/EOR--immediate---Bitwise-Exclusive-OR--immediate--> + pub fn eor(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Eor, sf: num_bits.into() } + } + + /// MOV (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MOV--bitmask-immediate---Move--bitmask-immediate---an-alias-of-ORR--immediate--?lang=en> + pub fn mov(rd: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORR (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORR--immediate---Bitwise-OR--immediate--> + pub fn orr(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Orr, sf: num_bits.into() } + } + + /// TST (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TST--immediate---Test-bits--immediate---an-alias-of-ANDS--immediate--?lang=en> + pub fn tst(rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self::ands(31, rn, imm, num_bits) + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#log_imm> +const FAMILY: u32 = 0b1001; + +impl From<LogicalImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LogicalImm) -> Self { + let imm: u32 = inst.imm.encode(); + + 0 + | ((inst.sf as u32) << 31) + | ((inst.opc as u32) << 29) + | (FAMILY << 25) + | (imm << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<LogicalImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LogicalImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_and() { + let inst = LogicalImm::and(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0x92400820, result); + } + + #[test] + fn test_ands() { + let inst = LogicalImm::ands(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf2400820, result); + } + + #[test] + fn test_eor() { + let inst = LogicalImm::eor(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xd2400820, result); + } + + #[test] + fn test_mov() { + let inst = LogicalImm::mov(0, 0x5555555555555555.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb200f3e0, result); + } + + #[test] + fn test_orr() { + let inst = LogicalImm::orr(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb2400820, result); + } + + #[test] + fn test_tst() { + let inst = LogicalImm::tst(1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf240083f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/logical_reg.rs b/zjit/src/asm/arm64/inst/logical_reg.rs new file mode 100644 index 0000000000..18edff606f --- /dev/null +++ b/zjit/src/asm/arm64/inst/logical_reg.rs @@ -0,0 +1,207 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// Whether or not this is a NOT instruction. +enum N { + /// This is not a NOT instruction. + No = 0, + + /// This is a NOT instruction. + Yes = 1 +} + +/// The type of shift to perform on the second operand register. +enum Shift { + LSL = 0b00, // logical shift left (unsigned) + LSR = 0b01, // logical shift right (unsigned) + ASR = 0b10, // arithmetic shift right (signed) + ROR = 0b11 // rotate right (unsigned) +} + +// Which operation to perform. +enum Opc { + /// The AND operation. + And = 0b00, + + /// The ORR operation. + Orr = 0b01, + + /// The EOR operation. + Eor = 0b10, + + /// The ANDS operation. + Ands = 0b11 +} + +/// The struct that represents an A64 logical register instruction that can be +/// encoded. +/// +/// AND/ORR/ANDS (shifted register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 | +/// | sf opc.. shift N rm.............. imm6............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LogicalReg { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The amount to shift the second operand register. + imm6: u8, + + /// The register number of the second operand register. + rm: u8, + + /// Whether or not this is a NOT instruction. + n: N, + + /// The type of shift to perform on the second operand register. + shift: Shift, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl LogicalReg { + /// AND (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/AND--shifted-register---Bitwise-AND--shifted-register--?lang=en> + pub fn and(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::And, sf: num_bits.into() } + } + + /// ANDS (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ANDS--shifted-register---Bitwise-AND--shifted-register---setting-flags-?lang=en> + pub fn ands(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Ands, sf: num_bits.into() } + } + + /// EOR (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/EOR--shifted-register---Bitwise-Exclusive-OR--shifted-register--> + pub fn eor(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Eor, sf: num_bits.into() } + } + + /// MOV (register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register--?lang=en> + pub fn mov(rd: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// MVN (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MVN--Bitwise-NOT--an-alias-of-ORN--shifted-register--?lang=en> + pub fn mvn(rd: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm6: 0, rm, n: N::Yes, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORN (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORN--shifted-register---Bitwise-OR-NOT--shifted-register--> + pub fn orn(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::Yes, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORR (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORR--shifted-register---Bitwise-OR--shifted-register--> + pub fn orr(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// TST (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TST--shifted-register---Test--shifted-register---an-alias-of-ANDS--shifted-register--?lang=en> + pub fn tst(rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd: 31, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Ands, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en> +const FAMILY: u32 = 0b0101; + +impl From<LogicalReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LogicalReg) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.opc as u32) << 29) + | (FAMILY << 25) + | ((inst.shift as u32) << 22) + | ((inst.n as u32) << 21) + | ((inst.rm as u32) << 16) + | (truncate_uimm::<_, 6>(inst.imm6) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<LogicalReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LogicalReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_and() { + let inst = LogicalReg::and(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0x8a020020, result); + } + + #[test] + fn test_ands() { + let inst = LogicalReg::ands(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xea020020, result); + } + + #[test] + fn test_eor() { + let inst = LogicalReg::eor(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xca020020, result); + } + + #[test] + fn test_mov() { + let inst = LogicalReg::mov(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa0103e0, result); + } + + #[test] + fn test_mvn() { + let inst = LogicalReg::mvn(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa2103e0, result); + } + + #[test] + fn test_orn() { + let inst = LogicalReg::orn(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa220020, result); + } + + #[test] + fn test_orr() { + let inst = LogicalReg::orr(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa020020, result); + } + + #[test] + fn test_tst() { + let inst = LogicalReg::tst(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xea01001f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/madd.rs b/zjit/src/asm/arm64/inst/madd.rs new file mode 100644 index 0000000000..71f2ab230a --- /dev/null +++ b/zjit/src/asm/arm64/inst/madd.rs @@ -0,0 +1,73 @@ +use super::super::arg::Sf; + +/// The struct that represents an A64 multiply-add instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 1 0 1 1 0 0 0 0 | +/// | sf rm.............. ra.............. rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct MAdd { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The number of the third general-purpose source register. + ra: u8, + + /// The number of the second general-purpose source register. + rm: u8, + + /// The size of the registers of this instruction. + sf: Sf +} + +impl MAdd { + /// MUL + /// <https://developer.arm.com/documentation/ddi0602/2023-06/Base-Instructions/MUL--Multiply--an-alias-of-MADD-> + pub fn mul(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, ra: 0b11111, rm, sf: num_bits.into() } + } +} + +impl From<MAdd> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: MAdd) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (0b11011 << 24) + | ((inst.rm as u32) << 16) + | ((inst.ra as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<MAdd> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: MAdd) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mul_32() { + let result: u32 = MAdd::mul(0, 1, 2, 32).into(); + assert_eq!(0x1B027C20, result); + } + + #[test] + fn test_mul_64() { + let result: u32 = MAdd::mul(0, 1, 2, 64).into(); + assert_eq!(0x9B027C20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/mod.rs b/zjit/src/asm/arm64/inst/mod.rs new file mode 100644 index 0000000000..270c784f27 --- /dev/null +++ b/zjit/src/asm/arm64/inst/mod.rs @@ -0,0 +1,56 @@ +// This module contains various A64 instructions and the logic necessary to +// encode them into u32s. + +mod atomic; +mod branch; +mod branch_cond; +mod breakpoint; +mod call; +mod conditional; +mod data_imm; +mod data_reg; +mod halfword_imm; +mod load_literal; +mod load_register; +mod load_store; +mod load_store_exclusive; +mod logical_imm; +mod logical_reg; +mod madd; +mod smulh; +mod mov; +mod nop; +mod pc_rel; +mod reg_pair; +mod sbfm; +mod shift_imm; +mod sys_reg; +mod test_bit; +mod udf; + +pub use atomic::Atomic; +pub use branch::Branch; +pub use branch_cond::BranchCond; +pub use breakpoint::Breakpoint; +pub use call::Call; +pub use conditional::Conditional; +pub use data_imm::DataImm; +pub use data_reg::DataReg; +pub use halfword_imm::HalfwordImm; +pub use load_literal::LoadLiteral; +pub use load_register::LoadRegister; +pub use load_store::LoadStore; +pub use load_store_exclusive::LoadStoreExclusive; +pub use logical_imm::LogicalImm; +pub use logical_reg::LogicalReg; +pub use madd::MAdd; +pub use smulh::SMulH; +pub use mov::Mov; +pub use nop::Nop; +pub use pc_rel::PCRelative; +pub use reg_pair::RegisterPair; +pub use sbfm::SBFM; +pub use shift_imm::ShiftImm; +pub use sys_reg::SysReg; +pub use test_bit::TestBit; +pub use udf::Udf; diff --git a/zjit/src/asm/arm64/inst/mov.rs b/zjit/src/asm/arm64/inst/mov.rs new file mode 100644 index 0000000000..e9f9091713 --- /dev/null +++ b/zjit/src/asm/arm64/inst/mov.rs @@ -0,0 +1,192 @@ +use super::super::arg::Sf; + +/// Which operation is being performed. +enum Op { + /// A movn operation which inverts the immediate and zeroes out the other bits. + MOVN = 0b00, + + /// A movz operation which zeroes out the other bits. + MOVZ = 0b10, + + /// A movk operation which keeps the other bits in place. + MOVK = 0b11 +} + +/// How much to shift the immediate by. +enum Hw { + LSL0 = 0b00, + LSL16 = 0b01, + LSL32 = 0b10, + LSL48 = 0b11 +} + +impl From<u8> for Hw { + fn from(shift: u8) -> Self { + match shift { + 0 => Hw::LSL0, + 16 => Hw::LSL16, + 32 => Hw::LSL32, + 48 => Hw::LSL48, + _ => panic!("Invalid value for shift: {shift}"), + } + } +} + +/// The struct that represents a MOVK or MOVZ instruction. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 1 | +/// | sf op... hw... imm16.................................................. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Mov { + /// The register number of the destination register. + rd: u8, + + /// The value to move into the register. + imm16: u16, + + /// The shift of the value to move. + hw: Hw, + + /// Which operation is being performed. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl Mov { + /// MOVK + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MOVK--Move-wide-with-keep-?lang=en> + pub fn movk(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVK, sf: num_bits.into() } + } + + /// MOVN + /// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> + pub fn movn(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVN, sf: num_bits.into() } + } + + /// MOVZ + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MOVZ--Move-wide-with-zero-?lang=en> + pub fn movz(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVZ, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<Mov> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Mov) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 29) + | (FAMILY << 25) + | (0b101 << 23) + | ((inst.hw as u32) << 21) + | ((inst.imm16 as u32) << 5) + | inst.rd as u32 + } +} + +impl From<Mov> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Mov) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_movk_unshifted() { + let inst = Mov::movk(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2800f60, result); + } + + #[test] + fn test_movn_unshifted() { + let inst = Mov::movn(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0x92800f60, result); + } + + #[test] + fn test_movn_shifted_16() { + let inst = Mov::movn(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0x92a00f60, result); + } + + #[test] + fn test_movn_shifted_32() { + let inst = Mov::movn(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0x92c00f60, result); + } + + #[test] + fn test_movn_shifted_48() { + let inst = Mov::movn(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0x92e00f60, result); + } + + #[test] + fn test_movk_shifted_16() { + let inst = Mov::movk(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2a00f60, result); + } + + #[test] + fn test_movk_shifted_32() { + let inst = Mov::movk(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2c00f60, result); + } + + #[test] + fn test_movk_shifted_48() { + let inst = Mov::movk(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2e00f60, result); + } + + #[test] + fn test_movz_unshifted() { + let inst = Mov::movz(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2800f60, result); + } + + #[test] + fn test_movz_shifted_16() { + let inst = Mov::movz(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2a00f60, result); + } + + #[test] + fn test_movz_shifted_32() { + let inst = Mov::movz(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2c00f60, result); + } + + #[test] + fn test_movz_shifted_48() { + let inst = Mov::movz(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2e00f60, result); + } +} diff --git a/zjit/src/asm/arm64/inst/nop.rs b/zjit/src/asm/arm64/inst/nop.rs new file mode 100644 index 0000000000..081d8558f5 --- /dev/null +++ b/zjit/src/asm/arm64/inst/nop.rs @@ -0,0 +1,44 @@ +/// The struct that represents an A64 nop instruction that can be encoded. +/// +/// NOP +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 1 0 0 0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 1 1 1 1 1 | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Nop; + +impl Nop { + /// NOP + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/NOP--No-Operation-> + pub fn nop() -> Self { + Self {} + } +} + +impl From<Nop> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(_inst: Nop) -> Self { + 0b11010101000000110010000000011111 + } +} + +impl From<Nop> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Nop) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_nop() { + let inst = Nop::nop(); + let result: u32 = inst.into(); + assert_eq!(0xd503201f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/pc_rel.rs b/zjit/src/asm/arm64/inst/pc_rel.rs new file mode 100644 index 0000000000..2ea586a778 --- /dev/null +++ b/zjit/src/asm/arm64/inst/pc_rel.rs @@ -0,0 +1,107 @@ +/// Which operation to perform for the PC-relative instruction. +enum Op { + /// Form a PC-relative address. + ADR = 0, + + /// Form a PC-relative address to a 4KB page. + ADRP = 1 +} + +/// The struct that represents an A64 PC-relative address instruction that can +/// be encoded. +/// +/// ADR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 0 0 | +/// | op immlo immhi........................................................... rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct PCRelative { + /// The number for the general-purpose register to load the address into. + rd: u8, + + /// The number of bytes to add to the PC to form the address. + imm: i32, + + /// Which operation to perform for this instruction. + op: Op +} + +impl PCRelative { + /// ADR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/ADR--Form-PC-relative-address-> + pub fn adr(rd: u8, imm: i32) -> Self { + Self { rd, imm, op: Op::ADR } + } + + /// ADRP + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page-> + pub fn adrp(rd: u8, imm: i32) -> Self { + Self { rd, imm: imm >> 12, op: Op::ADRP } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<PCRelative> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: PCRelative) -> Self { + let immlo = (inst.imm & 0b11) as u32; + let mut immhi = ((inst.imm >> 2) & ((1 << 18) - 1)) as u32; + + // Toggle the sign bit if necessary. + if inst.imm < 0 { + immhi |= 1 << 18; + } + + 0 + | ((inst.op as u32) << 31) + | (immlo << 29) + | (FAMILY << 25) + | (immhi << 5) + | inst.rd as u32 + } +} + +impl From<PCRelative> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: PCRelative) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_adr_positive() { + let inst = PCRelative::adr(0, 5); + let result: u32 = inst.into(); + assert_eq!(0x30000020, result); + } + + #[test] + fn test_adr_negative() { + let inst = PCRelative::adr(0, -5); + let result: u32 = inst.into(); + assert_eq!(0x70ffffc0, result); + } + + #[test] + fn test_adrp_positive() { + let inst = PCRelative::adrp(0, 0x4000); + let result: u32 = inst.into(); + assert_eq!(0x90000020, result); + } + + #[test] + fn test_adrp_negative() { + let inst = PCRelative::adrp(0, -0x4000); + let result: u32 = inst.into(); + assert_eq!(0x90ffffe0, result); + } +} diff --git a/zjit/src/asm/arm64/inst/reg_pair.rs b/zjit/src/asm/arm64/inst/reg_pair.rs new file mode 100644 index 0000000000..39a44c2416 --- /dev/null +++ b/zjit/src/asm/arm64/inst/reg_pair.rs @@ -0,0 +1,212 @@ +use super::super::arg::truncate_imm; + +/// The operation to perform for this instruction. +enum Opc { + /// When the registers are 32-bits wide. + Opc32 = 0b00, + + /// When the registers are 64-bits wide. + Opc64 = 0b10 +} + +/// The kind of indexing to perform for this instruction. +enum Index { + StorePostIndex = 0b010, + LoadPostIndex = 0b011, + StoreSignedOffset = 0b100, + LoadSignedOffset = 0b101, + StorePreIndex = 0b110, + LoadPreIndex = 0b111 +} + +/// A convenience function so that we can convert the number of bits of a +/// register operand directly into an Opc variant. +impl From<u8> for Opc { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Opc::Opc64, + 32 => Opc::Opc32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 register pair instruction that can be +/// encoded. +/// +/// STP/LDP +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 0 | +/// | opc index..... imm7.................... rt2............. rn.............. rt1............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct RegisterPair { + /// The number of the first register to be transferred. + rt1: u8, + + /// The number of the base register. + rn: u8, + + /// The number of the second register to be transferred. + rt2: u8, + + /// The signed immediate byte offset, a multiple of 8. + imm7: i16, + + /// The kind of indexing to use for this instruction. + index: Index, + + /// The operation to be performed (in terms of size). + opc: Opc +} + +impl RegisterPair { + /// Create a register pair instruction with a given indexing mode. + fn new(rt1: u8, rt2: u8, rn: u8, disp: i16, index: Index, num_bits: u8) -> Self { + Self { rt1, rn, rt2, imm7: disp / 8, index, opc: num_bits.into() } + } + + /// LDP (signed offset) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>{, #<imm>}]` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadSignedOffset, num_bits) + } + + /// LDP (pre-index) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>, #<imm>]!` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp_pre(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadPreIndex, num_bits) + } + + /// LDP (post-index) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>], #<imm>` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp_post(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadPostIndex, num_bits) + } + + /// STP (signed offset) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>{, #<imm>}]` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StoreSignedOffset, num_bits) + } + + /// STP (pre-index) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>, #<imm>]!` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp_pre(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StorePreIndex, num_bits) + } + + /// STP (post-index) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>], #<imm>` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp_post(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StorePostIndex, num_bits) + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<RegisterPair> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: RegisterPair) -> Self { + 0 + | ((inst.opc as u32) << 30) + | (1 << 29) + | (FAMILY << 25) + | ((inst.index as u32) << 22) + | (truncate_imm::<_, 7>(inst.imm7) << 15) + | ((inst.rt2 as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rt1 as u32) + } +} + +impl From<RegisterPair> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: RegisterPair) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldp() { + let inst = RegisterPair::ldp(0, 1, 2, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9400440, result); + } + + #[test] + fn test_ldp_maximum_displacement() { + let inst = RegisterPair::ldp(0, 1, 2, 504, 64); + let result: u32 = inst.into(); + assert_eq!(0xa95f8440, result); + } + + #[test] + fn test_ldp_minimum_displacement() { + let inst = RegisterPair::ldp(0, 1, 2, -512, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9600440, result); + } + + #[test] + fn test_ldp_pre() { + let inst = RegisterPair::ldp_pre(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9d00440, result); + } + + #[test] + fn test_ldp_post() { + let inst = RegisterPair::ldp_post(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa8d00440, result); + } + + #[test] + fn test_stp() { + let inst = RegisterPair::stp(0, 1, 2, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9000440, result); + } + + #[test] + fn test_stp_maximum_displacement() { + let inst = RegisterPair::stp(0, 1, 2, 504, 64); + let result: u32 = inst.into(); + assert_eq!(0xa91f8440, result); + } + + #[test] + fn test_stp_minimum_displacement() { + let inst = RegisterPair::stp(0, 1, 2, -512, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9200440, result); + } + + #[test] + fn test_stp_pre() { + let inst = RegisterPair::stp_pre(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9900440, result); + } + + #[test] + fn test_stp_post() { + let inst = RegisterPair::stp_post(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa8900440, result); + } +} diff --git a/zjit/src/asm/arm64/inst/sbfm.rs b/zjit/src/asm/arm64/inst/sbfm.rs new file mode 100644 index 0000000000..12944ba722 --- /dev/null +++ b/zjit/src/asm/arm64/inst/sbfm.rs @@ -0,0 +1,103 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// The struct that represents an A64 signed bitfield move instruction that can +/// be encoded. +/// +/// SBFM +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 0 0 1 1 0 | +/// | sf N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SBFM { + /// The number for the general-purpose register to load the value into. + rd: u8, + + /// The number for the general-purpose register to copy from. + rn: u8, + + /// The leftmost bit number to be moved from the source. + imms: u8, + + // The right rotate amount. + immr: u8, + + /// Whether or not this is a 64-bit operation. + n: bool, + + /// The size of this operation. + sf: Sf +} + +impl SBFM { + /// ASR + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ASR--immediate---Arithmetic-Shift-Right--immediate---an-alias-of-SBFM-?lang=en> + pub fn asr(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + let (imms, n) = if num_bits == 64 { + (0b111111, true) + } else { + (0b011111, false) + }; + + Self { rd, rn, immr: shift, imms, n, sf: num_bits.into() } + } + + /// SXTW + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SXTW--Sign-Extend-Word--an-alias-of-SBFM-?lang=en> + pub fn sxtw(rd: u8, rn: u8) -> Self { + Self { rd, rn, immr: 0, imms: 31, n: true, sf: Sf::Sf64 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#bitfield> +const FAMILY: u32 = 0b1001; + +impl From<SBFM> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SBFM) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (FAMILY << 25) + | (1 << 24) + | ((inst.n as u32) << 22) + | (truncate_uimm::<_, 6>(inst.immr) << 16) + | (truncate_uimm::<_, 6>(inst.imms) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<SBFM> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SBFM) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_asr_32_bits() { + let inst = SBFM::asr(0, 1, 2, 32); + let result: u32 = inst.into(); + assert_eq!(0x13027c20, result); + } + + #[test] + fn test_asr_64_bits() { + let inst = SBFM::asr(10, 11, 5, 64); + let result: u32 = inst.into(); + assert_eq!(0x9345fd6a, result); + } + + #[test] + fn test_sxtw() { + let inst = SBFM::sxtw(0, 1); + let result: u32 = inst.into(); + assert_eq!(0x93407c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/shift_imm.rs b/zjit/src/asm/arm64/inst/shift_imm.rs new file mode 100644 index 0000000000..9dac9a1408 --- /dev/null +++ b/zjit/src/asm/arm64/inst/shift_imm.rs @@ -0,0 +1,147 @@ +use super::super::arg::Sf; + +/// The operation to perform for this instruction. +enum Opc { + /// Logical left shift + LSL, + + /// Logical shift right + LSR +} + +/// The struct that represents an A64 unsigned bitfield move instruction that +/// can be encoded. +/// +/// LSL (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 1 0 0 1 1 0 | +/// | sf N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct ShiftImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The immediate value to shift by. + shift: u8, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl ShiftImm { + /// LSL (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LSL--immediate---Logical-Shift-Left--immediate---an-alias-of-UBFM-?lang=en> + pub fn lsl(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + ShiftImm { rd, rn, shift, opc: Opc::LSL, sf: num_bits.into() } + } + + /// LSR (immediate) + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/LSR--immediate---Logical-Shift-Right--immediate---an-alias-of-UBFM-?lang=en> + pub fn lsr(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + ShiftImm { rd, rn, shift, opc: Opc::LSR, sf: num_bits.into() } + } + + /// Returns a triplet of (n, immr, imms) encoded in u32s for this + /// instruction. This mirrors how they will be encoded in the actual bits. + fn bitmask(&self) -> (u32, u32, u32) { + match self.opc { + // The key insight is a little buried in the docs, but effectively: + // LSL <Wd>, <Wn>, #<shift> == UBFM <Wd>, <Wn>, #(-<shift> MOD 32), #(31-<shift>) + // LSL <Xd>, <Xn>, #<shift> == UBFM <Xd>, <Xn>, #(-<shift> MOD 64), #(63-<shift>) + Opc::LSL => { + let shift = -(self.shift as i16); + + match self.sf { + Sf::Sf32 => ( + 0, + (shift.rem_euclid(32) & 0x3f) as u32, + ((31 - self.shift) & 0x3f) as u32 + ), + Sf::Sf64 => ( + 1, + (shift.rem_euclid(64) & 0x3f) as u32, + ((63 - self.shift) & 0x3f) as u32 + ) + } + }, + // Similar to LSL: + // LSR <Wd>, <Wn>, #<shift> == UBFM <Wd>, <Wn>, #<shift>, #31 + // LSR <Xd>, <Xn>, #<shift> == UBFM <Xd>, <Xn>, #<shift>, #63 + Opc::LSR => { + match self.sf { + Sf::Sf32 => (0, (self.shift & 0x3f) as u32, 31), + Sf::Sf64 => (1, (self.shift & 0x3f) as u32, 63) + } + } + } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#bitfield> +const FAMILY: u32 = 0b10011; + +impl From<ShiftImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: ShiftImm) -> Self { + let (n, immr, imms) = inst.bitmask(); + + 0 + | ((inst.sf as u32) << 31) + | (1 << 30) + | (FAMILY << 24) + | (n << 22) + | (immr << 16) + | (imms << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<ShiftImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: ShiftImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lsl_32() { + let inst = ShiftImm::lsl(0, 1, 7, 32); + let result: u32 = inst.into(); + assert_eq!(0x53196020, result); + } + + #[test] + fn test_lsl_64() { + let inst = ShiftImm::lsl(0, 1, 7, 64); + let result: u32 = inst.into(); + assert_eq!(0xd379e020, result); + } + + #[test] + fn test_lsr_32() { + let inst = ShiftImm::lsr(0, 1, 7, 32); + let result: u32 = inst.into(); + assert_eq!(0x53077c20, result); + } + + #[test] + fn test_lsr_64() { + let inst = ShiftImm::lsr(0, 1, 7, 64); + let result: u32 = inst.into(); + assert_eq!(0xd347fc20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/smulh.rs b/zjit/src/asm/arm64/inst/smulh.rs new file mode 100644 index 0000000000..f355cb6531 --- /dev/null +++ b/zjit/src/asm/arm64/inst/smulh.rs @@ -0,0 +1,60 @@ +/// The struct that represents an A64 signed multiply high instruction +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 1 0 1 1 0 1 0 0 | +/// | rm.............. ra.............. rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SMulH { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The number of the third general-purpose source register. + ra: u8, + + /// The number of the second general-purpose source register. + rm: u8, +} + +impl SMulH { + /// SMULH + /// <https://developer.arm.com/documentation/ddi0602/2023-06/Base-Instructions/SMULH--Signed-Multiply-High-> + pub fn smulh(rd: u8, rn: u8, rm: u8) -> Self { + Self { rd, rn, ra: 0b11111, rm } + } +} + +impl From<SMulH> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SMulH) -> Self { + 0 + | (0b10011011010 << 21) + | ((inst.rm as u32) << 16) + | ((inst.ra as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<SMulH> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SMulH) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_smulh() { + let result: u32 = SMulH::smulh(0, 1, 2).into(); + assert_eq!(0x9b427c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/sys_reg.rs b/zjit/src/asm/arm64/inst/sys_reg.rs new file mode 100644 index 0000000000..7191dfbfd9 --- /dev/null +++ b/zjit/src/asm/arm64/inst/sys_reg.rs @@ -0,0 +1,86 @@ +use super::super::arg::SystemRegister; + +/// Which operation to perform (loading or storing the system register value). +enum L { + /// Store the value of a general-purpose register in a system register. + MSR = 0, + + /// Store the value of a system register in a general-purpose register. + MRS = 1 +} + +/// The struct that represents an A64 system register instruction that can be +/// encoded. +/// +/// MSR/MRS (register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 1 0 0 1 | +/// | L o0 op1..... CRn........ CRm........ op2..... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SysReg { + /// The register to load the system register value into. + rt: u8, + + /// Which system register to load or store. + systemreg: SystemRegister, + + /// Which operation to perform (loading or storing the system register value). + l: L +} + +impl SysReg { + /// MRS (register) + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MRS--Move-System-Register-?lang=en> + pub fn mrs(rt: u8, systemreg: SystemRegister) -> Self { + SysReg { rt, systemreg, l: L::MRS } + } + + /// MSR (register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MSR--register---Move-general-purpose-register-to-System-Register-?lang=en> + pub fn msr(systemreg: SystemRegister, rt: u8) -> Self { + SysReg { rt, systemreg, l: L::MSR } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en#systemmove> +const FAMILY: u32 = 0b110101010001; + +impl From<SysReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SysReg) -> Self { + 0 + | (FAMILY << 20) + | ((inst.l as u32) << 21) + | ((inst.systemreg as u32) << 5) + | inst.rt as u32 + } +} + +impl From<SysReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SysReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mrs() { + let inst = SysReg::mrs(0, SystemRegister::NZCV); + let result: u32 = inst.into(); + assert_eq!(0xd53b4200, result); + } + + #[test] + fn test_msr() { + let inst = SysReg::msr(SystemRegister::NZCV, 0); + let result: u32 = inst.into(); + assert_eq!(0xd51b4200, result); + } +} diff --git a/zjit/src/asm/arm64/inst/test_bit.rs b/zjit/src/asm/arm64/inst/test_bit.rs new file mode 100644 index 0000000000..45f0c2317e --- /dev/null +++ b/zjit/src/asm/arm64/inst/test_bit.rs @@ -0,0 +1,133 @@ +use super::super::arg::truncate_imm; + +/// The upper bit of the bit number to test. +#[derive(Debug)] +enum B5 { + /// When the bit number is below 32. + B532 = 0, + + /// When the bit number is equal to or above 32. + B564 = 1 +} + +/// A convenience function so that we can convert the bit number directly into a +/// B5 variant. +impl From<u8> for B5 { + fn from(bit_num: u8) -> Self { + match bit_num { + 0..=31 => B5::B532, + 32..=63 => B5::B564, + _ => panic!("Invalid bit number: {bit_num}"), + } + } +} + +/// The operation to perform for this instruction. +enum Op { + /// The test bit zero operation. + TBZ = 0, + + /// The test bit not zero operation. + TBNZ = 1 +} + +/// The struct that represents an A64 test bit instruction that can be encoded. +/// +/// TBNZ/TBZ +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 0 1 1 | +/// | b5 op b40............. imm14.......................................... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct TestBit { + /// The number of the register to test. + rt: u8, + + /// The PC-relative offset to the target instruction in term of number of + /// instructions. + imm14: i16, + + /// The lower 5 bits of the bit number to be tested. + b40: u8, + + /// The operation to perform for this instruction. + op: Op, + + /// The upper bit of the bit number to test. + b5: B5 +} + +impl TestBit { + /// TBNZ + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TBNZ--Test-bit-and-Branch-if-Nonzero-?lang=en> + pub fn tbnz(rt: u8, bit_num: u8, offset: i16) -> Self { + Self { rt, imm14: offset, b40: bit_num & 0b11111, op: Op::TBNZ, b5: bit_num.into() } + } + + /// TBZ + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TBZ--Test-bit-and-Branch-if-Zero-?lang=en> + pub fn tbz(rt: u8, bit_num: u8, offset: i16) -> Self { + Self { rt, imm14: offset, b40: bit_num & 0b11111, op: Op::TBZ, b5: bit_num.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b11011; + +impl From<TestBit> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: TestBit) -> Self { + let b40 = (inst.b40 & 0b11111) as u32; + let imm14 = truncate_imm::<_, 14>(inst.imm14); + + 0 + | ((inst.b5 as u32) << 31) + | (FAMILY << 25) + | ((inst.op as u32) << 24) + | (b40 << 19) + | (imm14 << 5) + | inst.rt as u32 + } +} + +impl From<TestBit> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: TestBit) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tbnz() { + let inst = TestBit::tbnz(0, 0, 0); + let result: u32 = inst.into(); + assert_eq!(0x37000000, result); + } + + #[test] + fn test_tbnz_negative() { + let inst = TestBit::tbnz(0, 0, -1); + let result: u32 = inst.into(); + assert_eq!(0x3707ffe0, result); + } + + #[test] + fn test_tbz() { + let inst = TestBit::tbz(0, 0, 0); + let result: u32 = inst.into(); + assert_eq!(0x36000000, result); + } + + #[test] + fn test_tbz_negative() { + let inst = TestBit::tbz(0, 0, -1); + let result: u32 = inst.into(); + assert_eq!(0x3607ffe0, result); + } +} diff --git a/zjit/src/asm/arm64/inst/udf.rs b/zjit/src/asm/arm64/inst/udf.rs new file mode 100644 index 0000000000..297d17ed62 --- /dev/null +++ b/zjit/src/asm/arm64/inst/udf.rs @@ -0,0 +1,52 @@ +/// The struct that represents an A64 permanently undefined instruction. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 | +/// | imm16..................................................| +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Udf { + /// The immediate value encoded in the instruction + imm16: u16 +} + +impl Udf { + /// UDF - Permanently Undefined + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UDF--Permanently-Undefined-> + pub fn udf(imm16: u16) -> Self { + Self { imm16 } + } +} + +impl From<Udf> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Udf) -> Self { + inst.imm16 as u32 + } +} + +impl From<Udf> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Udf) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_udf() { + let result: u32 = Udf::udf(0).into(); + assert_eq!(0x00000000, result); + } + + #[test] + fn test_udf_imm() { + let result: u32 = Udf::udf(1).into(); + assert_eq!(0x00000001, result); + } +} diff --git a/zjit/src/asm/arm64/mod.rs b/zjit/src/asm/arm64/mod.rs new file mode 100644 index 0000000000..b53f1cf673 --- /dev/null +++ b/zjit/src/asm/arm64/mod.rs @@ -0,0 +1,1987 @@ +#![allow(dead_code)] // For instructions and operands we're not currently using. +#![allow(clippy::upper_case_acronyms)] +#![allow(clippy::identity_op)] +#![allow(clippy::self_named_constructors)] +#![allow(clippy::unusual_byte_groupings)] + +use crate::asm::CodeBlock; + +mod arg; +mod inst; +mod opnd; + +use inst::*; + +// We're going to make these public to make using these things easier in the +// backend (so they don't have to have knowledge about the submodule). +pub use arg::*; +pub use opnd::*; + +/// The extend type for register operands in extended register instructions. +/// It's the result size is determined by the destination register and +/// the source size interpreted using the last letter. +#[derive(Clone, Copy)] +pub enum ExtendType { + UXTB = 0b000, // unsigned extend byte + UXTH = 0b001, // unsigned extend halfword + UXTW = 0b010, // unsigned extend word + UXTX = 0b011, // unsigned extend doubleword + SXTB = 0b100, // signed extend byte + SXTH = 0b101, // signed extend halfword + SXTW = 0b110, // signed extend word + SXTX = 0b111, // signed extend doubleword +} + +/// Checks that a signed value fits within the specified number of bits. +pub const fn imm_fits_bits(imm: i64, num_bits: u8) -> bool { + let minimum = if num_bits == 64 { i64::MIN } else { -(2_i64.pow((num_bits as u32) - 1)) }; + let maximum = if num_bits == 64 { i64::MAX } else { 2_i64.pow((num_bits as u32) - 1) - 1 }; + + imm >= minimum && imm <= maximum +} + +/// Checks that an unsigned value fits within the specified number of bits. +pub const fn uimm_fits_bits(uimm: u64, num_bits: u8) -> bool { + let maximum = if num_bits == 64 { u64::MAX } else { 2_u64.pow(num_bits as u32) - 1 }; + + uimm <= maximum +} + +/// ADD - add rn and rm, put the result in rd, don't update flags +pub fn add(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::add(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::add(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::sub(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::add(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to add instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// Encode ADD (extended register) +/// +/// <https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions/ADD--extended-register---Add--extended-register--> +/// +/// 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00 +/// 0 1 0 1 1 0 0 1 │ │ │ │ │ │ │ │ │ │ +/// sf op S └────rm─────┘ └option┘ └─imm3─┘ └────rn─────┘ └────rd─────┘ +fn encode_add_extend(rd: u8, rn: u8, rm: u8, extend_type: ExtendType, shift: u8, num_bits: u8) -> [u8; 4] { + assert!(shift <= 4, "shift must be 0-4"); + + ((Sf::from(num_bits) as u32) << 31 | + 0b0 << 30 | // op = 0 for add + 0b0 << 29 | // S = 0 for non-flag-setting + 0b01011001 << 21 | + (rm as u32) << 16 | + (extend_type as u32) << 13 | + (shift as u32) << 10 | + (rn as u32) << 5 | + rd as u32).to_le_bytes() +} + +/// ADD (extended register) - add rn and rm with UXTX extension (no extension for 64-bit registers) +/// This is equivalent to a regular ADD for 64-bit registers since UXTX with shift 0 means no modification. +/// For reg_no=31, rd and rn mean SP while with rm means the zero register. +pub fn add_extended(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + encode_add_extend(rd.reg_no, rn.reg_no, rm.reg_no, ExtendType::UXTX, 0, rd.num_bits) + }, + _ => panic!("Invalid operand combination to add_extend instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADDS - add rn and rm, put the result in rd, update flags +pub fn adds(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::adds(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::adds(rd.reg_no, rn.reg_no, imm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::subs(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::adds(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to adds instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADR - form a PC-relative address and load it into a register +pub fn adr(cb: &mut CodeBlock, rd: A64Opnd, imm: A64Opnd) { + let bytes: [u8; 4] = match (rd, imm) { + (A64Opnd::Reg(rd), A64Opnd::Imm(imm)) => { + assert!(rd.num_bits == 64, "The destination register must be 64 bits."); + assert!(imm_fits_bits(imm, 21), "The immediate operand must be 21 bits or less."); + + PCRelative::adr(rd.reg_no, imm as i32).into() + }, + _ => panic!("Invalid operand combination to adr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADRP - form a PC-relative address to a 4KB page and load it into a register. +/// This is effectively the same as ADR except that the immediate must be a +/// multiple of 4KB. +pub fn adrp(cb: &mut CodeBlock, rd: A64Opnd, imm: A64Opnd) { + let bytes: [u8; 4] = match (rd, imm) { + (A64Opnd::Reg(rd), A64Opnd::Imm(imm)) => { + assert!(rd.num_bits == 64, "The destination register must be 64 bits."); + assert!(imm_fits_bits(imm, 32), "The immediate operand must be 32 bits or less."); + + PCRelative::adrp(rd.reg_no, imm as i32).into() + }, + _ => panic!("Invalid operand combination to adr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// AND - and rn and rm, put the result in rd, don't update flags +pub fn and(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::and(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::and(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to and instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ANDS - and rn and rm, put the result in rd, update flags +pub fn ands(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::ands(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::ands(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to ands instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ASR - arithmetic shift right rn by shift, put the result in rd, don't update +/// flags +pub fn asr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(shift)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + assert!(uimm_fits_bits(shift, 6), "The shift operand must be 6 bits or less."); + + SBFM::asr(rd.reg_no, rn.reg_no, shift.try_into().unwrap(), rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to asr instruction: asr {rd:?}, {rn:?}, {shift:?}"), + }; + + cb.write_bytes(&bytes); +} + +/// Whether or not the offset between two instructions fits into the branch with +/// or without link instruction. If it doesn't, then we have to load the value +/// into a register first. +pub const fn b_offset_fits_bits(offset: i64) -> bool { + imm_fits_bits(offset, 26) +} + +/// B - branch without link (offset is number of instructions to jump) +pub fn b(cb: &mut CodeBlock, offset: InstructionOffset) { + assert!(b_offset_fits_bits(offset.into()), "The immediate operand must be 26 bits or less."); + let bytes: [u8; 4] = Call::b(offset).into(); + + cb.write_bytes(&bytes); +} + +/// Whether or not the offset in number of instructions between two instructions +/// fits into the b.cond instruction. If it doesn't, then we have to load the +/// value into a register first, then use the b.cond instruction to skip past a +/// direct jump. +pub const fn bcond_offset_fits_bits(offset: i64) -> bool { + imm_fits_bits(offset, 19) +} + +/// B.cond - branch to target if condition is true +pub fn bcond(cb: &mut CodeBlock, cond: u8, offset: InstructionOffset) { + _ = Condition; + assert!(bcond_offset_fits_bits(offset.into()), "The offset must be 19 bits or less."); + let bytes: [u8; 4] = BranchCond::bcond(cond, offset).into(); + + cb.write_bytes(&bytes); +} + +/// BL - branch with link (offset is number of instructions to jump) +pub fn bl(cb: &mut CodeBlock, offset: InstructionOffset) { + assert!(b_offset_fits_bits(offset.into()), "The offset must be 26 bits or less."); + let bytes: [u8; 4] = Call::bl(offset).into(); + + cb.write_bytes(&bytes); +} + +/// BLR - branch with link to a register +pub fn blr(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::Reg(rn) => Branch::blr(rn.reg_no).into(), + _ => panic!("Invalid operand to blr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// BR - branch to a register +pub fn br(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::Reg(rn) => Branch::br(rn.reg_no).into(), + _ => panic!("Invalid operand to br instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// BRK - create a breakpoint +pub fn brk(cb: &mut CodeBlock, imm16: A64Opnd) { + let bytes: [u8; 4] = match imm16 { + A64Opnd::None => Breakpoint::brk(0xf000).into(), + A64Opnd::UImm(imm16) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + Breakpoint::brk(imm16 as u16).into() + }, + _ => panic!("Invalid operand combination to brk instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// UDF - permanently undefined instruction +pub fn udf(cb: &mut CodeBlock, imm16: u16) { + let bytes: [u8; 4] = Udf::udf(imm16).into(); + cb.write_bytes(&bytes); +} + +/// CMP - compare rn and rm, update flags +pub fn cmp(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rn, rm) { + (A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::cmp(rn.reg_no, rm.reg_no, rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + DataImm::cmp(rn.reg_no, (imm12 as u64).try_into().unwrap(), rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::UImm(imm12)) => { + DataImm::cmp(rn.reg_no, imm12.try_into().unwrap(), rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to cmp instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// CSEL - conditionally select between two registers +pub fn csel(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd, cond: u8) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + Conditional::csel(rd.reg_no, rn.reg_no, rm.reg_no, cond, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to csel instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// EOR - perform a bitwise XOR of rn and rm, put the result in rd, don't update flags +pub fn eor(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::eor(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::eor(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to eor instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDADDAL - atomic add with acquire and release semantics +pub fn ldaddal(cb: &mut CodeBlock, rs: A64Opnd, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rs, rt, rn) { + (A64Opnd::Reg(rs), A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert!( + rs.num_bits == rt.num_bits && rt.num_bits == rn.num_bits, + "All operands must be of the same size." + ); + + Atomic::ldaddal(rs.reg_no, rt.reg_no, rn.reg_no, rs.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldaddal instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDAXR - atomic load with acquire semantics +pub fn ldaxr(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert_eq!(rn.num_bits, 64, "rn must be a 64-bit register."); + + LoadStoreExclusive::ldaxr(rt.reg_no, rn.reg_no, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldaxr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDP (signed offset) - load a pair of registers from memory +pub fn ldp(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDP (pre-index) - load a pair of registers from memory, update the base pointer before loading it +pub fn ldp_pre(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp_pre(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDP (post-index) - load a pair of registers from memory, update the base pointer after loading it +pub fn ldp_post(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp_post(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDR - load a memory address into a register with a register offset +pub fn ldr(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn, rm) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LoadRegister::ldr(rt.reg_no, rn.reg_no, rm.reg_no, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDR - load a PC-relative memory address into a register +pub fn ldr_literal(cb: &mut CodeBlock, rt: A64Opnd, rn: InstructionOffset) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + LoadLiteral::ldr_literal(rt.reg_no, rn, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDRH - load a halfword from memory +pub fn ldrh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 12), "The displacement must be 12 bits or less."); + + HalfwordImm::ldrh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDRH (pre-index) - load a halfword from memory, update the base pointer before loading it +pub fn ldrh_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::ldrh_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDRH (post-index) - load a halfword from memory, update the base pointer after loading it +pub fn ldrh_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::ldrh_post(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// Whether or not a memory address displacement fits into the maximum number of +/// bits such that it can be used without loading it into a register first. +pub fn mem_disp_fits_bits(disp: i32) -> bool { + imm_fits_bits(disp.into(), 9) +} + +/// LDR (post-index) - load a register from memory, update the base pointer after loading it +pub fn ldr_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::ldr_post(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDR (pre-index) - load a register from memory, update the base pointer before loading it +pub fn ldr_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::ldr_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDUR - load a memory address into a register +pub fn ldur(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + + LoadStore::ldur(rt.reg_no, rn.reg_no, 0, rt.num_bits).into() + }, + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldur(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operands for LDUR") + }; + + cb.write_bytes(&bytes); +} + +/// LDURH - load a byte from memory, zero-extend it, and write it to a register +pub fn ldurh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldurh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operands for LDURH") + }; + + cb.write_bytes(&bytes); +} + +/// LDURB - load a byte from memory, zero-extend it, and write it to a register +pub fn ldurb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(rt.num_bits == 8, "Expected registers to have size 8"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldurb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operands for LDURB") + }; + + cb.write_bytes(&bytes); +} + +/// LDURSW - load a 32-bit memory address into a register and sign-extend it +pub fn ldursw(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldursw(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldursw instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LSL - logical shift left a register by an immediate +pub fn lsl(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm)) => { + assert!(rd.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(uimm_fits_bits(uimm, 6), "Expected shift to be 6 bits or less"); + + ShiftImm::lsl(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() + }, + _ => panic!("Invalid operands combination {rd:?} {rn:?} {shift:?} to lsl instruction") + }; + + cb.write_bytes(&bytes); +} + +/// LSR - logical shift right a register by an immediate +pub fn lsr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm)) => { + assert!(rd.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(uimm_fits_bits(uimm, 6), "Expected shift to be 6 bits or less"); + + ShiftImm::lsr(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() + }, + _ => panic!("Invalid operands combination to lsr instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MOV - move a value in a register to another register +pub fn mov(cb: &mut CodeBlock, rd: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rm) { + (A64Opnd::Reg(A64Reg { reg_no: 31, num_bits: 64 }), A64Opnd::Reg(rm)) => { + assert!(rm.num_bits == 64, "Expected rm to be 64 bits"); + + DataImm::add(31, rm.reg_no, 0.try_into().unwrap(), 64).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(A64Reg { reg_no: 31, num_bits: 64 })) => { + assert!(rd.num_bits == 64, "Expected rd to be 64 bits"); + + DataImm::add(rd.reg_no, 31, 0.try_into().unwrap(), 64).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::mov(rd.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::UImm(0)) => { + LogicalReg::mov(rd.reg_no, XZR_REG.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::UImm(imm)) => { + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::mov(rd.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mov instruction: {rd:?}, {rm:?}") + }; + + cb.write_bytes(&bytes); +} + +/// MOVK - move a 16 bit immediate into a register, keep the other bits in place +pub fn movk(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movk(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movk instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MOVN - load a register with the complement of a shifted then zero extended 16-bit immediate +/// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> +pub fn movn(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movn(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movn instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MOVZ - move a 16 bit immediate into a register, zero the other bits +pub fn movz(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movz(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MRS - move a system register into a general-purpose register +pub fn mrs(cb: &mut CodeBlock, rt: A64Opnd, systemregister: SystemRegister) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + SysReg::mrs(rt.reg_no, systemregister).into() + }, + _ => panic!("Invalid operand combination to mrs instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MSR - move a general-purpose register into a system register +pub fn msr(cb: &mut CodeBlock, systemregister: SystemRegister, rt: A64Opnd) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + SysReg::msr(systemregister, rt.reg_no).into() + }, + _ => panic!("Invalid operand combination to msr instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MUL - multiply two registers, put the result in a third register +pub fn mul(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + MAdd::mul(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mul instruction") + }; + + cb.write_bytes(&bytes); +} + +/// SMULH - multiply two 64-bit registers to produce a 128-bit result, put the high 64-bits of the result into rd +pub fn smulh(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + assert!(rd.num_bits == 64, "smulh only applicable to 64-bit registers"); + + SMulH::smulh(rd.reg_no, rn.reg_no, rm.reg_no).into() + }, + _ => panic!("Invalid operand combination to mul instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MVN - move a value in a register to another register, negating it +pub fn mvn(cb: &mut CodeBlock, rd: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::mvn(rd.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mvn instruction") + }; + + cb.write_bytes(&bytes); +} + +/// NOP - no-operation, used for alignment purposes +pub fn nop(cb: &mut CodeBlock) { + let bytes: [u8; 4] = Nop::nop().into(); + + cb.write_bytes(&bytes); +} + +/// ORN - perform a bitwise OR of rn and NOT rm, put the result in rd, don't update flags +pub fn orn(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::orn(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to orn instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// ORR - perform a bitwise OR of rn and rm, put the result in rd, don't update flags +pub fn orr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::orr(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::orr(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to orr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STLXR - store a value to memory, release exclusive access +pub fn stlxr(cb: &mut CodeBlock, rs: A64Opnd, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rs, rt, rn) { + (A64Opnd::Reg(rs), A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert_eq!(rs.num_bits, 32, "rs must be a 32-bit register."); + assert_eq!(rn.num_bits, 64, "rn must be a 64-bit register."); + + LoadStoreExclusive::stlxr(rs.reg_no, rt.reg_no, rn.reg_no, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to stlxr instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (signed offset) - store a pair of registers to memory +pub fn stp(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (pre-index) - store a pair of registers to memory, update the base pointer before loading it +pub fn stp_pre(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp_pre(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (post-index) - store a pair of registers to memory, update the base pointer after loading it +pub fn stp_post(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp_post(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STR (post-index) - store a register to memory, update the base pointer after loading it +pub fn str_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::str_post(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to str instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STR (pre-index) - store a register to memory, update the base pointer before loading it +pub fn str_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::str_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to str instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STRH - store a halfword into memory +pub fn strh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 12), "The displacement must be 12 bits or less."); + + HalfwordImm::strh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STRH (pre-index) - store a halfword into memory, update the base pointer before loading it +pub fn strh_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::strh_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STRH (post-index) - store a halfword into memory, update the base pointer after loading it +pub fn strh_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::strh_post(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STUR - store a value in a register at a memory address +pub fn stur(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 32 || rn.num_bits == 64); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::stur(rt.reg_no, rn.base_reg_no, rn.disp as i16, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to stur instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +/// STURH - store a value in a register at a memory address +pub fn sturh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 16); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::sturh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturh instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +pub fn sturb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 8); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::sturb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturb instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +/// SUB - subtract rm from rn, put the result in rd, don't update flags +pub fn sub(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::sub(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::sub(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::add(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::sub(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to sub instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// SUBS - subtract rm from rn, put the result in rd, update flags +pub fn subs(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::subs(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::subs(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::adds(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::subs(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to subs instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// SXTW - sign extend a 32-bit register into a 64-bit register +pub fn sxtw(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn)) => { + assert_eq!(rd.num_bits, 64, "rd must be 64-bits wide."); + assert_eq!(rn.num_bits, 32, "rn must be 32-bits wide."); + + SBFM::sxtw(rd.reg_no, rn.reg_no).into() + }, + _ => panic!("Invalid operand combination to sxtw instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// RET - unconditionally return to a location in a register, defaults to X30 +pub fn ret(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::None => Branch::ret(30).into(), + A64Opnd::Reg(reg) => Branch::ret(reg.reg_no).into(), + _ => panic!("Invalid operand to ret instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TBNZ - test bit and branch if not zero +pub fn tbnz(cb: &mut CodeBlock, rt: A64Opnd, bit_num: A64Opnd, offset: A64Opnd) { + let bytes: [u8; 4] = match (rt, bit_num, offset) { + (A64Opnd::Reg(rt), A64Opnd::UImm(bit_num), A64Opnd::Imm(offset)) => { + TestBit::tbnz(rt.reg_no, bit_num.try_into().unwrap(), offset.try_into().unwrap()).into() + }, + _ => panic!("Invalid operand combination to tbnz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TBZ - test bit and branch if zero +pub fn tbz(cb: &mut CodeBlock, rt: A64Opnd, bit_num: A64Opnd, offset: A64Opnd) { + let bytes: [u8; 4] = match (rt, bit_num, offset) { + (A64Opnd::Reg(rt), A64Opnd::UImm(bit_num), A64Opnd::Imm(offset)) => { + TestBit::tbz(rt.reg_no, bit_num.try_into().unwrap(), offset.try_into().unwrap()).into() + }, + _ => panic!("Invalid operand combination to tbz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TST - test the bits of a register against a mask, then update flags +pub fn tst(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rn, rm) { + (A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rn.num_bits == rm.num_bits, "All operands must be of the same size."); + + LogicalReg::tst(rn.reg_no, rm.reg_no, rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + let bitmask_imm = if rn.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::tst(rn.reg_no, bitmask_imm, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to tst instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// CBZ - branch if a register is zero +pub fn cbz(cb: &mut CodeBlock, rt: A64Opnd, offset: InstructionOffset) { + assert!(imm_fits_bits(offset.into(), 19), "jump offset for cbz must fit in 19 bits"); + let bytes: [u8; 4] = if let A64Opnd::Reg(rt) = rt { + cbz_cbnz(rt.num_bits, false, offset, rt.reg_no) + } else { + panic!("Invalid operand combination to cbz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// CBNZ - branch if a register is non-zero +pub fn cbnz(cb: &mut CodeBlock, rt: A64Opnd, offset: InstructionOffset) { + assert!(imm_fits_bits(offset.into(), 19), "jump offset for cbz must fit in 19 bits"); + let bytes: [u8; 4] = if let A64Opnd::Reg(rt) = rt { + cbz_cbnz(rt.num_bits, true, offset, rt.reg_no) + } else { + panic!("Invalid operand combination to cbnz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// Encode Compare and Branch on Zero (CBZ) with `op=0` or Compare and Branch on Nonzero (CBNZ) +/// with `op=1`. +/// +/// <https://developer.arm.com/documentation/ddi0602/2024-03/Base-Instructions/CBZ--Compare-and-Branch-on-Zero-> +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | sf 0 1 1 0 1 0 op | +/// | imm19........................................................... Rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +fn cbz_cbnz(num_bits: u8, op: bool, offset: InstructionOffset, rt: u8) -> [u8; 4] { + ((Sf::from(num_bits) as u32) << 31 | + 0b11010 << 25 | + u32::from(op) << 24 | + truncate_imm::<_, 19>(offset) << 5 | + rt as u32).to_le_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + + fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb + } + + #[test] + fn test_imm_fits_bits() { + assert!(imm_fits_bits(i8::MAX.into(), 8)); + assert!(imm_fits_bits(i8::MIN.into(), 8)); + + assert!(imm_fits_bits(i16::MAX.into(), 16)); + assert!(imm_fits_bits(i16::MIN.into(), 16)); + + assert!(imm_fits_bits(i32::MAX.into(), 32)); + assert!(imm_fits_bits(i32::MIN.into(), 32)); + + assert!(imm_fits_bits(i64::MAX, 64)); + assert!(imm_fits_bits(i64::MIN, 64)); + } + + #[test] + fn test_uimm_fits_bits() { + assert!(uimm_fits_bits(u8::MAX.into(), 8)); + assert!(uimm_fits_bits(u16::MAX.into(), 16)); + assert!(uimm_fits_bits(u32::MAX.into(), 32)); + assert!(uimm_fits_bits(u64::MAX, 64)); + } + + #[test] + fn test_add_reg() { + let cb = compile(|cb| add(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028b"); + } + + #[test] + fn test_add_uimm() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_add_imm_positive() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_add_imm_negative() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_adds_reg() { + let cb = compile(|cb| adds(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ab"); + } + + #[test] + fn test_adds_uimm() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_adds_imm_positive() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_adds_imm_negative() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_adr() { + let cb = compile(|cb| adr(cb, X10, A64Opnd::new_imm(20))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adr x10, #0x14"); + assert_snapshot!(cb.hexdump(), @"aa000010"); + } + + #[test] + fn test_adrp() { + let cb = compile(|cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adrp x10, #0x8000"); + assert_snapshot!(cb.hexdump(), @"4a000090"); + } + + #[test] + fn test_and_register() { + let cb = compile(|cb| and(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028a"); + } + + #[test] + fn test_and_immediate() { + let cb = compile(|cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"20084092"); + } + + #[test] + fn test_and_32b_immediate() { + let cb = compile(|cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and w0, w2, #0xfffff"); + assert_snapshot!(cb.hexdump(), @"404c0012"); + } + + #[test] + fn test_ands_register() { + let cb = compile(|cb| ands(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ea"); + } + + #[test] + fn test_ands_immediate() { + let cb = compile(|cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"200840f2"); + } + + #[test] + fn test_asr() { + let cb = compile(|cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: asr x20, x21, #0xa"); + assert_snapshot!(cb.hexdump(), @"b4fe4a93"); + } + + #[test] + fn test_bcond() { + let offset = InstructionOffset::from_insns(0x100); + let cb = compile(|cb| bcond(cb, Condition::NE, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b.ne #0x400"); + assert_snapshot!(cb.hexdump(), @"01200054"); + } + + #[test] + fn test_b() { + let offset = InstructionOffset::from_insns((1 << 25) - 1); + let cb = compile(|cb| b(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b #0x7fffffc"); + assert_snapshot!(cb.hexdump(), @"ffffff15"); + } + + #[test] + #[should_panic] + fn test_b_too_big() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(1 << 25); + compile(|cb| b(cb, offset)); + } + + #[test] + #[should_panic] + fn test_b_too_small() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(-(1 << 25) - 1); + compile(|cb| b(cb, offset)); + } + + #[test] + fn test_bl() { + let offset = InstructionOffset::from_insns(-(1 << 25)); + let cb = compile(|cb| bl(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: bl #0xfffffffff8000000"); + assert_snapshot!(cb.hexdump(), @"00000096"); + } + + #[test] + #[should_panic] + fn test_bl_too_big() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(1 << 25); + compile(|cb| bl(cb, offset)); + } + + #[test] + #[should_panic] + fn test_bl_too_small() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(-(1 << 25) - 1); + compile(|cb| bl(cb, offset)); + } + + #[test] + fn test_blr() { + let cb = compile(|cb| blr(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: blr x20"); + assert_snapshot!(cb.hexdump(), @"80023fd6"); + } + + #[test] + fn test_br() { + let cb = compile(|cb| br(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: br x20"); + assert_snapshot!(cb.hexdump(), @"80021fd6"); + } + + #[test] + fn test_cbz() { + let offset = InstructionOffset::from_insns(-1); + let cb = compile(|cb| { + cbz(cb, X0, offset); + cbz(cb, W0, offset); + }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cbz x0, #0xfffffffffffffffc + 0x4: cbz w0, #0 + "); + assert_snapshot!(cb.hexdump(), @"e0ffffb4e0ffff34"); + } + + #[test] + fn test_cbnz() { + let offset = InstructionOffset::from_insns(2); + let cb = compile(|cb| { + cbnz(cb, X20, offset); + cbnz(cb, W20, offset); + }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cbnz x20, #8 + 0x4: cbnz w20, #0xc + "); + assert_snapshot!(cb.hexdump(), @"540000b554000035"); + } + + #[test] + fn test_brk_none() { + let cb = compile(|cb| brk(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xf000"); + assert_snapshot!(cb.hexdump(), @"00003ed4"); + } + + #[test] + fn test_brk_uimm() { + let cb = compile(|cb| brk(cb, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xe"); + assert_snapshot!(cb.hexdump(), @"c00120d4"); + } + + #[test] + fn test_cmp_register() { + let cb = compile(|cb| cmp(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, x11"); + assert_snapshot!(cb.hexdump(), @"5f010beb"); + } + + #[test] + fn test_cmp_immediate() { + let cb = compile(|cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, #0xe"); + assert_snapshot!(cb.hexdump(), @"5f3900f1"); + } + + #[test] + fn test_csel() { + let cb = compile(|cb| csel(cb, X10, X11, X12, Condition::EQ)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: csel x10, x11, x12, eq"); + assert_snapshot!(cb.hexdump(), @"6a018c9a"); + } + + #[test] + fn test_eor_register() { + let cb = compile(|cb| eor(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010cca"); + } + + #[test] + fn test_eor_immediate() { + let cb = compile(|cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940d2"); + } + + #[test] + fn test_eor_32b_immediate() { + let cb = compile(|cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor w9, w1, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"29040152"); + } + + #[test] + fn test_ldaddal() { + let cb = compile(|cb| ldaddal(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaddal x10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8b01eaf8"); + } + + #[test] + fn test_ldaxr() { + let cb = compile(|cb| ldaxr(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaxr x10, [x11]"); + assert_snapshot!(cb.hexdump(), @"6afd5fc8"); + } + + #[test] + fn test_ldp() { + let cb = compile(|cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d4da9"); + } + + #[test] + fn test_ldp_pre() { + let cb = compile(|cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2dcda9"); + } + + #[test] + fn test_ldp_post() { + let cb = compile(|cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2dcda8"); + } + + #[test] + fn test_ldr() { + let cb = compile(|cb| ldr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, x12]"); + assert_snapshot!(cb.hexdump(), @"6a696cf8"); + } + + #[test] + fn test_ldr_literal() { + let cb = compile(|cb| ldr_literal(cb, X0, 10.into())); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x0, #0x28"); + assert_snapshot!(cb.hexdump(), @"40010058"); + } + + #[test] + fn test_ldr_post() { + let cb = compile(|cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11], #0x10"); + assert_snapshot!(cb.hexdump(), @"6a0541f8"); + } + + #[test] + fn test_ldr_pre() { + let cb = compile(|cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, #0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d41f8"); + } + + #[test] + fn test_ldrh() { + let cb = compile(|cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a194079"); + } + + #[test] + fn test_ldrh_pre() { + let cb = compile(|cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd4078"); + } + + #[test] + fn test_ldrh_post() { + let cb = compile(|cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac54078"); + } + + #[test] + fn test_ldurh_memory() { + let cb = compile(|cb| { + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0)); + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123)); + }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldurh w10, [x1] + 0x4: ldurh w10, [x1, #0x7b] + "); + assert_snapshot!(cb.hexdump(), @"2a0040782ab04778"); + } + + #[test] + fn test_ldur_memory() { + let cb = compile(|cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"20b047f8"); + } + + #[test] + fn test_ldur_register() { + let cb = compile(|cb| ldur(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1]"); + assert_snapshot!(cb.hexdump(), @"200040f8"); + } + + #[test] + fn test_ldursw() { + let cb = compile(|cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldursw x10, [x11, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"6ab187b8"); + } + + #[test] + fn test_lsl() { + let cb = compile(|cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsl x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6ac572d3"); + } + + #[test] + fn test_lsr() { + let cb = compile(|cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsr x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6afd4ed3"); + } + + #[test] + fn test_mov_registers() { + let cb = compile(|cb| mov(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea030baa"); + } + + #[test] + fn test_mov_immediate() { + let cb = compile(|cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, #0x5555555555555555"); + assert_snapshot!(cb.hexdump(), @"eaf300b2"); + } + + #[test] + fn test_mov_32b_immediate() { + let cb = compile(|cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov w10, #-0x7fffffff"); + assert_snapshot!(cb.hexdump(), @"ea070132"); + } + #[test] + fn test_mov_into_sp() { + let cb = compile(|cb| mov(cb, X31, X0)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov sp, x0"); + assert_snapshot!(cb.hexdump(), @"1f000091"); + } + + #[test] + fn test_mov_from_sp() { + let cb = compile(|cb| mov(cb, X0, X31)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, sp"); + assert_snapshot!(cb.hexdump(), @"e0030091"); + } + + #[test] + fn test_movk() { + let cb = compile(|cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movk x0, #0x7b, lsl #16"); + assert_snapshot!(cb.hexdump(), @"600fa0f2"); + } + + #[test] + fn test_movn() { + let cb = compile(|cb| movn(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #-0x7b0001"); + assert_snapshot!(cb.hexdump(), @"600fa092"); + } + + #[test] + fn test_movz() { + let cb = compile(|cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #0x7b0000"); + assert_snapshot!(cb.hexdump(), @"600fa0d2"); + } + + #[test] + fn test_mrs() { + let cb = compile(|cb| mrs(cb, X10, SystemRegister::NZCV)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mrs x10, nzcv"); + assert_snapshot!(cb.hexdump(), @"0a423bd5"); + } + + #[test] + fn test_msr() { + let cb = compile(|cb| msr(cb, SystemRegister::NZCV, X10)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: msr nzcv, x10"); + assert_snapshot!(cb.hexdump(), @"0a421bd5"); + } + + #[test] + fn test_mul() { + let cb = compile(|cb| mul(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mul x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a7d0c9b"); + } + + #[test] + fn test_mvn() { + let cb = compile(|cb| mvn(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mvn x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea032baa"); + } + + #[test] + fn test_nop() { + let cb = compile(nop); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: nop"); + assert_snapshot!(cb.hexdump(), @"1f2003d5"); + } + + #[test] + fn test_orn() { + let cb = compile(|cb| orn(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orn x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a012caa"); + } + + #[test] + fn test_orr_register() { + let cb = compile(|cb| orr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010caa"); + } + + #[test] + fn test_orr_immediate() { + let cb = compile(|cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940b2"); + } + + #[test] + fn test_orr_32b_immediate() { + let cb = compile(|cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr w10, w11, #1"); + assert_snapshot!(cb.hexdump(), @"6a010032"); + } + + #[test] + fn test_ret_none() { + let cb = compile(|cb| ret(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c0035fd6"); + } + + #[test] + fn test_ret_register() { + let cb = compile(|cb| ret(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret x20"); + assert_snapshot!(cb.hexdump(), @"80025fd6"); + } + + #[test] + fn test_stlxr() { + let cb = compile(|cb| stlxr(cb, W10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stlxr w10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8bfd0ac8"); + } + + #[test] + fn test_stp() { + let cb = compile(|cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d0da9"); + } + + #[test] + fn test_stp_pre() { + let cb = compile(|cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2d8da9"); + } + + #[test] + fn test_stp_post() { + let cb = compile(|cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2d8da8"); + } + + #[test] + fn test_str_post() { + let cb = compile(|cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11], #0xfffffffffffffff0"); + assert_snapshot!(cb.hexdump(), @"6a051ff8"); + } + + #[test] + fn test_str_pre() { + let cb = compile(|cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11, #-0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d1ff8"); + } + + #[test] + fn test_strh() { + let cb = compile(|cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a190079"); + } + + #[test] + fn test_strh_pre() { + let cb = compile(|cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd0078"); + } + + #[test] + fn test_strh_post() { + let cb = compile(|cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac50078"); + } + + #[test] + fn test_stur_64_bits() { + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur x10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108f8"); + } + + #[test] + fn test_stur_32_bits() { + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur w10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108b8"); + } + + #[test] + fn test_sub_reg() { + let cb = compile(|cb| sub(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002cb"); + } + + #[test] + fn test_sub_uimm() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_sub_imm_positive() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_sub_imm_negative() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_subs_reg() { + let cb = compile(|cb| subs(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002eb"); + } + + #[test] + fn test_subs_imm_positive() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_subs_imm_negative() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_subs_uimm() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_sxtw() { + let cb = compile(|cb| sxtw(cb, X10, W11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sxtw x10, w11"); + assert_snapshot!(cb.hexdump(), @"6a7d4093"); + } + + #[test] + fn test_tbnz() { + let cb = compile(|cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbnz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005037"); + } + + #[test] + fn test_tbz() { + let cb = compile(|cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005036"); + } + + #[test] + fn test_tst_register() { + let cb = compile(|cb| tst(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); + } + + #[test] + fn test_tst_immediate() { + let cb = compile(|cb| tst(cb, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x1, #7"); + assert_snapshot!(cb.hexdump(), @"3f0840f2"); + } + + #[test] + fn test_tst_32b_immediate() { + let cb = compile(|cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0xffff"); + assert_snapshot!(cb.hexdump(), @"1f3c0072"); + } + + #[test] + fn test_add_extend_various_regs() { + let mut cb = CodeBlock::new_dummy(); + + add_extended(&mut cb, X10, X11, X9); + add_extended(&mut cb, X30, X30, X30); + add_extended(&mut cb, X31, X31, X31); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x10, x11, x9, uxtx + 0x4: add x30, x30, x30, uxtx + 0x8: add sp, sp, xzr + "); + assert_snapshot!(cb.hexdump(), @"6a61298bde633e8bff633f8b"); + } +} diff --git a/zjit/src/asm/arm64/opnd.rs b/zjit/src/asm/arm64/opnd.rs new file mode 100644 index 0000000000..3e6245826b --- /dev/null +++ b/zjit/src/asm/arm64/opnd.rs @@ -0,0 +1,270 @@ +use std::fmt; + +/// This operand represents a register. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Ord, PartialOrd)] +pub struct A64Reg +{ + // Size in bits + pub num_bits: u8, + + // Register index number + pub reg_no: u8, +} + +impl A64Reg { + pub fn with_num_bits(&self, num_bits: u8) -> Self { + assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); + Self { num_bits, reg_no: self.reg_no } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct A64Mem +{ + // Size in bits + pub num_bits: u8, + + /// Base register number + pub base_reg_no: u8, + + /// Constant displacement from the base, not scaled + pub disp: i32, +} + +impl A64Mem { + pub fn new(num_bits: u8, reg: A64Opnd, disp: i32) -> Self { + match reg { + A64Opnd::Reg(reg) => { + Self { num_bits, base_reg_no: reg.reg_no, disp } + }, + _ => panic!("Expected register operand") + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum A64Opnd +{ + // Dummy operand + None, + + // Immediate value + Imm(i64), + + // Unsigned immediate + UImm(u64), + + // Register + Reg(A64Reg), + + // Memory + Mem(A64Mem) +} + +impl A64Opnd { + /// Create a new immediate value operand. + pub fn new_imm(value: i64) -> Self { + A64Opnd::Imm(value) + } + + /// Create a new unsigned immediate value operand. + pub fn new_uimm(value: u64) -> Self { + A64Opnd::UImm(value) + } + + /// Creates a new memory operand. + pub fn new_mem(num_bits: u8, reg: A64Opnd, disp: i32) -> Self { + A64Opnd::Mem(A64Mem::new(num_bits, reg, disp)) + } + + /// Convenience function to check if this operand is a register. + pub fn is_reg(&self) -> bool { + matches!(self, A64Opnd::Reg(_)) + } + + /// Unwrap a register from an operand. + pub fn unwrap_reg(&self) -> A64Reg { + match self { + A64Opnd::Reg(reg) => *reg, + _ => panic!("Expected register operand") + } + } +} + +// argument registers +pub const X0_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 0 }; +pub const X1_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 1 }; +pub const X2_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 2 }; +pub const X3_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 3 }; +pub const X4_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 4 }; +pub const X5_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 5 }; + +// caller-save registers +pub const X9_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 9 }; +pub const X10_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 10 }; +pub const X11_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 11 }; +pub const X12_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 12 }; +pub const X13_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 13 }; +pub const X14_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 14 }; +pub const X15_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 15 }; +pub const X16_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 16 }; +pub const X17_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 17 }; + +// callee-save registers +pub const X19_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 19 }; +pub const X20_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 20 }; +pub const X21_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 21 }; +pub const X22_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 22 }; + +// frame pointer (base pointer) +pub const X29_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 29 }; + +// link register +pub const X30_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 30 }; + +// zero register +pub const XZR_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 31 }; + +// 64-bit registers +pub const X0: A64Opnd = A64Opnd::Reg(X0_REG); +pub const X1: A64Opnd = A64Opnd::Reg(X1_REG); +pub const X2: A64Opnd = A64Opnd::Reg(X2_REG); +pub const X3: A64Opnd = A64Opnd::Reg(X3_REG); +pub const X4: A64Opnd = A64Opnd::Reg(X4_REG); +pub const X5: A64Opnd = A64Opnd::Reg(X5_REG); +pub const X6: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 6 }); +pub const X7: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 7 }); +pub const X8: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 8 }); +pub const X9: A64Opnd = A64Opnd::Reg(X9_REG); +pub const X10: A64Opnd = A64Opnd::Reg(X10_REG); +pub const X11: A64Opnd = A64Opnd::Reg(X11_REG); +pub const X12: A64Opnd = A64Opnd::Reg(X12_REG); +pub const X13: A64Opnd = A64Opnd::Reg(X13_REG); +pub const X14: A64Opnd = A64Opnd::Reg(X14_REG); +pub const X15: A64Opnd = A64Opnd::Reg(X15_REG); +pub const X16: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 16 }); +pub const X17: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 17 }); +pub const X18: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 18 }); +pub const X19: A64Opnd = A64Opnd::Reg(X19_REG); +pub const X20: A64Opnd = A64Opnd::Reg(X20_REG); +pub const X21: A64Opnd = A64Opnd::Reg(X21_REG); +pub const X22: A64Opnd = A64Opnd::Reg(X22_REG); +pub const X23: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 23 }); +pub const X24: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 24 }); +pub const X25: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 25 }); +pub const X26: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 26 }); +pub const X27: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 27 }); +pub const X28: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 28 }); +pub const X29: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 29 }); +pub const X30: A64Opnd = A64Opnd::Reg(X30_REG); +pub const X31: A64Opnd = A64Opnd::Reg(XZR_REG); + +// 32-bit registers +pub const W0: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 0 }); +pub const W1: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 1 }); +pub const W2: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 2 }); +pub const W3: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 3 }); +pub const W4: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 4 }); +pub const W5: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 5 }); +pub const W6: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 6 }); +pub const W7: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 7 }); +pub const W8: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 8 }); +pub const W9: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 9 }); +pub const W10: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 10 }); +pub const W11: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 11 }); +pub const W12: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 12 }); +pub const W13: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 13 }); +pub const W14: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 14 }); +pub const W15: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 15 }); +pub const W16: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 16 }); +pub const W17: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 17 }); +pub const W18: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 18 }); +pub const W19: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 19 }); +pub const W20: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 20 }); +pub const W21: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 21 }); +pub const W22: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 22 }); +pub const W23: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 23 }); +pub const W24: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 24 }); +pub const W25: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 25 }); +pub const W26: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 26 }); +pub const W27: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 27 }); +pub const W28: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 28 }); +pub const W29: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 29 }); +pub const W30: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 30 }); +pub const W31: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 31 }); + +// C argument registers +pub const C_ARG_REGS: [A64Opnd; 6] = [X0, X1, X2, X3, X4, X5]; +pub const C_ARG_REGREGS: [A64Reg; 6] = [X0_REG, X1_REG, X2_REG, X3_REG, X4_REG, X5_REG]; + +impl fmt::Display for A64Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match A64Opnd::Reg(*self) { + X0 => write!(f, "x0"), + X1 => write!(f, "x1"), + X2 => write!(f, "x2"), + X3 => write!(f, "x3"), + X4 => write!(f, "x4"), + X5 => write!(f, "x5"), + X6 => write!(f, "x6"), + X7 => write!(f, "x7"), + X8 => write!(f, "x8"), + X9 => write!(f, "x9"), + X10 => write!(f, "x10"), + X11 => write!(f, "x11"), + X12 => write!(f, "x12"), + X13 => write!(f, "x13"), + X14 => write!(f, "x14"), + X15 => write!(f, "x15"), + X16 => write!(f, "x16"), + X17 => write!(f, "x17"), + X18 => write!(f, "x18"), + X19 => write!(f, "x19"), + X20 => write!(f, "x20"), + X21 => write!(f, "x21"), + X22 => write!(f, "x22"), + X23 => write!(f, "x23"), + X24 => write!(f, "x24"), + X25 => write!(f, "x25"), + X26 => write!(f, "x26"), + X27 => write!(f, "x27"), + X28 => write!(f, "x28"), + X29 => write!(f, "x29"), + X30 => write!(f, "x30"), + X31 => write!(f, "x31"), + W0 => write!(f, "w0"), + W1 => write!(f, "w1"), + W2 => write!(f, "w2"), + W3 => write!(f, "w3"), + W4 => write!(f, "w4"), + W5 => write!(f, "w5"), + W6 => write!(f, "w6"), + W7 => write!(f, "w7"), + W8 => write!(f, "w8"), + W9 => write!(f, "w9"), + W10 => write!(f, "w10"), + W11 => write!(f, "w11"), + W12 => write!(f, "w12"), + W13 => write!(f, "w13"), + W14 => write!(f, "w14"), + W15 => write!(f, "w15"), + W16 => write!(f, "w16"), + W17 => write!(f, "w17"), + W18 => write!(f, "w18"), + W19 => write!(f, "w19"), + W20 => write!(f, "w20"), + W21 => write!(f, "w21"), + W22 => write!(f, "w22"), + W23 => write!(f, "w23"), + W24 => write!(f, "w24"), + W25 => write!(f, "w25"), + W26 => write!(f, "w26"), + W27 => write!(f, "w27"), + W28 => write!(f, "w28"), + W29 => write!(f, "w29"), + W30 => write!(f, "w30"), + W31 => write!(f, "w31"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs new file mode 100644 index 0000000000..6583476594 --- /dev/null +++ b/zjit/src/asm/mod.rs @@ -0,0 +1,463 @@ +//! Model for creating generating textual assembler code. + +use std::collections::BTreeMap; +use std::fmt; +use std::ops::Range; +use std::rc::Rc; +use std::cell::RefCell; +use std::mem; +use crate::virtualmem::*; + +// Lots of manual vertical alignment in there that rustfmt doesn't handle well. +#[rustfmt::skip] +#[cfg(target_arch = "x86_64")] +pub mod x86_64; +#[cfg(target_arch = "aarch64")] +pub mod arm64; + +/// Index to a label created by cb.new_label() +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +pub struct Label(pub usize); + +/// The object that knows how to encode the branch instruction. +type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64) -> Result<(), ()>>; + +/// Reference to an ASM label +pub struct LabelRef { + // Position in the code block where the label reference exists + pos: usize, + + // Label which this refers to + label: Label, + + /// The number of bytes that this label reference takes up in the memory. + /// It's necessary to know this ahead of time so that when we come back to + /// patch it it takes the same amount of space. + num_bytes: usize, + + /// The object that knows how to encode the branch instruction. + encode: BranchEncoder, +} + +/// Block of memory into which instructions can be assembled +pub struct CodeBlock { + // Memory for storing the encoded instructions + mem_block: Rc<RefCell<VirtualMem>>, + + // Memory block size + mem_size: usize, + + // Current writing position + write_pos: usize, + + // Table of registered label addresses + label_addrs: Vec<usize>, + + // Table of registered label names + label_names: Vec<String>, + + // References to labels + label_refs: Vec<LabelRef>, + + // A switch for keeping comments. They take up memory. + keep_comments: bool, + + // Comments for assembly instructions, if that feature is enabled + asm_comments: BTreeMap<usize, Vec<String>>, + + // Set if the CodeBlock is unable to output some instructions, + // for example, when there is not enough space or when a jump + // target is too far away. + dropped_bytes: bool, +} + +impl CodeBlock { + /// Make a new CodeBlock + pub fn new(mem_block: Rc<RefCell<VirtualMem>>, keep_comments: bool) -> Self { + let mem_size = mem_block.borrow().virtual_region_size(); + Self { + mem_block, + mem_size, + write_pos: 0, + label_addrs: Vec::new(), + label_names: Vec::new(), + label_refs: Vec::new(), + keep_comments, + asm_comments: BTreeMap::new(), + dropped_bytes: false, + } + } + + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mem_block.borrow().mapped_region_size() + } + + /// Add an assembly comment if the feature is on. + pub fn add_comment(&mut self, comment: &str) { + if !self.keep_comments { + return; + } + + let cur_ptr = self.get_write_ptr().raw_addr(self); + + // If there's no current list of comments for this line number, add one. + let this_line_comments = self.asm_comments.entry(cur_ptr).or_default(); + + // Unless this comment is the same as the last one at this same line, add it. + if this_line_comments.last().map(String::as_str) != Some(comment) { + this_line_comments.push(comment.to_string()); + } + } + + pub fn comments_at(&self, pos: usize) -> Option<&Vec<String>> { + self.asm_comments.get(&pos) + } + + pub fn get_write_pos(&self) -> usize { + self.write_pos + } + + pub fn write_mem(&self, write_ptr: CodePtr, byte: u8) -> Result<(), WriteError> { + self.mem_block.borrow_mut().write_byte(write_ptr, byte) + } + + /// Get a (possibly dangling) direct pointer to the current write position + pub fn get_write_ptr(&self) -> CodePtr { + self.get_ptr(self.write_pos) + } + + /// Set the current write position from a pointer + pub fn set_write_ptr(&mut self, code_ptr: CodePtr) { + let pos = code_ptr.as_offset() - self.mem_block.borrow().start_ptr().as_offset(); + self.write_pos = pos.try_into().unwrap(); + } + + /// Invoke a callback with write_ptr temporarily adjusted to a given address + pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) -> Range<CodePtr> { + // Temporarily update the write_pos. Ignore the dropped_bytes flag at the old address. + let old_write_pos = self.write_pos; + let old_dropped_bytes = self.dropped_bytes; + self.set_write_ptr(code_ptr); + self.dropped_bytes = false; + + // Invoke the callback + callback(self); + + // Build a code range modified by the callback + let ret = code_ptr..self.get_write_ptr(); + + // Restore the original write_pos and dropped_bytes flag. + self.dropped_bytes = old_dropped_bytes; + self.write_pos = old_write_pos; + ret + } + + /// Get a (possibly dangling) direct pointer into the executable memory block + pub fn get_ptr(&self, offset: usize) -> CodePtr { + self.mem_block.borrow().start_ptr().add_bytes(offset) + } + + /// Write a single byte at the current position. + pub fn write_byte(&mut self, byte: u8) { + let write_ptr = self.get_write_ptr(); + // TODO: check has_capacity() + if self.mem_block.borrow_mut().write_byte(write_ptr, byte).is_ok() { + self.write_pos += 1; + } else { + self.dropped_bytes = true; + } + } + + /// Write multiple bytes starting from the current position. + pub fn write_bytes(&mut self, bytes: &[u8]) { + for byte in bytes { + self.write_byte(*byte); + } + } + + /// Write an integer over the given number of bits at the current position. + pub fn write_int(&mut self, val: u64, num_bits: u32) { + assert!(num_bits > 0); + assert!(num_bits % 8 == 0); + + // Switch on the number of bits + match num_bits { + 8 => self.write_byte(val as u8), + 16 => self.write_bytes(&[(val & 0xff) as u8, ((val >> 8) & 0xff) as u8]), + 32 => self.write_bytes(&[ + (val & 0xff) as u8, + ((val >> 8) & 0xff) as u8, + ((val >> 16) & 0xff) as u8, + ((val >> 24) & 0xff) as u8, + ]), + _ => { + let mut cur = val; + + // Write out the bytes + for _byte in 0..(num_bits / 8) { + self.write_byte((cur & 0xff) as u8); + cur >>= 8; + } + } + } + } + + /// Check if bytes have been dropped (unwritten because of insufficient space) + pub fn has_dropped_bytes(&self) -> bool { + self.dropped_bytes + } + + /// Set dropped_bytes to false if the current zjit_alloc_bytes() + code_region_size + /// + page_size is below --zjit-mem-size. + pub fn update_dropped_bytes(&mut self) { + if self.mem_block.borrow().can_allocate() { + self.dropped_bytes = false; + } + } + + /// Allocate a new label with a given name + pub fn new_label(&mut self, name: String) -> Label { + assert!(!name.contains(' '), "use underscores in label names, not spaces"); + + // This label doesn't have an address yet + self.label_addrs.push(0); + self.label_names.push(name); + + Label(self.label_addrs.len() - 1) + } + + /// Write a label at the current address + pub fn write_label(&mut self, label: Label) { + self.label_addrs[label.0] = self.write_pos; + } + + // Add a label reference at the current write position + pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) -> Result<(), ()> + 'static) { + assert!(label.0 < self.label_addrs.len()); + + // Keep track of the reference + self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode: Box::new(encode) }); + + // Move past however many bytes the instruction takes up + if self.write_pos + num_bytes < self.mem_size { + self.write_pos += num_bytes; + } else { + self.dropped_bytes = true; // retry emitting the Insn after next_page + } + } + + // Link internal label references + pub fn link_labels(&mut self) -> Result<(), ()> { + let orig_pos = self.write_pos; + let mut link_result = Ok(()); + + // For each label reference + for label_ref in mem::take(&mut self.label_refs) { + let ref_pos = label_ref.pos; + let label_idx = label_ref.label.0; + assert!(ref_pos < self.mem_size); + + let label_addr = self.label_addrs[label_idx]; + assert!(label_addr < self.mem_size); + + self.write_pos = ref_pos; + let encode_result = (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + link_result = link_result.and(encode_result); + + // Verify number of bytes written when the callback returns Ok + if encode_result.is_ok() { + assert_eq!(self.write_pos, ref_pos + label_ref.num_bytes, "label_ref \ + callback didn't write number of bytes it claimed to write upfront"); + } + } + + self.write_pos = orig_pos; + + // Clear the label positions and references + self.label_addrs.clear(); + self.label_names.clear(); + assert!(self.label_refs.is_empty()); + + link_result + } + + /// Convert a Label to CodePtr + pub fn resolve_label(&self, label: Label) -> CodePtr { + self.get_ptr(self.label_addrs[label.0]) + } + + pub fn clear_labels(&mut self) { + self.label_addrs.clear(); + self.label_names.clear(); + self.label_refs.clear(); + } + + /// Make all the code in the region executable. Call this at the end of a write session. + pub fn mark_all_writable(&mut self) { + self.mem_block.borrow_mut().mark_all_writable(); + } + + pub fn mark_all_executable(&mut self) { + self.mem_block.borrow_mut().mark_all_executable(); + } + + /// Call a func with the disasm of generated code for testing + #[allow(unused_variables)] + #[cfg(all(test, feature = "disasm"))] + pub fn disasm(&self) -> String { + let start_addr = self.get_ptr(0).raw_addr(self); + let end_addr = self.get_write_ptr().raw_addr(self); + crate::disasm::disasm_addr_range(self, start_addr, end_addr) + } + + /// Return the hex dump of generated code for testing + #[cfg(test)] + pub fn hexdump(&self) -> String { + format!("{:x}", self) + } +} + +/// Run assert_snapshot! only if cfg!(feature = "disasm"). +/// $actual can be not only `cb.disasm()` but also `disasms!(cb1, cb2, ...)`. +#[cfg(test)] +#[macro_export] +macro_rules! assert_disasm_snapshot { + ($actual: expr, @$($tt: tt)*) => {{ + #[cfg(feature = "disasm")] + assert_snapshot!($actual, @$($tt)*) + }}; +} + +/// Combine multiple cb.disasm() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms { + ($( $cb:expr ),+ $(,)?) => {{ + crate::disasms_with!("", $( $cb ),+) + }}; +} + +/// Basically `disasms!` but allows a non-"" delimiter, such as "\n" +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms_with { + ($join:expr, $( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.disasm() ),+].join($join) + }}; +} + +/// Combine multiple cb.hexdump() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(test)] +#[macro_export] +macro_rules! hexdumps { + ($( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.hexdump() ),+].join("\n") + }}; +} + +/// Produce hex string output from the bytes in a code block +impl fmt::LowerHex for CodeBlock { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + for pos in 0..self.write_pos { + let mem_block = &*self.mem_block.borrow(); + let byte = unsafe { mem_block.start_ptr().raw_ptr(mem_block).add(pos).read() }; + fmtr.write_fmt(format_args!("{byte:02x}"))?; + } + Ok(()) + } +} + +#[cfg(test)] +impl CodeBlock { + /// Stubbed CodeBlock for testing. Can't execute generated code. + pub fn new_dummy() -> Self { + const DEFAULT_MEM_SIZE: usize = 1024 * 1024; + CodeBlock::new_dummy_sized(DEFAULT_MEM_SIZE) + } + + pub fn new_dummy_sized(mem_size: usize) -> Self { + use crate::virtualmem::*; + let virt_mem = VirtualMem::alloc(mem_size, None); + Self::new(Rc::new(RefCell::new(virt_mem)), false) + } +} + +impl crate::virtualmem::CodePtrBase for CodeBlock { + fn base_ptr(&self) -> std::ptr::NonNull<u8> { + self.mem_block.borrow().base_ptr() + } +} + +/// Compute the number of bits needed to encode a signed value +pub fn imm_num_bits(imm: i64) -> u8 +{ + // Compute the smallest size this immediate fits in + if imm >= i8::MIN.into() && imm <= i8::MAX.into() { + return 8; + } + if imm >= i16::MIN.into() && imm <= i16::MAX.into() { + return 16; + } + if imm >= i32::MIN.into() && imm <= i32::MAX.into() { + return 32; + } + + 64 +} + +/// Compute the number of bits needed to encode an unsigned value +pub fn uimm_num_bits(uimm: u64) -> u8 +{ + // Compute the smallest size this immediate fits in + if uimm <= u8::MAX.into() { + return 8; + } + else if uimm <= u16::MAX.into() { + return 16; + } + else if uimm <= u32::MAX.into() { + return 32; + } + + 64 +} + +#[cfg(test)] +mod tests +{ + use super::*; + + #[test] + fn test_imm_num_bits() + { + assert_eq!(imm_num_bits(i8::MIN.into()), 8); + assert_eq!(imm_num_bits(i8::MAX.into()), 8); + + assert_eq!(imm_num_bits(i16::MIN.into()), 16); + assert_eq!(imm_num_bits(i16::MAX.into()), 16); + + assert_eq!(imm_num_bits(i32::MIN.into()), 32); + assert_eq!(imm_num_bits(i32::MAX.into()), 32); + + assert_eq!(imm_num_bits(i64::MIN), 64); + assert_eq!(imm_num_bits(i64::MAX), 64); + } + + #[test] + fn test_uimm_num_bits() { + assert_eq!(uimm_num_bits(u8::MIN.into()), 8); + assert_eq!(uimm_num_bits(u8::MAX.into()), 8); + + assert_eq!(uimm_num_bits(((u8::MAX as u16) + 1).into()), 16); + assert_eq!(uimm_num_bits(u16::MAX.into()), 16); + + assert_eq!(uimm_num_bits(((u16::MAX as u32) + 1).into()), 32); + assert_eq!(uimm_num_bits(u32::MAX.into()), 32); + + assert_eq!(uimm_num_bits((u32::MAX as u64) + 1), 64); + assert_eq!(uimm_num_bits(u64::MAX), 64); + } +} diff --git a/zjit/src/asm/x86_64/mod.rs b/zjit/src/asm/x86_64/mod.rs new file mode 100644 index 0000000000..c2733e783f --- /dev/null +++ b/zjit/src/asm/x86_64/mod.rs @@ -0,0 +1,1439 @@ +#![allow(dead_code)] // For instructions we don't currently generate + +use crate::asm::*; + +// Import the assembler tests module +mod tests; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86Imm +{ + // Size in bits + pub num_bits: u8, + + // The value of the immediate + pub value: i64 +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86UImm +{ + // Size in bits + pub num_bits: u8, + + // The value of the immediate + pub value: u64 +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum RegType +{ + GP, + //FP, + //XMM, + IP, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct X86Reg +{ + // Size in bits + pub num_bits: u8, + + // Register type + pub reg_type: RegType, + + // Register index number + pub reg_no: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct X86Mem +{ + // Size in bits + pub num_bits: u8, + + /// Base register number + pub base_reg_no: u8, + + /// Index register number + pub idx_reg_no: Option<u8>, + + /// SIB scale exponent value (power of two, two bits) + pub scale_exp: u8, + + /// Constant displacement from the base, not scaled + pub disp: i32, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum X86Opnd +{ + // Dummy operand + None, + + // Immediate value + Imm(X86Imm), + + // Unsigned immediate + UImm(X86UImm), + + // General-purpose register + Reg(X86Reg), + + // Memory location + Mem(X86Mem), + + // IP-relative memory location + IPRel(i32) +} + +impl X86Reg { + pub fn with_num_bits(&self, num_bits: u8) -> Self { + assert!( + num_bits == 8 || + num_bits == 16 || + num_bits == 32 || + num_bits == 64 + ); + Self { + num_bits, + reg_type: self.reg_type, + reg_no: self.reg_no + } + } + + pub fn rex_needed(&self) -> bool { + self.reg_no > 7 || self.num_bits == 8 && self.reg_no >= 4 + } +} + +impl X86Opnd { + fn rex_needed(&self) -> bool { + match self { + X86Opnd::None => false, + X86Opnd::Imm(_) => false, + X86Opnd::UImm(_) => false, + X86Opnd::Reg(reg) => reg.rex_needed(), + X86Opnd::Mem(mem) => mem.base_reg_no > 7 || (mem.idx_reg_no.unwrap_or(0) > 7), + X86Opnd::IPRel(_) => false + } + } + + // Check if an SIB byte is needed to encode this operand + fn sib_needed(&self) -> bool { + match self { + X86Opnd::Mem(mem) => { + mem.idx_reg_no.is_some() || + mem.base_reg_no == RSP_REG_NO || + mem.base_reg_no == R12_REG_NO + }, + _ => false + } + } + + fn disp_size(&self) -> u32 { + match self { + X86Opnd::IPRel(_) => 32, + X86Opnd::Mem(mem) => { + if mem.disp != 0 { + // Compute the required displacement size + let num_bits = imm_num_bits(mem.disp.into()); + if num_bits > 32 { + panic!("displacement does not fit in 32 bits"); + } + + // x86 can only encode 8-bit and 32-bit displacements + if num_bits == 16 { 32 } else { 8 } + } else if mem.base_reg_no == RBP_REG_NO || mem.base_reg_no == R13_REG_NO { + // If EBP or RBP or R13 is used as the base, displacement must be encoded + 8 + } else { + 0 + } + }, + _ => 0 + } + } + + pub fn num_bits(&self) -> u8 { + match self { + X86Opnd::Reg(reg) => reg.num_bits, + X86Opnd::Imm(imm) => imm.num_bits, + X86Opnd::UImm(uimm) => uimm.num_bits, + X86Opnd::Mem(mem) => mem.num_bits, + _ => unreachable!() + } + } + + pub fn is_some(&self) -> bool { + !matches!(self, X86Opnd::None) + } + +} + +// Instruction pointer +pub const RIP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 64, reg_type: RegType::IP, reg_no: 5 }); + +// 64-bit GP registers +const RAX_REG_NO: u8 = 0; +const RSP_REG_NO: u8 = 4; +const RBP_REG_NO: u8 = 5; +const R12_REG_NO: u8 = 12; +const R13_REG_NO: u8 = 13; + +pub const RAX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RAX_REG_NO }; +pub const RCX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 1 }; +pub const RDX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 2 }; +pub const RBX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 3 }; +pub const RSP_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RSP_REG_NO }; +pub const RBP_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RBP_REG_NO }; +pub const RSI_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 6 }; +pub const RDI_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 7 }; +pub const R8_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 8 }; +pub const R9_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 9 }; +pub const R10_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 10 }; +pub const R11_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 11 }; +pub const R12_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: R12_REG_NO }; +pub const R13_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: R13_REG_NO }; +pub const R14_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 14 }; +pub const R15_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 15 }; + +pub const RAX: X86Opnd = X86Opnd::Reg(RAX_REG); +pub const RCX: X86Opnd = X86Opnd::Reg(RCX_REG); +pub const RDX: X86Opnd = X86Opnd::Reg(RDX_REG); +pub const RBX: X86Opnd = X86Opnd::Reg(RBX_REG); +pub const RSP: X86Opnd = X86Opnd::Reg(RSP_REG); +pub const RBP: X86Opnd = X86Opnd::Reg(RBP_REG); +pub const RSI: X86Opnd = X86Opnd::Reg(RSI_REG); +pub const RDI: X86Opnd = X86Opnd::Reg(RDI_REG); +pub const R8: X86Opnd = X86Opnd::Reg(R8_REG); +pub const R9: X86Opnd = X86Opnd::Reg(R9_REG); +pub const R10: X86Opnd = X86Opnd::Reg(R10_REG); +pub const R11: X86Opnd = X86Opnd::Reg(R11_REG); +pub const R12: X86Opnd = X86Opnd::Reg(R12_REG); +pub const R13: X86Opnd = X86Opnd::Reg(R13_REG); +pub const R14: X86Opnd = X86Opnd::Reg(R14_REG); +pub const R15: X86Opnd = X86Opnd::Reg(R15_REG); + +// 32-bit GP registers +pub const EAX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 0 }); +pub const ECX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 1 }); +pub const EDX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 2 }); +pub const EBX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 3 }); +pub const ESP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 4 }); +pub const EBP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 5 }); +pub const ESI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 6 }); +pub const EDI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 7 }); +pub const R8D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 8 }); +pub const R9D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 9 }); +pub const R10D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 10 }); +pub const R11D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 11 }); +pub const R12D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 12 }); +pub const R13D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 13 }); +pub const R14D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 14 }); +pub const R15D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 15 }); + +// 16-bit GP registers +pub const AX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 0 }); +pub const CX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 1 }); +pub const DX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 2 }); +pub const BX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 3 }); +//pub const SP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 4 }); +pub const BP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 5 }); +pub const SI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 6 }); +pub const DI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 7 }); +pub const R8W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 8 }); +pub const R9W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 9 }); +pub const R10W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 10 }); +pub const R11W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 11 }); +pub const R12W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 12 }); +pub const R13W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 13 }); +pub const R14W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 14 }); +pub const R15W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 15 }); + +// 8-bit GP registers +pub const AL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 0 }); +pub const CL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 1 }); +pub const DL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 2 }); +pub const BL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 3 }); +pub const SPL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 4 }); +pub const BPL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 5 }); +pub const SIL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 6 }); +pub const DIL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 7 }); +pub const R8B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 8 }); +pub const R9B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 9 }); +pub const R10B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 10 }); +pub const R11B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 11 }); +pub const R12B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 12 }); +pub const R13B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 13 }); +pub const R14B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 14 }); +pub const R15B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 15 }); + +//=========================================================================== + +/// Shorthand for memory operand with base register and displacement +pub fn mem_opnd(num_bits: u8, base_reg: X86Opnd, disp: i32) -> X86Opnd +{ + let base_reg = match base_reg { + X86Opnd::Reg(reg) => reg, + _ => unreachable!() + }; + + if base_reg.reg_type == RegType::IP { + X86Opnd::IPRel(disp) + } else { + X86Opnd::Mem( + X86Mem { + num_bits, + base_reg_no: base_reg.reg_no, + idx_reg_no: None, + scale_exp: 0, + disp, + } + ) + } +} + +/// Memory operand with SIB (Scale Index Base) indexing +pub fn mem_opnd_sib(num_bits: u8, base_opnd: X86Opnd, index_opnd: X86Opnd, scale: i32, disp: i32) -> X86Opnd { + if let (X86Opnd::Reg(base_reg), X86Opnd::Reg(index_reg)) = (base_opnd, index_opnd) { + let scale_exp: u8 = match scale { + 8 => 3, + 4 => 2, + 2 => 1, + 1 => 0, + _ => unreachable!() + }; + + X86Opnd::Mem(X86Mem { + num_bits, + base_reg_no: base_reg.reg_no, + idx_reg_no: Some(index_reg.reg_no), + scale_exp, + disp + }) + } else { + unreachable!() + } +} + +pub fn imm_opnd(value: i64) -> X86Opnd +{ + X86Opnd::Imm(X86Imm { num_bits: imm_num_bits(value), value }) +} + +pub fn uimm_opnd(value: u64) -> X86Opnd +{ + X86Opnd::UImm(X86UImm { num_bits: uimm_num_bits(value), value }) +} + +pub fn const_ptr_opnd(ptr: *const u8) -> X86Opnd +{ + uimm_opnd(ptr as u64) +} + +/// Write the REX byte +fn write_rex(cb: &mut CodeBlock, w_flag: bool, reg_no: u8, idx_reg_no: u8, rm_reg_no: u8) { + // 0 1 0 0 w r x b + // w - 64-bit operand size flag + // r - MODRM.reg extension + // x - SIB.index extension + // b - MODRM.rm or SIB.base extension + let w: u8 = if w_flag { 1 } else { 0 }; + let r: u8 = if (reg_no & 8) > 0 { 1 } else { 0 }; + let x: u8 = if (idx_reg_no & 8) > 0 { 1 } else { 0 }; + let b: u8 = if (rm_reg_no & 8) > 0 { 1 } else { 0 }; + + // Encode and write the REX byte + cb.write_byte(0x40 + (w << 3) + (r << 2) + (x << 1) + (b)); +} + +/// Write an opcode byte with an embedded register operand +fn write_opcode(cb: &mut CodeBlock, opcode: u8, reg: X86Reg) { + let op_byte: u8 = opcode | (reg.reg_no & 7); + cb.write_byte(op_byte); +} + +/// Encode an RM instruction +fn write_rm(cb: &mut CodeBlock, sz_pref: bool, rex_w: bool, r_opnd: X86Opnd, rm_opnd: X86Opnd, op_ext: Option<u8>, bytes: &[u8]) { + let op_len = bytes.len(); + assert!(op_len > 0 && op_len <= 3); + assert!(matches!(r_opnd, X86Opnd::Reg(_) | X86Opnd::None), "Can only encode an RM instruction with a register or a none"); + + // Flag to indicate the REX prefix is needed + let need_rex = rex_w || r_opnd.rex_needed() || rm_opnd.rex_needed(); + + // Flag to indicate SIB byte is needed + let need_sib = r_opnd.sib_needed() || rm_opnd.sib_needed(); + + // Add the operand-size prefix, if needed + if sz_pref { + cb.write_byte(0x66); + } + + // Add the REX prefix, if needed + if need_rex { + // 0 1 0 0 w r x b + // w - 64-bit operand size flag + // r - MODRM.reg extension + // x - SIB.index extension + // b - MODRM.rm or SIB.base extension + + let w = if rex_w { 1 } else { 0 }; + let r = match r_opnd { + X86Opnd::None => 0, + X86Opnd::Reg(reg) => if (reg.reg_no & 8) > 0 { 1 } else { 0 }, + _ => unreachable!() + }; + + let x = match (need_sib, rm_opnd) { + (true, X86Opnd::Mem(mem)) => if (mem.idx_reg_no.unwrap_or(0) & 8) > 0 { 1 } else { 0 }, + _ => 0 + }; + + let b = match rm_opnd { + X86Opnd::Reg(reg) => if (reg.reg_no & 8) > 0 { 1 } else { 0 }, + X86Opnd::Mem(mem) => if (mem.base_reg_no & 8) > 0 { 1 } else { 0 }, + _ => 0 + }; + + // Encode and write the REX byte + let rex_byte: u8 = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b); + cb.write_byte(rex_byte); + } + + // Write the opcode bytes to the code block + for byte in bytes { + cb.write_byte(*byte) + } + + // MODRM.mod (2 bits) + // MODRM.reg (3 bits) + // MODRM.rm (3 bits) + + assert!( + !(op_ext.is_some() && r_opnd.is_some()), + "opcode extension and register operand present" + ); + + // Encode the mod field + let rm_mod = match rm_opnd { + X86Opnd::Reg(_) => 3, + X86Opnd::IPRel(_) => 0, + X86Opnd::Mem(_mem) => { + match rm_opnd.disp_size() { + 0 => 0, + 8 => 1, + 32 => 2, + _ => unreachable!() + } + }, + _ => unreachable!() + }; + + // Encode the reg field + let reg: u8; + if let Some(val) = op_ext { + reg = val; + } else { + reg = match r_opnd { + X86Opnd::Reg(reg) => reg.reg_no & 7, + _ => 0 + }; + } + + // Encode the rm field + let rm = match rm_opnd { + X86Opnd::Reg(reg) => reg.reg_no & 7, + X86Opnd::Mem(mem) => if need_sib { 4 } else { mem.base_reg_no & 7 }, + X86Opnd::IPRel(_) => 0b101, + _ => unreachable!() + }; + + // Encode and write the ModR/M byte + let rm_byte: u8 = (rm_mod << 6) + (reg << 3) + (rm); + cb.write_byte(rm_byte); + + // Add the SIB byte, if needed + if need_sib { + // SIB.scale (2 bits) + // SIB.index (3 bits) + // SIB.base (3 bits) + + match rm_opnd { + X86Opnd::Mem(mem) => { + // Encode the scale value + let scale = mem.scale_exp; + + // Encode the index value + let index = mem.idx_reg_no.map(|no| no & 7).unwrap_or(4); + + // Encode the base register + let base = mem.base_reg_no & 7; + + // Encode and write the SIB byte + let sib_byte: u8 = (scale << 6) + (index << 3) + (base); + cb.write_byte(sib_byte); + }, + _ => panic!("Expected mem operand") + } + } + + // Add the displacement + match rm_opnd { + X86Opnd::Mem(mem) => { + let disp_size = rm_opnd.disp_size(); + if disp_size > 0 { + cb.write_int(mem.disp as u64, disp_size); + } + }, + X86Opnd::IPRel(rel) => { + cb.write_int(rel as u64, 32); + }, + _ => () + }; +} + +// Encode a mul-like single-operand RM instruction +fn write_rm_unary(cb: &mut CodeBlock, op_mem_reg_8: u8, op_mem_reg_pref: u8, op_ext: Option<u8>, opnd: X86Opnd) { + assert!(matches!(opnd, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + let opnd_size = opnd.num_bits(); + assert!(opnd_size == 8 || opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd, op_ext, &[op_mem_reg_8]); + } else { + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd, op_ext, &[op_mem_reg_pref]); + } +} + +// Encode an add-like RM instruction with multiple possible encodings +fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_reg_mem8: u8, op_reg_mem_pref: u8, op_mem_imm8: u8, op_mem_imm_sml: u8, op_mem_imm_lrg: u8, op_ext_imm: Option<u8>, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_)), "unexpected opnd0: {opnd0:?}, {opnd1:?}"); + + // Check the size of opnd0 + let opnd_size = opnd0.num_bits(); + assert!(opnd_size == 8 || opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + // Check the size of opnd1 + match opnd1 { + X86Opnd::Reg(reg) => assert_eq!(reg.num_bits, opnd_size), + X86Opnd::Mem(mem) => assert_eq!(mem.num_bits, opnd_size), + X86Opnd::Imm(imm) => assert!(imm.num_bits <= opnd_size), + X86Opnd::UImm(uimm) => assert!(uimm.num_bits <= opnd_size), + _ => () + }; + + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + + match (opnd0, opnd1) { + // R/M + Reg + (X86Opnd::Mem(_), X86Opnd::Reg(_)) | (X86Opnd::Reg(_), X86Opnd::Reg(_)) => { + if opnd_size == 8 { + write_rm(cb, false, false, opnd1, opnd0, None, &[op_mem_reg8]); + } else { + write_rm(cb, sz_pref, rex_w, opnd1, opnd0, None, &[op_mem_reg_pref]); + } + }, + // Reg + R/M/IPRel + (X86Opnd::Reg(_), X86Opnd::Mem(_) | X86Opnd::IPRel(_)) => { + if opnd_size == 8 { + write_rm(cb, false, false, opnd0, opnd1, None, &[op_reg_mem8]); + } else { + write_rm(cb, sz_pref, rex_w, opnd0, opnd1, None, &[op_reg_mem_pref]); + } + }, + // R/M + Imm + (_, X86Opnd::Imm(imm)) => { + if imm.num_bits <= 8 { + // 8-bit immediate + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm8]); + } else { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_sml]); + } + + cb.write_int(imm.value as u64, 8); + } else if imm.num_bits <= 32 { + // 32-bit immediate + + assert!(imm.num_bits <= opnd_size); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_lrg]); + cb.write_int(imm.value as u64, if opnd_size > 32 { 32 } else { opnd_size.into() }); + } else { + panic!("immediate value too large"); + } + }, + // R/M + UImm + (_, X86Opnd::UImm(uimm)) => { + // If the size of left hand operand equals the number of bits + // required to represent the right hand immediate, then we + // don't care about sign extension when calculating the immediate + let num_bits = if opnd0.num_bits() == uimm_num_bits(uimm.value) { + uimm_num_bits(uimm.value) + } else { + imm_num_bits(uimm.value.try_into().unwrap()) + }; + + if num_bits <= 8 { + // 8-bit immediate + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm8]); + } else { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_sml]); + } + + cb.write_int(uimm.value, 8); + } else if num_bits <= 32 { + // 32-bit immediate + + assert!(num_bits <= opnd_size); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_lrg]); + cb.write_int(uimm.value, if opnd_size > 32 { 32 } else { opnd_size.into() }); + } else { + panic!("immediate value too large (num_bits={}, num={uimm:?})", num_bits); + } + }, + _ => panic!("unknown encoding combo: {opnd0:?} {opnd1:?}") + }; +} + +// LOCK - lock prefix for atomic shared memory operations +pub fn write_lock_prefix(cb: &mut CodeBlock) { + cb.write_byte(0xf0); +} + +/// add - Integer addition +pub fn add(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x00, // opMemReg8 + 0x01, // opMemRegPref + 0x02, // opRegMem8 + 0x03, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x00), // opExtImm + opnd0, + opnd1 + ); +} + +/// and - Bitwise AND +pub fn and(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x20, // opMemReg8 + 0x21, // opMemRegPref + 0x22, // opRegMem8 + 0x23, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x04), // opExtImm + opnd0, + opnd1 + ); +} + +/// call - Call to a pointer with a 32-bit displacement offset +pub fn call_rel32(cb: &mut CodeBlock, rel32: i32) { + // Write the opcode + cb.write_byte(0xe8); + + // Write the relative 32-bit jump offset + cb.write_bytes(&rel32.to_le_bytes()); +} + +/// call - Call a pointer, encode with a 32-bit offset if possible +pub fn call_ptr(cb: &mut CodeBlock, scratch_opnd: X86Opnd, dst_ptr: *const u8) { + if let X86Opnd::Reg(_scratch_reg) = scratch_opnd { + // TODO: implement a counter + + // Pointer to the end of this call instruction + let end_ptr = cb.get_ptr(cb.write_pos + 5); + + // Compute the jump offset + let rel64: i64 = dst_ptr as i64 - end_ptr.raw_ptr(cb) as i64; + + // If the offset fits in 32-bit + if rel64 >= i32::MIN.into() && rel64 <= i32::MAX.into() { + call_rel32(cb, rel64.try_into().unwrap()); + return; + } + + // Move the pointer into the scratch register and call + mov(cb, scratch_opnd, const_ptr_opnd(dst_ptr)); + call(cb, scratch_opnd); + } else { + unreachable!(); + } +} + +/// call - Call to label with 32-bit offset +pub fn call_label(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 5, |cb, src_addr, dst_addr| { + cb.write_byte(0xE8); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// call - Indirect call with an R/M operand +pub fn call(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(2), &[0xff]); +} + +/// Encode a conditional move instruction +fn write_cmov(cb: &mut CodeBlock, opcode1: u8, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(reg) = dst { + match src { + X86Opnd::Reg(_) => (), + X86Opnd::Mem(_) => (), + _ => unreachable!() + }; + + assert!(reg.num_bits >= 16); + let sz_pref = reg.num_bits == 16; + let rex_w = reg.num_bits == 64; + + write_rm(cb, sz_pref, rex_w, dst, src, None, &[0x0f, opcode1]); + } else { + unreachable!() + } +} + +// cmovcc - Conditional move +pub fn cmova(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x47, dst, src); } +pub fn cmovae(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovb(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmovbe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x46, dst, src); } +pub fn cmovc(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmove(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x44, dst, src); } +pub fn cmovg(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4f, dst, src); } +pub fn cmovge(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4d, dst, src); } +pub fn cmovl(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4c, dst, src); } +pub fn cmovle(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4e, dst, src); } +pub fn cmovna(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x46, dst, src); } +pub fn cmovnae(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmovnb(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovnbe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x47, dst, src); } +pub fn cmovnc(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovne(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x45, dst, src); } +pub fn cmovng(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4e, dst, src); } +pub fn cmovnge(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4c, dst, src); } +pub fn cmovnl(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4d, dst, src); } +pub fn cmovnle(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4f, dst, src); } +pub fn cmovno(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x41, dst, src); } +pub fn cmovnp(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4b, dst, src); } +pub fn cmovns(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x49, dst, src); } +pub fn cmovnz(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x45, dst, src); } +pub fn cmovo(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x40, dst, src); } +pub fn cmovp(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4a, dst, src); } +pub fn cmovpe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4a, dst, src); } +pub fn cmovpo(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4b, dst, src); } +pub fn cmovs(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x48, dst, src); } +pub fn cmovz(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x44, dst, src); } + +/// cmp - Compare and set flags +pub fn cmp(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x38, // opMemReg8 + 0x39, // opMemRegPref + 0x3A, // opRegMem8 + 0x3B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x07), // opExtImm + opnd0, + opnd1 + ); +} + +/// cdq - Convert doubleword to quadword +pub fn cdq(cb: &mut CodeBlock) { + cb.write_byte(0x99); +} + +/// cqo - Convert quadword to octaword +pub fn cqo(cb: &mut CodeBlock) { + cb.write_bytes(&[0x48, 0x99]); +} + +/// imul - signed integer multiply +pub fn imul(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(opnd0.num_bits() == 64); + assert!(opnd1.num_bits() == 64); + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + assert!(matches!(opnd1, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + match (opnd0, opnd1) { + (X86Opnd::Reg(_), X86Opnd::Reg(_) | X86Opnd::Mem(_)) => { + //REX.W + 0F AF /rIMUL r64, r/m64 + // Quadword register := Quadword register * r/m64. + write_rm(cb, false, true, opnd0, opnd1, None, &[0x0F, 0xAF]); + } + + _ => unreachable!() + } +} + +/// Interrupt 3 - trap to debugger +pub fn int3(cb: &mut CodeBlock) { + cb.write_byte(0xcc); +} + +// Encode a conditional relative jump to a label +// Note: this always encodes a 32-bit offset +fn write_jcc<const OP: u8>(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 6, |cb, src_addr, dst_addr| { + cb.write_byte(0x0F); + cb.write_byte(OP); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// jcc - relative jumps to a label +pub fn ja_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x87>(cb, label); } +pub fn jae_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jb_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn jbe_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x86>(cb, label); } +pub fn jc_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn je_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x84>(cb, label); } +pub fn jg_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8F>(cb, label); } +pub fn jge_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8D>(cb, label); } +pub fn jl_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8C>(cb, label); } +pub fn jle_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8E>(cb, label); } +pub fn jna_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x86>(cb, label); } +pub fn jnae_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn jnb_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jnbe_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x87>(cb, label); } +pub fn jnc_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jne_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x85>(cb, label); } +pub fn jng_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8E>(cb, label); } +pub fn jnge_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x8C>(cb, label); } +pub fn jnl_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8D>(cb, label); } +pub fn jnle_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x8F>(cb, label); } +pub fn jno_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x81>(cb, label); } +pub fn jnp_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8b>(cb, label); } +pub fn jns_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x89>(cb, label); } +pub fn jnz_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x85>(cb, label); } +pub fn jo_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x80>(cb, label); } +pub fn jp_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8A>(cb, label); } +pub fn jpe_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8A>(cb, label); } +pub fn jpo_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8B>(cb, label); } +pub fn js_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x88>(cb, label); } +pub fn jz_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x84>(cb, label); } + +pub fn jmp_label(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 5, |cb, src_addr, dst_addr| { + cb.write_byte(0xE9); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// Encode a relative jump to a pointer at a 32-bit offset (direct or conditional) +fn write_jcc_ptr(cb: &mut CodeBlock, op0: u8, op1: u8, dst_ptr: CodePtr) { + // Write the opcode + if op0 != 0xFF { + cb.write_byte(op0); + } + + cb.write_byte(op1); + + // Pointer to the end of this jump instruction + let end_ptr = cb.get_ptr(cb.write_pos + 4); + + // Compute the jump offset + let rel64 = dst_ptr.as_offset() - end_ptr.as_offset(); + + if rel64 >= i32::MIN.into() && rel64 <= i32::MAX.into() { + // Write the relative 32-bit jump offset + cb.write_int(rel64 as u64, 32); + } + else { + // Offset doesn't fit in 4 bytes. Report error. + //cb.dropped_bytes = true; + panic!("we should refactor to avoid dropped_bytes"); + } +} + +/// jcc - relative jumps to a pointer (32-bit offset) +pub fn ja_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x87, ptr); } +pub fn jae_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jb_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn jbe_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x86, ptr); } +pub fn jc_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn je_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x84, ptr); } +pub fn jg_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8F, ptr); } +pub fn jge_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8D, ptr); } +pub fn jl_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8C, ptr); } +pub fn jle_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8E, ptr); } +pub fn jna_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x86, ptr); } +pub fn jnae_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn jnb_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jnbe_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x87, ptr); } +pub fn jnc_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jne_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x85, ptr); } +pub fn jng_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8E, ptr); } +pub fn jnge_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8C, ptr); } +pub fn jnl_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8D, ptr); } +pub fn jnle_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8F, ptr); } +pub fn jno_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x81, ptr); } +pub fn jnp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8b, ptr); } +pub fn jns_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x89, ptr); } +pub fn jnz_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x85, ptr); } +pub fn jo_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x80, ptr); } +pub fn jp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8A, ptr); } +pub fn jpe_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8A, ptr); } +pub fn jpo_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8B, ptr); } +pub fn js_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x88, ptr); } +pub fn jz_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x84, ptr); } +pub fn jmp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0xFF, 0xE9, ptr); } + +/// jmp - Indirect jump near to an R/M operand. +pub fn jmp_rm(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(4), &[0xff]); +} + +// jmp - Jump with relative 32-bit offset +pub fn jmp32(cb: &mut CodeBlock, offset: i32) { + cb.write_byte(0xE9); + cb.write_int(offset as u64, 32); +} + +/// lea - Load Effective Address +pub fn lea(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(reg) = dst { + assert!(reg.num_bits == 64); + assert!(matches!(src, X86Opnd::Mem(_) | X86Opnd::IPRel(_))); + write_rm(cb, false, true, dst, src, None, &[0x8d]); + } else { + unreachable!(); + } +} + +/// mov - Data move operation +pub fn mov(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + fn emit_reg_imm(cb: &mut CodeBlock, reg: X86Reg, imm: X86Imm) { + // In case the source immediate could be zero extended to be 64 + // bit, we can use the 32-bit operands version of the instruction. + // For example, we can turn mov(rax, 0x34) into the equivalent + // mov(eax, 0x34). + if (reg.num_bits == 64) && u32::try_from(imm.value).is_ok() { + if reg.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0xB8, reg); + cb.write_int(imm.value as u64, 32); + } else if reg.num_bits == 64 && imm.num_bits <= 32 { + // Use 32-to-64 bit sign-extension when possible + write_rm(cb, false, true /* REX.w */, X86Opnd::None, X86Opnd::Reg(reg), Some(0) /* /0 */, &[0xc7]); + cb.write_int(imm.value as u64, 32); + } else { + if reg.num_bits == 16 { + cb.write_byte(0x66); + } + + if reg.rex_needed() || reg.num_bits == 64 { + write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); + } + + write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); + cb.write_int(imm.value as u64, reg.num_bits.into()); + } + } + match (dst, src) { + // R + Imm + (X86Opnd::Reg(reg), X86Opnd::Imm(imm)) => emit_reg_imm(cb, reg, imm), + // R + UImm + (X86Opnd::Reg(reg), X86Opnd::UImm(uimm)) => { + // u64->i64 type cast is a bit pattern no-op + let value: u64 = uimm.value; + let value = value as i64; + emit_reg_imm(cb, reg, X86Imm { num_bits: imm_num_bits(value), value }); + }, + // M + Imm + (X86Opnd::Mem(mem), X86Opnd::Imm(imm)) => { + assert!(imm.num_bits <= mem.num_bits); + + if mem.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, dst, None, &[0xc6]); + } else { + write_rm(cb, mem.num_bits == 16, mem.num_bits == 64, X86Opnd::None, dst, Some(0), &[0xc7]); + } + + let output_num_bits:u32 = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; + assert!( + mem.num_bits < 64 || imm_num_bits(imm.value) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); + cb.write_int(imm.value as u64, output_num_bits); + }, + // M + UImm + (X86Opnd::Mem(mem), X86Opnd::UImm(uimm)) => { + assert!(uimm.num_bits <= mem.num_bits); + + if mem.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, dst, None, &[0xc6]); + } + else { + write_rm(cb, mem.num_bits == 16, mem.num_bits == 64, X86Opnd::None, dst, Some(0), &[0xc7]); + } + + let output_num_bits = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; + assert!( + mem.num_bits < 64 || imm_num_bits(uimm.value as i64) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); + cb.write_int(uimm.value, output_num_bits); + }, + // * + Imm/UImm + (_, X86Opnd::Imm(_) | X86Opnd::UImm(_)) => unreachable!(), + // * + * + (_, _) => { + write_rm_multi( + cb, + 0x88, // opMemReg8 + 0x89, // opMemRegPref + 0x8A, // opRegMem8 + 0x8B, // opRegMemPref + 0xC6, // opMemImm8 + 0xFF, // opMemImmSml (not available) + 0xFF, // opMemImmLrg + None, // opExtImm + dst, + src + ); + } + }; +} + +/// A variant of mov used for always writing the value in 64 bits for GC offsets. +pub fn movabs(cb: &mut CodeBlock, dst: X86Opnd, value: u64) { + match dst { + X86Opnd::Reg(reg) => { + assert_eq!(reg.num_bits, 64); + write_rex(cb, true, 0, 0, reg.reg_no); + + write_opcode(cb, 0xb8, reg); + cb.write_int(value, 64); + }, + _ => unreachable!() + } +} + +/// movsx - Move with sign extension (signed integers) +pub fn movsx(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(_dst_reg) = dst { + assert!(matches!(src, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + let src_num_bits = src.num_bits(); + let dst_num_bits = dst.num_bits(); + assert!(src_num_bits < dst_num_bits); + + match src_num_bits { + 8 => write_rm(cb, dst_num_bits == 16, dst_num_bits == 64, dst, src, None, &[0x0f, 0xbe]), + 16 => write_rm(cb, dst_num_bits == 16, dst_num_bits == 64, dst, src, None, &[0x0f, 0xbf]), + 32 => write_rm(cb, false, true, dst, src, None, &[0x63]), + _ => unreachable!() + }; + } else { + unreachable!(); + } +} + +/// nop - Noop, one or multiple bytes long +pub fn nop(cb: &mut CodeBlock, length: u32) { + match length { + 0 => {}, + 1 => cb.write_byte(0x90), + 2 => cb.write_bytes(&[0x66, 0x90]), + 3 => cb.write_bytes(&[0x0f, 0x1f, 0x00]), + 4 => cb.write_bytes(&[0x0f, 0x1f, 0x40, 0x00]), + 5 => cb.write_bytes(&[0x0f, 0x1f, 0x44, 0x00, 0x00]), + 6 => cb.write_bytes(&[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00]), + 7 => cb.write_bytes(&[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00]), + 8 => cb.write_bytes(&[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00]), + 9 => cb.write_bytes(&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00]), + _ => { + let mut written: u32 = 0; + while written + 9 <= length { + nop(cb, 9); + written += 9; + } + nop(cb, length - written); + } + }; +} + +/// not - Bitwise NOT +pub fn not(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm_unary( + cb, + 0xf6, // opMemReg8 + 0xf7, // opMemRegPref + Some(0x02), // opExt + opnd + ); +} + +/// or - Bitwise OR +pub fn or(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x08, // opMemReg8 + 0x09, // opMemRegPref + 0x0A, // opRegMem8 + 0x0B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x01), // opExtImm + opnd0, + opnd1 + ); +} + +/// pop - Pop a register off the stack +pub fn pop(cb: &mut CodeBlock, opnd: X86Opnd) { + match opnd { + X86Opnd::Reg(reg) => { + assert!(reg.num_bits == 64); + + if opnd.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0x58, reg); + }, + X86Opnd::Mem(mem) => { + assert!(mem.num_bits == 64); + + write_rm(cb, false, false, X86Opnd::None, opnd, Some(0), &[0x8f]); + }, + _ => unreachable!() + }; +} + +/// popfq - Pop the flags register (64-bit) +pub fn popfq(cb: &mut CodeBlock) { + // REX.W + 0x9D + cb.write_bytes(&[0x48, 0x9d]); +} + +/// push - Push an operand on the stack +pub fn push(cb: &mut CodeBlock, opnd: X86Opnd) { + match opnd { + X86Opnd::Reg(reg) => { + if opnd.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0x50, reg); + }, + X86Opnd::Mem(_mem) => { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(6), &[0xff]); + }, + X86Opnd::Imm(X86Imm { value: 0, .. }) | X86Opnd::UImm(X86UImm { value: 0, .. }) => { + cb.write_bytes(&[0x6a, 0x00]); + } + _ => unreachable!() + } +} + +/// pushfq - Push the flags register (64-bit) +pub fn pushfq(cb: &mut CodeBlock) { + cb.write_byte(0x9C); +} + +/// ret - Return from call, popping only the return address +pub fn ret(cb: &mut CodeBlock) { + cb.write_byte(0xC3); +} + +// Encode a bitwise shift instruction +fn write_shift(cb: &mut CodeBlock, op_mem_one_pref: u8, op_mem_cl_pref: u8, op_mem_imm_pref: u8, op_ext: u8, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + // Check the size of opnd0 + let opnd_size = opnd0.num_bits(); + assert!(opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + + match opnd1 { + X86Opnd::UImm(imm) => { + if imm.value == 1 { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_one_pref]); + } else { + assert!(imm.num_bits <= 8); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_imm_pref]); + cb.write_byte(imm.value as u8); + } + } + + X86Opnd::Reg(reg) => { + // We can only use CL/RCX as the shift amount + assert!(reg.reg_no == RCX_REG.reg_no); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_cl_pref]); + } + + _ => { + unreachable!("unsupported operands: {:?}, {:?}", opnd0, opnd1); + } + } +} + +// sal - Shift arithmetic left +pub fn sal(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x04, + opnd0, + opnd1 + ); +} + +/// sar - Shift arithmetic right (signed) +pub fn sar(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x07, + opnd0, + opnd1 + ); +} + +// shl - Shift logical left +pub fn shl(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x04, + opnd0, + opnd1 + ); +} + +/// shr - Shift logical right (unsigned) +pub fn shr(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x05, + opnd0, + opnd1 + ); +} + +/// sub - Integer subtraction +pub fn sub(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x28, // opMemReg8 + 0x29, // opMemRegPref + 0x2A, // opRegMem8 + 0x2B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x05), // opExtImm + opnd0, + opnd1 + ); +} + +fn resize_opnd(opnd: X86Opnd, num_bits: u8) -> X86Opnd { + match opnd { + X86Opnd::Reg(reg) => { + let mut cloned = reg; + cloned.num_bits = num_bits; + X86Opnd::Reg(cloned) + }, + X86Opnd::Mem(mem) => { + let mut cloned = mem; + cloned.num_bits = num_bits; + X86Opnd::Mem(cloned) + }, + _ => unreachable!() + } +} + +/// test - Logical Compare +pub fn test(cb: &mut CodeBlock, rm_opnd: X86Opnd, test_opnd: X86Opnd) { + assert!(matches!(rm_opnd, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + let rm_num_bits = rm_opnd.num_bits(); + + match test_opnd { + X86Opnd::UImm(uimm) => { + assert!(uimm.num_bits <= 32); + assert!(uimm.num_bits <= rm_num_bits); + + // Use the smallest operand size possible + assert!(rm_num_bits % 8 == 0); + let rm_resized = resize_opnd(rm_opnd, uimm.num_bits); + + if uimm.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, rm_resized, Some(0x00), &[0xf6]); + cb.write_int(uimm.value, uimm.num_bits.into()); + } else { + write_rm(cb, uimm.num_bits == 16, false, X86Opnd::None, rm_resized, Some(0x00), &[0xf7]); + cb.write_int(uimm.value, uimm.num_bits.into()); + } + }, + X86Opnd::Imm(imm) => { + // This mode only applies to 64-bit R/M operands with 32-bit signed immediates + assert!(imm.num_bits <= 32); + assert!(rm_num_bits == 64); + + write_rm(cb, false, true, X86Opnd::None, rm_opnd, Some(0x00), &[0xf7]); + cb.write_int(imm.value as u64, 32); + }, + X86Opnd::Reg(reg) => { + assert!(reg.num_bits == rm_num_bits); + + if rm_num_bits == 8 { + write_rm(cb, false, false, test_opnd, rm_opnd, None, &[0x84]); + } else { + write_rm(cb, rm_num_bits == 16, rm_num_bits == 64, test_opnd, rm_opnd, None, &[0x85]); + } + }, + _ => unreachable!("unexpected operands for test: {rm_opnd:?}, {test_opnd:?}") + }; +} + +/// Undefined opcode +pub fn ud2(cb: &mut CodeBlock) { + cb.write_bytes(&[0x0f, 0x0b]); +} + +/// xchg - Exchange Register/Memory with Register +pub fn xchg(cb: &mut CodeBlock, rm_opnd: X86Opnd, r_opnd: X86Opnd) { + if let (X86Opnd::Reg(rm_reg), X86Opnd::Reg(r_reg)) = (rm_opnd, r_opnd) { + assert!(rm_reg.num_bits == 64); + assert!(r_reg.num_bits == 64); + + // If we're exchanging with RAX + if rm_reg.reg_no == RAX_REG_NO { + // Write the REX byte + write_rex(cb, true, 0, 0, r_reg.reg_no); + + // Write the opcode and register number + cb.write_byte(0x90 + (r_reg.reg_no & 7)); + } else { + write_rm(cb, false, true, r_opnd, rm_opnd, None, &[0x87]); + } + } else { + unreachable!(); + } +} + +/// xor - Exclusive bitwise OR +pub fn xor(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x30, // opMemReg8 + 0x31, // opMemRegPref + 0x32, // opRegMem8 + 0x33, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x06), // opExtImm + opnd0, + opnd1 + ); +} + +impl fmt::Display for X86Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match X86Opnd::Reg(*self) { + RAX => write!(f, "rax"), + RCX => write!(f, "rcx"), + RDX => write!(f, "rdx"), + RBX => write!(f, "rbx"), + RSP => write!(f, "rsp"), + RBP => write!(f, "rbp"), + RSI => write!(f, "rsi"), + RDI => write!(f, "rdi"), + R8 => write!(f, "r8"), + R9 => write!(f, "r9"), + R10 => write!(f, "r10"), + R11 => write!(f, "r11"), + R12 => write!(f, "r12"), + R13 => write!(f, "r13"), + R14 => write!(f, "r14"), + R15 => write!(f, "r15"), + EAX => write!(f, "eax"), + ECX => write!(f, "ecx"), + EDX => write!(f, "edx"), + EBX => write!(f, "ebx"), + ESP => write!(f, "esp"), + EBP => write!(f, "ebp"), + ESI => write!(f, "esi"), + EDI => write!(f, "edi"), + R8D => write!(f, "r8d"), + R9D => write!(f, "r9d"), + R10D => write!(f, "r10d"), + R11D => write!(f, "r11d"), + R12D => write!(f, "r12d"), + R13D => write!(f, "r13d"), + R14D => write!(f, "r14d"), + R15D => write!(f, "r15d"), + AX => write!(f, "ax"), + CX => write!(f, "cx"), + DX => write!(f, "dx"), + BX => write!(f, "bx"), + BP => write!(f, "bp"), + SI => write!(f, "si"), + DI => write!(f, "di"), + R8W => write!(f, "r8w"), + R9W => write!(f, "r9w"), + R10W => write!(f, "r10w"), + R11W => write!(f, "r11w"), + R12W => write!(f, "r12w"), + R13W => write!(f, "r13w"), + R14W => write!(f, "r14w"), + R15W => write!(f, "r15w"), + AL => write!(f, "al"), + CL => write!(f, "cl"), + DL => write!(f, "dl"), + BL => write!(f, "bl"), + SPL => write!(f, "spl"), + BPL => write!(f, "bpl"), + SIL => write!(f, "sil"), + DIL => write!(f, "dil"), + R8B => write!(f, "r8b"), + R9B => write!(f, "r9b"), + R10B => write!(f, "r10b"), + R11B => write!(f, "r11b"), + R12B => write!(f, "r12b"), + R13B => write!(f, "r13b"), + R14B => write!(f, "r14b"), + R15B => write!(f, "r15b"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/x86_64/tests.rs b/zjit/src/asm/x86_64/tests.rs new file mode 100644 index 0000000000..0dfee26496 --- /dev/null +++ b/zjit/src/asm/x86_64/tests.rs @@ -0,0 +1,966 @@ +#![cfg(test)] + +use insta::assert_snapshot; + +#[cfg(feature = "disasm")] +use crate::disasms; +use crate::{asm::x86_64::*, hexdumps, assert_disasm_snapshot}; + +/// Check that the bytes for an instruction sequence match a hex string +fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + assert_eq!(format!("{:x}", cb), bytes); +} + +fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb +} + +#[test] +fn test_add() { + let cb01 = compile(|cb| add(cb, CL, imm_opnd(3))); + let cb02 = compile(|cb| add(cb, CL, BL)); + let cb03 = compile(|cb| add(cb, CL, SPL)); + let cb04 = compile(|cb| add(cb, CX, BX)); + let cb05 = compile(|cb| add(cb, RAX, RBX)); + let cb06 = compile(|cb| add(cb, ECX, EDX)); + let cb07 = compile(|cb| add(cb, RDX, R14)); + let cb08 = compile(|cb| add(cb, mem_opnd(64, RAX, 0), RDX)); + let cb09 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 0))); + let cb10 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 8))); + let cb11 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 255))); + let cb12 = compile(|cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); + let cb13 = compile(|cb| add(cb, mem_opnd(32, RAX, 0), EDX)); + let cb14 = compile(|cb| add(cb, RSP, imm_opnd(8))); + let cb15 = compile(|cb| add(cb, ECX, imm_opnd(8))); + let cb16 = compile(|cb| add(cb, ECX, imm_opnd(255))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @" + 0x0: add cl, 3 + 0x0: add cl, bl + 0x0: add cl, spl + 0x0: add cx, bx + 0x0: add rax, rbx + 0x0: add ecx, edx + 0x0: add rdx, r14 + 0x0: add qword ptr [rax], rdx + 0x0: add rdx, qword ptr [rax] + 0x0: add rdx, qword ptr [rax + 8] + 0x0: add rdx, qword ptr [rax + 0xff] + 0x0: add qword ptr [rax + 0x7f], 0xff + 0x0: add dword ptr [rax], edx + 0x0: add rsp, 8 + 0x0: add ecx, 8 + 0x0: add ecx, 0xff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @" + 80c103 + 00d9 + 4000e1 + 6601d9 + 4801d8 + 01d1 + 4c01f2 + 480110 + 480310 + 48035008 + 480390ff000000 + 4881407fff000000 + 0110 + 4883c408 + 83c108 + 81c1ff000000 + "); +} + +#[test] +fn test_add_unsigned() { + // ADD r/m8, imm8 + let cb1 = compile(|cb| add(cb, R8B, uimm_opnd(1))); + let cb2 = compile(|cb| add(cb, R8B, imm_opnd(i8::MAX.into()))); + // ADD r/m16, imm16 + let cb3 = compile(|cb| add(cb, R8W, uimm_opnd(1))); + let cb4 = compile(|cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); + // ADD r/m32, imm32 + let cb5 = compile(|cb| add(cb, R8D, uimm_opnd(1))); + let cb6 = compile(|cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); + // ADD r/m64, imm32 + let cb7 = compile(|cb| add(cb, R8, uimm_opnd(1))); + let cb8 = compile(|cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: add r8b, 1 + 0x0: add r8b, 0x7f + 0x0: add r8w, 1 + 0x0: add r8w, 0x7fff + 0x0: add r8d, 1 + 0x0: add r8d, 0x7fffffff + 0x0: add r8, 1 + 0x0: add r8, 0x7fffffff + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 4180c001 + 4180c07f + 664183c001 + 664181c0ff7f + 4183c001 + 4181c0ffffff7f + 4983c001 + 4981c0ffffff7f + "); +} + +#[test] +fn test_and() { + let cb1 = compile(|cb| and(cb, EBP, R12D)); + let cb2 = compile(|cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: and ebp, r12d + 0x0: and qword ptr [rax], 8 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 4421e5 + 48832008 + "); +} + +#[test] +fn test_call_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("fn".to_owned()); + call_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); +} + +#[test] +fn test_call_ptr() { + // calling a lower address + let cb = compile(|cb| { + let ptr = cb.get_write_ptr(); + call_ptr(cb, RAX, ptr.raw_ptr(cb)); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); +} + +#[test] +fn test_call_reg() { + let cb = compile(|cb| call(cb, RAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call rax"); + assert_snapshot!(cb.hexdump(), @"ffd0"); +} + +#[test] +fn test_call_mem() { + let cb = compile(|cb| call(cb, mem_opnd(64, RSP, 8))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call qword ptr [rsp + 8]"); + assert_snapshot!(cb.hexdump(), @"ff542408"); +} + +#[test] +fn test_cmovcc() { + let cb1 = compile(|cb| cmovg(cb, ESI, EDI)); + let cb2 = compile(|cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); + let cb3 = compile(|cb| cmovl(cb, EAX, ECX)); + let cb4 = compile(|cb| cmovl(cb, RBX, RBP)); + let cb5 = compile(|cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: cmovg esi, edi + 0x0: cmovg esi, dword ptr [rbp + 0xc] + 0x0: cmovl eax, ecx + 0x0: cmovl rbx, rbp + 0x0: cmovle esi, dword ptr [rsp + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 0f4ff7 + 0f4f750c + 0f4cc1 + 480f4cdd + 0f4e742404 + "); +} + +#[test] +fn test_cmp() { + let cb1 = compile(|cb| cmp(cb, CL, DL)); + let cb2 = compile(|cb| cmp(cb, ECX, EDI)); + let cb3 = compile(|cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); + let cb4 = compile(|cb| cmp(cb, RAX, imm_opnd(2))); + let cb5 = compile(|cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: cmp cl, dl + 0x0: cmp ecx, edi + 0x0: cmp rdx, qword ptr [r12] + 0x0: cmp rax, 2 + 0x0: cmp ecx, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 38d1 + 39f9 + 493b1424 + 4883f802 + 81f900000080 + "); +} + +#[test] +fn test_cqo() { + let cb = compile(cqo); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cqo"); + assert_snapshot!(cb.hexdump(), @"4899"); +} + +#[test] +fn test_imul() { + let cb1 = compile(|cb| imul(cb, RAX, RBX)); + let cb2 = compile(|cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: imul rax, rbx + 0x0: imul rdx, qword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 480fafc3 + 480faf10 + "); +} + +#[test] +#[should_panic] +fn test_imul_mem_reg() { + // imul doesn't have (Mem, Reg) encoding. Since multiplication is communicative, imul() could + // swap operands. However, x86_scratch_split may need to move the result to the output operand, + // which can be complicated if the assembler may sometimes change the result operand. + // So x86_scratch_split should be responsible for that swap, not the assembler. + compile(|cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); +} + +#[test] +fn test_jge_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + jge_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jge 0"); + assert_snapshot!(cb.hexdump(), @"0f8dfaffffff"); +} + +#[test] +fn test_jmp_label() { + // Forward jump + let cb1 = compile(|cb| { + let label_idx = cb.new_label("next".to_owned()); + jmp_label(cb, label_idx); + cb.write_label(label_idx); + cb.link_labels().unwrap(); + }); + // Backwards jump + let cb2 = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + cb.write_label(label_idx); + jmp_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: jmp 5 + 0x0: jmp 0 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + e900000000 + e9fbffffff + "); +} + +#[test] +fn test_jmp_rm() { + let cb = compile(|cb| jmp_rm(cb, R12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jmp r12"); + assert_snapshot!(cb.hexdump(), @"41ffe4"); +} + +#[test] +fn test_jo_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + jo_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jo 0"); + assert_snapshot!(cb.hexdump(), @"0f80faffffff"); +} + +#[test] +fn test_lea() { + let cb1 = compile(|cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); + let cb2 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); + let cb3 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); + let cb4 = compile(|cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @" + 0x0: lea rdx, [rcx + 8] + 0x0: lea rax, [rip] + 0x0: lea rax, [rip + 5] + 0x0: lea rdi, [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @" + 488d5108 + 488d0500000000 + 488d0505000000 + 488d3d05000000 + "); +} + +#[test] +fn test_mov() { + let cb01 = compile(|cb| mov(cb, EAX, imm_opnd(7))); + let cb02 = compile(|cb| mov(cb, EAX, imm_opnd(-3))); + let cb03 = compile(|cb| mov(cb, R15, imm_opnd(3))); + let cb04 = compile(|cb| mov(cb, EAX, EBX)); + let cb05 = compile(|cb| mov(cb, EAX, ECX)); + let cb06 = compile(|cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); + let cb07 = compile(|cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); + // Test `mov rax, 3` => `mov eax, 3` optimization + let cb08 = compile(|cb| mov(cb, R8, imm_opnd(0x34))); + let cb09 = compile(|cb| mov(cb, R8, imm_opnd(0x80000000))); + let cb10 = compile(|cb| mov(cb, R8, imm_opnd(-1))); + let cb11 = compile(|cb| mov(cb, RAX, imm_opnd(0x34))); + let cb12 = compile(|cb| mov(cb, RAX, imm_opnd(-18014398509481982))); + let cb13 = compile(|cb| mov(cb, RAX, imm_opnd(0x80000000))); + let cb14 = compile(|cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword + let cb15 = compile(|cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword + let cb16 = compile(|cb| mov(cb, CL, R9B)); + let cb17 = compile(|cb| mov(cb, RBX, RAX)); + let cb18 = compile(|cb| mov(cb, RDI, RBX)); + let cb19 = compile(|cb| mov(cb, SIL, imm_opnd(11))); + let cb20 = compile(|cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); + let cb21 = compile(|cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); + //let cb = compile(|cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? + let cb22 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); + let cb23 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), uimm_opnd(0x80000001))); + let cb24 = compile(|cb| mov(cb, mem_opnd(32, R8, 20), EBX)); + let cb25 = compile(|cb| mov(cb, mem_opnd(64, R11, 0), R10)); + let cb26 = compile(|cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + + assert_disasm_snapshot!(disasms!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26, + ), @" + 0x0: mov eax, 7 + 0x0: mov eax, 0xfffffffd + 0x0: mov r15d, 3 + 0x0: mov eax, ebx + 0x0: mov eax, ecx + 0x0: mov edx, dword ptr [rbx + 0x80] + 0x0: mov rax, qword ptr [rsp + 4] + 0x0: mov r8d, 0x34 + 0x0: mov r8d, 0x80000000 + 0x0: mov r8, 0xffffffffffffffff + 0x0: mov eax, 0x34 + 0x0: movabs rax, 0xffc0000000000002 + 0x0: mov eax, 0x80000000 + 0x0: mov rax, 0xffffffffffffffcc + 0x0: mov rax, 0xffffffffffffffff + 0x0: mov cl, r9b + 0x0: mov rbx, rax + 0x0: mov rdi, rbx + 0x0: mov sil, 0xb + 0x0: mov byte ptr [rsp], 0xfd + 0x0: mov qword ptr [rdi + 8], 1 + 0x0: mov dword ptr [rax + 4], 0x11 + 0x0: mov dword ptr [rax + 4], 0x80000001 + 0x0: mov dword ptr [r8 + 0x14], ebx + 0x0: mov qword ptr [r11], r10 + 0x0: mov qword ptr [rdx - 8], 0xfffffffffffffff4 + "); + + assert_snapshot!(hexdumps!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26 + ), @" + b807000000 + b8fdffffff + 41bf03000000 + 89d8 + 89c8 + 8b9380000000 + 488b442404 + 41b834000000 + 41b800000080 + 49c7c0ffffffff + b834000000 + 48b8020000000000c0ff + b800000080 + 48c7c0ccffffff + 48c7c0ffffffff + 4488c9 + 4889c3 + 4889df + 40b60b + c60424fd + 48c7470801000000 + c7400411000000 + c7400401000080 + 41895814 + 4d8913 + 48c742f8f4ffffff + "); +} + +#[test] +fn test_movabs() { + let cb1 = compile(|cb| movabs(cb, R8, 0x34)); + let cb2 = compile(|cb| movabs(cb, R8, 0x80000000)); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: movabs r8, 0x34 + 0x0: movabs r8, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 49b83400000000000000 + 49b80000008000000000 + "); +} + +#[test] +fn test_mov_unsigned() { + // MOV AL, imm8 + let cb01 = compile(|cb| mov(cb, AL, uimm_opnd(1))); + let cb02 = compile(|cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); + // MOV AX, imm16 + let cb03 = compile(|cb| mov(cb, AX, uimm_opnd(1))); + let cb04 = compile(|cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); + // MOV EAX, imm32 + let cb05 = compile(|cb| mov(cb, EAX, uimm_opnd(1))); + let cb06 = compile(|cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); + let cb07 = compile(|cb| mov(cb, R8, uimm_opnd(0))); + let cb08 = compile(|cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); + // MOV RAX, imm64, will move down into EAX since it fits into 32 bits + let cb09 = compile(|cb| mov(cb, RAX, uimm_opnd(1))); + let cb10 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); + // MOV RAX, imm64, will not move down into EAX since it does not fit into 32 bits + let cb11 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); + let cb12 = compile(|cb| mov(cb, RAX, uimm_opnd(u64::MAX))); + let cb13 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + // MOV r8, imm8 + let cb14 = compile(|cb| mov(cb, R8B, uimm_opnd(1))); + let cb15 = compile(|cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); + // MOV r16, imm16 + let cb16 = compile(|cb| mov(cb, R8W, uimm_opnd(1))); + let cb17 = compile(|cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); + // MOV r32, imm32 + let cb18 = compile(|cb| mov(cb, R8D, uimm_opnd(1))); + let cb19 = compile(|cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); + // MOV r64, imm64, will move down into 32 bit since it fits into 32 bits + let cb20 = compile(|cb| mov(cb, R8, uimm_opnd(1))); + // MOV r64, imm64, will not move down into 32 bit since it does not fit into 32 bits + let cb21 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @" + 0x0: mov al, 1 + 0x0: mov al, 0xff + 0x0: mov ax, 1 + 0x0: mov ax, 0xffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: mov r8d, 0 + 0x0: mov r8d, 0xffffffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: movabs rax, 0x100000000 + 0x0: mov rax, 0xffffffffffffffff + 0x0: mov r8, 0xffffffffffffffff + 0x0: mov r8b, 1 + 0x0: mov r8b, 0xff + 0x0: mov r8w, 1 + 0x0: mov r8w, 0xffff + 0x0: mov r8d, 1 + 0x0: mov r8d, 0xffffffff + 0x0: mov r8d, 1 + 0x0: mov r8, 0xffffffffffffffff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @" + b001 + b0ff + 66b80100 + 66b8ffff + b801000000 + b8ffffffff + 41b800000000 + 41b8ffffffff + b801000000 + b8ffffffff + 48b80000000001000000 + 48c7c0ffffffff + 49c7c0ffffffff + 41b001 + 41b0ff + 6641b80100 + 6641b8ffff + 41b801000000 + 41b8ffffffff + 41b801000000 + 49c7c0ffffffff + "); +} + +#[test] +fn test_mov_iprel() { + let cb1 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); + let cb2 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); + let cb3 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); + let cb4 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); + let cb5 = compile(|cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: mov eax, dword ptr [rip] + 0x0: mov eax, dword ptr [rip + 5] + 0x0: mov rax, qword ptr [rip] + 0x0: mov rax, qword ptr [rip + 5] + 0x0: mov rdi, qword ptr [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 8b0500000000 + 8b0505000000 + 488b0500000000 + 488b0505000000 + 488b3d05000000 + "); +} + +#[test] +fn test_movsx() { + let cb1 = compile(|cb| movsx(cb, AX, AL)); + let cb2 = compile(|cb| movsx(cb, EDX, AL)); + let cb3 = compile(|cb| movsx(cb, RAX, BL)); + let cb4 = compile(|cb| movsx(cb, ECX, AX)); + let cb5 = compile(|cb| movsx(cb, R11, CL)); + let cb6 = compile(|cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); + let cb7 = compile(|cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); + let cb8 = compile(|cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: movsx ax, al + 0x0: movsx edx, al + 0x0: movsx rax, bl + 0x0: movsx ecx, ax + 0x0: movsx r11, cl + 0x0: movsxd r10, dword ptr [rsp + 0xc] + 0x0: movsx rax, byte ptr [rsp] + 0x0: movsx rdx, word ptr [r13 + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 660fbec0 + 0fbed0 + 480fbec3 + 0fbfc8 + 4c0fbed9 + 4c6354240c + 480fbe0424 + 490fbf5504 + "); +} + +#[test] +fn test_nop() { + let cb01 = compile(|cb| nop(cb, 1)); + let cb02 = compile(|cb| nop(cb, 2)); + let cb03 = compile(|cb| nop(cb, 3)); + let cb04 = compile(|cb| nop(cb, 4)); + let cb05 = compile(|cb| nop(cb, 5)); + let cb06 = compile(|cb| nop(cb, 6)); + let cb07 = compile(|cb| nop(cb, 7)); + let cb08 = compile(|cb| nop(cb, 8)); + let cb09 = compile(|cb| nop(cb, 9)); + let cb10 = compile(|cb| nop(cb, 10)); + let cb11 = compile(|cb| nop(cb, 11)); + let cb12 = compile(|cb| nop(cb, 12)); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @" + 0x0: nop + 0x0: nop + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop dword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @" + 90 + 6690 + 0f1f00 + 0f1f4000 + 0f1f440000 + 660f1f440000 + 0f1f8000000000 + 0f1f840000000000 + 660f1f840000000000 + 660f1f84000000000090 + 660f1f8400000000006690 + 660f1f8400000000000f1f00 + "); +} + +#[test] +fn test_not() { + let cb01 = compile(|cb| not(cb, AX)); + let cb02 = compile(|cb| not(cb, EAX)); + let cb03 = compile(|cb| not(cb, mem_opnd(64, R12, 0))); + let cb04 = compile(|cb| not(cb, mem_opnd(32, RSP, 301))); + let cb05 = compile(|cb| not(cb, mem_opnd(32, RSP, 0))); + let cb06 = compile(|cb| not(cb, mem_opnd(32, RSP, 3))); + let cb07 = compile(|cb| not(cb, mem_opnd(32, RBP, 0))); + let cb08 = compile(|cb| not(cb, mem_opnd(32, RBP, 13))); + let cb09 = compile(|cb| not(cb, RAX)); + let cb10 = compile(|cb| not(cb, R11)); + let cb11 = compile(|cb| not(cb, mem_opnd(32, RAX, 0))); + let cb12 = compile(|cb| not(cb, mem_opnd(32, RSI, 0))); + let cb13 = compile(|cb| not(cb, mem_opnd(32, RDI, 0))); + let cb14 = compile(|cb| not(cb, mem_opnd(32, RDX, 55))); + let cb15 = compile(|cb| not(cb, mem_opnd(32, RDX, 1337))); + let cb16 = compile(|cb| not(cb, mem_opnd(32, RDX, -55))); + let cb17 = compile(|cb| not(cb, mem_opnd(32, RDX, -555))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @" + 0x0: not ax + 0x0: not eax + 0x0: not qword ptr [r12] + 0x0: not dword ptr [rsp + 0x12d] + 0x0: not dword ptr [rsp] + 0x0: not dword ptr [rsp + 3] + 0x0: not dword ptr [rbp] + 0x0: not dword ptr [rbp + 0xd] + 0x0: not rax + 0x0: not r11 + 0x0: not dword ptr [rax] + 0x0: not dword ptr [rsi] + 0x0: not dword ptr [rdi] + 0x0: not dword ptr [rdx + 0x37] + 0x0: not dword ptr [rdx + 0x539] + 0x0: not dword ptr [rdx - 0x37] + 0x0: not dword ptr [rdx - 0x22b] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @" + 66f7d0 + f7d0 + 49f71424 + f794242d010000 + f71424 + f7542403 + f75500 + f7550d + 48f7d0 + 49f7d3 + f710 + f716 + f717 + f75237 + f79239050000 + f752c9 + f792d5fdffff + "); +} + +#[test] +fn test_or() { + let cb = compile(|cb| or(cb, EDX, ESI)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or edx, esi"); + assert_snapshot!(cb.hexdump(), @"09f2"); +} + +#[test] +fn test_pop() { + let cb01 = compile(|cb| pop(cb, RAX)); + let cb02 = compile(|cb| pop(cb, RBX)); + let cb03 = compile(|cb| pop(cb, RSP)); + let cb04 = compile(|cb| pop(cb, RBP)); + let cb05 = compile(|cb| pop(cb, R12)); + let cb06 = compile(|cb| pop(cb, mem_opnd(64, RAX, 0))); + let cb07 = compile(|cb| pop(cb, mem_opnd(64, R8, 0))); + let cb08 = compile(|cb| pop(cb, mem_opnd(64, R8, 3))); + let cb09 = compile(|cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb10 = compile(|cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @" + 0x0: pop rax + 0x0: pop rbx + 0x0: pop rsp + 0x0: pop rbp + 0x0: pop r12 + 0x0: pop qword ptr [rax] + 0x0: pop qword ptr [r8] + 0x0: pop qword ptr [r8 + 3] + 0x0: pop qword ptr [rax + rcx*8 + 3] + 0x0: pop qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @" + 58 + 5b + 5c + 5d + 415c + 8f00 + 418f00 + 418f4003 + 8f44c803 + 418f44c803 + "); +} + +#[test] +fn test_push() { + let cb1 = compile(|cb| push(cb, RAX)); + let cb2 = compile(|cb| push(cb, RBX)); + let cb3 = compile(|cb| push(cb, R12)); + let cb4 = compile(|cb| push(cb, mem_opnd(64, RAX, 0))); + let cb5 = compile(|cb| push(cb, mem_opnd(64, R8, 0))); + let cb6 = compile(|cb| push(cb, mem_opnd(64, R8, 3))); + let cb7 = compile(|cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb8 = compile(|cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: push rax + 0x0: push rbx + 0x0: push r12 + 0x0: push qword ptr [rax] + 0x0: push qword ptr [r8] + 0x0: push qword ptr [r8 + 3] + 0x0: push qword ptr [rax + rcx*8 + 3] + 0x0: push qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 50 + 53 + 4154 + ff30 + 41ff30 + 41ff7003 + ff74c803 + 41ff74c803 + "); +} + +#[test] +fn test_ret() { + let cb = compile(ret); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c3"); +} + +#[test] +fn test_sal() { + let cb1 = compile(|cb| sal(cb, CX, uimm_opnd(1))); + let cb2 = compile(|cb| sal(cb, ECX, uimm_opnd(1))); + let cb3 = compile(|cb| sal(cb, EBP, uimm_opnd(5))); + let cb4 = compile(|cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); + let cb5 = compile(|cb| sal(cb, RCX, CL)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: shl cx, 1 + 0x0: shl ecx, 1 + 0x0: shl ebp, 5 + 0x0: shl dword ptr [rsp + 0x44], 1 + 0x0: shl rcx, cl + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 66d1e1 + d1e1 + c1e505 + d1642444 + 48d3e1 + "); +} + +#[test] +fn test_sar() { + let cb = compile(|cb| sar(cb, EDX, uimm_opnd(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sar edx, 1"); + assert_snapshot!(cb.hexdump(), @"d1fa"); +} + +#[test] +fn test_shr() { + let cb = compile(|cb| shr(cb, R14, uimm_opnd(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: shr r14, 7"); + assert_snapshot!(cb.hexdump(), @"49c1ee07"); +} + +#[test] +fn test_sub() { + let cb1 = compile(|cb| sub(cb, EAX, imm_opnd(1))); + let cb2 = compile(|cb| sub(cb, RAX, imm_opnd(2))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: sub eax, 1 + 0x0: sub rax, 2 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 83e801 + 4883e802 + "); +} + +#[test] +#[should_panic] +fn test_sub_uimm_too_large() { + // This immediate becomes a different value after + // sign extension, so not safe to encode. + compile(|cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); +} + +#[test] +fn test_test() { + let cb01 = compile(|cb| test(cb, AL, AL)); + let cb02 = compile(|cb| test(cb, AX, AX)); + let cb03 = compile(|cb| test(cb, CL, uimm_opnd(8))); + let cb04 = compile(|cb| test(cb, DL, uimm_opnd(7))); + let cb05 = compile(|cb| test(cb, RCX, uimm_opnd(8))); + let cb06 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); + let cb07 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); + let cb08 = compile(|cb| test(cb, DX, uimm_opnd(0xffff))); + let cb09 = compile(|cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); + let cb10 = compile(|cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); + let cb11 = compile(|cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); + let cb12 = compile(|cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); + let cb13 = compile(|cb| test(cb, mem_opnd(32, RSI, 64), EAX)); + let cb14 = compile(|cb| test(cb, mem_opnd(64, RDI, 42), RAX)); + let cb15 = compile(|cb| test(cb, RAX, RAX)); + let cb16 = compile(|cb| test(cb, RAX, RSI)); + let cb17 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); + let cb18 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); + let cb19 = compile(|cb| test(cb, RCX, imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @" + 0x0: test al, al + 0x0: test ax, ax + 0x0: test cl, 8 + 0x0: test dl, 7 + 0x0: test cl, 8 + 0x0: test byte ptr [rdx + 8], 8 + 0x0: test byte ptr [rdx + 8], 0xff + 0x0: test dx, 0xffff + 0x0: test word ptr [rdx + 8], 0xffff + 0x0: test byte ptr [rsi], 1 + 0x0: test byte ptr [rsi + 0x10], 1 + 0x0: test byte ptr [rsi - 0x10], 1 + 0x0: test dword ptr [rsi + 0x40], eax + 0x0: test qword ptr [rdi + 0x2a], rax + 0x0: test rax, rax + 0x0: test rax, rsi + 0x0: test qword ptr [rsi + 0x40], -9 + 0x0: test qword ptr [rsi + 0x40], 8 + 0x0: test rcx, 8 + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @" + 84c0 + 6685c0 + f6c108 + f6c207 + f6c108 + f6420808 + f64208ff + 66f7c2ffff + 66f74208ffff + f60601 + f6461001 + f646f001 + 854640 + 4885472a + 4885c0 + 4885f0 + 48f74640f7ffffff + 48f7464008000000 + 48f7c108000000 + "); +} + +#[test] +fn test_xchg() { + let cb1 = compile(|cb| xchg(cb, RAX, RCX)); + let cb2 = compile(|cb| xchg(cb, RAX, R13)); + let cb3 = compile(|cb| xchg(cb, RCX, RBX)); + let cb4 = compile(|cb| xchg(cb, R9, R15)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @" + 0x0: xchg rcx, rax + 0x0: xchg r13, rax + 0x0: xchg rcx, rbx + 0x0: xchg r9, r15 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @" + 4891 + 4995 + 4887d9 + 4d87f9 + "); +} + +#[test] +fn test_xor() { + let cb = compile(|cb| xor(cb, EAX, EAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor eax, eax"); + assert_snapshot!(cb.hexdump(), @"31c0"); +} + +#[test] +#[cfg(feature = "disasm")] +fn basic_capstone_usage() -> std::result::Result<(), capstone::Error> { + // Test drive Capstone with simple input + use capstone::prelude::*; + let cs = Capstone::new() + .x86() + .mode(arch::x86::ArchMode::Mode64) + .syntax(arch::x86::ArchSyntax::Intel) + .build()?; + + let insns = cs.disasm_all(&[0xCC], 0x1000)?; + + match insns.as_ref() { + [insn] => { + assert_eq!(Some("int3"), insn.mnemonic()); + Ok(()) + } + _ => Err(capstone::Error::CustomError( + "expected to disassemble to int3", + )), + } +} diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs new file mode 100644 index 0000000000..32ba1e3de0 --- /dev/null +++ b/zjit/src/backend/arm64/mod.rs @@ -0,0 +1,2929 @@ +use crate::asm::{CodeBlock, Label}; +use crate::asm::arm64::*; +use crate::codegen::split_patch_point; +use crate::cruby::*; +use crate::backend::lir::*; +use crate::options::asm_dump; +use crate::stats::{CompileError, trace_compile_phase}; +use crate::virtualmem::CodePtr; +use crate::cast::*; + +// Use the arm64 register type for this platform +pub type Reg = A64Reg; + +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_no } +} + +// Callee-saved registers +pub const CFP: Opnd = Opnd::Reg(X19_REG); +pub const EC: Opnd = Opnd::Reg(X20_REG); +pub const SP: Opnd = Opnd::Reg(X21_REG); + +// C argument registers on this platform +pub const C_ARG_OPNDS: [Opnd; 6] = [ + Opnd::Reg(X0_REG), + Opnd::Reg(X1_REG), + Opnd::Reg(X2_REG), + Opnd::Reg(X3_REG), + Opnd::Reg(X4_REG), + Opnd::Reg(X5_REG) +]; + +// Make sure we're using the same c args everywhere +const _: () = ::core::assert!(C_ARG_OPNDS.len() == C_ARG_REGS.len()); +const _: () = ::core::assert!(C_ARG_OPNDS.len() == C_ARG_REGREGS.len()); + +// C return value register on this platform +pub const C_RET_REG: Reg = X0_REG; +pub const C_RET_OPND: Opnd = Opnd::Reg(X0_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(XZR_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(X29_REG); + +// These constants define the way we work with Arm64's stack pointer. The stack +// pointer always needs to be aligned to a 16-byte boundary. +pub const C_SP_REG: A64Opnd = X31; +pub const C_SP_STEP: i32 = 16; + +impl CodeBlock { + // The maximum number of bytes that can be generated by emit_jmp_ptr. + pub fn jmp_ptr_bytes(&self) -> usize { + // b instruction's offset is encoded as imm26 times 4. It can jump to + // +/-128MiB, so this can be used when --yjit-exec-mem-size <= 128. + /* + let num_insns = if b_offset_fits_bits(self.virtual_region_size() as i64 / 4) { + 1 // b instruction + } else { + 5 // 4 instructions to load a 64-bit absolute address + br instruction + }; + */ + let num_insns = 5; // TODO: support virtual_region_size() check + num_insns * 4 + } + + // The maximum number of instructions that can be generated by emit_conditional_jump. + fn conditional_jump_insns(&self) -> i32 { + // The worst case is instructions for a jump + bcond. + self.jmp_ptr_bytes() as i32 / 4 + 1 + } +} + +/// Map Opnd to A64Opnd +impl From<Opnd> for A64Opnd { + fn from(opnd: Opnd) -> Self { + match opnd { + Opnd::UImm(value) => A64Opnd::new_uimm(value), + Opnd::Imm(value) => A64Opnd::new_imm(value), + Opnd::Reg(reg) => A64Opnd::Reg(reg), + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), num_bits, disp }) => { + A64Opnd::new_mem(num_bits, A64Opnd::Reg(A64Reg { num_bits, reg_no }), disp) + }, + Opnd::Mem(Mem { base: MemBase::VReg(_), .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::VReg base") + }, + Opnd::Mem(Mem { base: MemBase::Stack { .. } | MemBase::StackIndirect { .. }, .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::Stack/StackIndirect base") + }, + Opnd::VReg { .. } => panic!("attempted to lower an Opnd::VReg"), + Opnd::Value(_) => panic!("attempted to lower an Opnd::Value"), + Opnd::None => panic!( + "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." + ), + } + } +} + +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for A64Opnd { + fn from(opnd: &Opnd) -> Self { + A64Opnd::from(*opnd) + } +} + +/// Call emit_jmp_ptr and immediately invalidate the written range. +/// This is needed when next_page also moves other_cb that is not invalidated +/// by compile_with_regs. Doing it here allows you to avoid invalidating a lot +/// more than necessary when other_cb jumps from a position early in the page. +/// This invalidates a small range of cb twice, but we accept the small cost. +fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { + let start = cb.get_write_ptr(); + emit_jmp_ptr(cb, dst_ptr, true); + let end = cb.get_write_ptr(); + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; +} + +fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { + let src_addr = cb.get_write_ptr().as_offset(); + let dst_addr = dst_ptr.as_offset(); + + // If the offset is short enough, then we'll use the + // branch instruction. Otherwise, we'll move the + // destination into a register and use the branch + // register instruction. + let num_insns = if b_offset_fits_bits((dst_addr - src_addr) / 4) { + b(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + 1 + } else { + let num_insns = emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + num_insns + 1 + }; + + if padding { + // Make sure it's always a consistent number of + // instructions in case it gets patched and has to + // use the other branch. + assert!(num_insns * 4 <= cb.jmp_ptr_bytes()); + for _ in num_insns..(cb.jmp_ptr_bytes() / 4) { + nop(cb); + } + } +} + +/// Emit the required instructions to load the given value into the +/// given register. Our goal here is to use as few instructions as +/// possible to get this value into the register. +fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { + let mut current = value; + + if current <= 0xffff { + // If the value fits into a single movz + // instruction, then we'll use that. + movz(cb, rd, A64Opnd::new_uimm(current), 0); + 1 + } else if u16::try_from(!value).is_ok() { + // For small negative values, use a single movn + movn(cb, rd, A64Opnd::new_uimm(!value), 0); + 1 + } else if BitmaskImmediate::try_from(current).is_ok() { + // Otherwise, if the immediate can be encoded + // with the special bitmask immediate encoding, + // we'll use that. + mov(cb, rd, A64Opnd::new_uimm(current)); + 1 + } else { + // Finally we'll fall back to encoding the value + // using movz for the first 16 bits and movk for + // each subsequent set of 16 bits as long we + // they are necessary. + movz(cb, rd, A64Opnd::new_uimm(current & 0xffff), 0); + let mut num_insns = 1; + + // (We're sure this is necessary since we + // checked if it only fit into movz above). + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 16); + num_insns += 1; + + if current > 0xffff { + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 32); + num_insns += 1; + } + + if current > 0xffff { + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 48); + num_insns += 1; + } + num_insns + } +} + +/// List of registers that can be used for register allocation. +/// This has the same number of registers for x86_64 and arm64. +/// SCRATCH_OPND, SCRATCH1_OPND, and EMIT_OPND are excluded. +pub const ALLOC_REGS: &[Reg] = &[ + X0_REG, + X1_REG, + X2_REG, + X3_REG, + X4_REG, + X5_REG, + X11_REG, + X12_REG, +]; + +/// Special scratch registers for intermediate processing. They should be used only by +/// [`Assembler::arm64_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(X15_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(X17_REG); + +/// A scratch register available for use by resolve_ssa to break register copy cycles. +/// Must not overlap with ALLOC_REGS or other preserved registers. +pub const SCRATCH_REG: Reg = X15_REG; +const SCRATCH2_OPND: Opnd = Opnd::Reg(X14_REG); + +impl Assembler { + const MAX_FRAME_STACK_SLOTS: usize = 2048; + + /// Special register for intermediate processing in arm64_emit. It should be used only by arm64_emit. + const EMIT_REG: Reg = X16_REG; + const EMIT_OPND: A64Opnd = A64Opnd::Reg(Self::EMIT_REG); + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } + + /// Get the list of registers from which we will allocate on this platform + pub fn get_alloc_regs() -> Vec<Reg> { + ALLOC_REGS.to_vec() + } + + /// Get a list of all of the caller-saved registers + pub fn get_caller_save_regs() -> Vec<Reg> { + vec![X1_REG, X9_REG, X10_REG, X11_REG, X12_REG, X13_REG, X14_REG, X15_REG] + } + + /// How many bytes a call and a [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + + /// Split platform-specific instructions + /// The transformations done here are meant to make our lives simpler in later + /// stages of the compilation pipeline. + /// Here we may want to make sure that all instructions (except load and store) + /// have no memory operands. + fn arm64_split(mut self) -> Assembler + { + /// When you're storing a register into a memory location or loading a + /// memory location into a register, the displacement from the base + /// register of the memory location must fit into 9 bits. If it doesn't, + /// then we need to load that memory address into a register first. + fn split_memory_address(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(mem) => { + if mem_disp_fits_bits(mem.disp) { + opnd + } else { + let base = asm.lea(Opnd::Mem(Mem { num_bits: 64, ..mem })); + Opnd::mem(mem.num_bits, base, 0) + } + }, + _ => unreachable!("Can only split memory addresses.") + } + } + + /// Any memory operands you're sending into an Op::Load instruction need + /// to be split in case their displacement doesn't fit into 9 bits. + fn split_load_operand(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => { + let split_opnd = split_memory_address(asm, opnd); + let out_opnd = asm.load(split_opnd); + // Many Arm insns support only 32-bit or 64-bit operands. asm.load with fewer + // bits zero-extends the value, so it's safe to recognize it as a 32-bit value. + if out_opnd.rm_num_bits() < 32 { + out_opnd.with_num_bits(32) + } else { + out_opnd + } + }, + _ => asm.load(opnd) + } + } + + /// Operands that take the place of bitmask immediates must follow a + /// certain encoding. In this function we ensure that those operands + /// do follow that encoding, and if they don't then we load them first. + fn split_bitmask_immediate(asm: &mut Assembler, opnd: Opnd, dest_num_bits: u8) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => split_load_operand(asm, opnd), + Opnd::Imm(imm) => { + if imm == 0 { + Opnd::Reg(XZR_REG) + } else if (dest_num_bits == 64 && + BitmaskImmediate::try_from(imm as u64).is_ok()) || + (dest_num_bits == 32 && + u32::try_from(imm).is_ok() && + BitmaskImmediate::new_32b_reg(imm as u32).is_ok()) { + Opnd::UImm(imm as u64) + } else { + asm.load(opnd).with_num_bits(dest_num_bits) + } + }, + Opnd::UImm(uimm) => { + if (dest_num_bits == 64 && BitmaskImmediate::try_from(uimm).is_ok()) || + (dest_num_bits == 32 && + u32::try_from(uimm).is_ok() && + BitmaskImmediate::new_32b_reg(uimm as u32).is_ok()) { + opnd + } else { + asm.load(opnd).with_num_bits(dest_num_bits) + } + }, + Opnd::None | Opnd::Value(_) => unreachable!() + } + } + + /// Operands that take the place of a shifted immediate must fit within + /// a certain size. If they don't then we need to load them first. + fn split_shifted_immediate(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => split_load_operand(asm, opnd), + Opnd::Imm(imm) => if ShiftedImmediate::try_from(imm as u64).is_ok() { + opnd + } else { + asm.load(opnd) + } + Opnd::UImm(uimm) => { + if ShiftedImmediate::try_from(uimm).is_ok() { + opnd + } else { + asm.load(opnd) + } + }, + Opnd::None | Opnd::Value(_) => unreachable!() + } + } + + /// Returns the operands that should be used for a boolean logic + /// instruction. + fn split_boolean_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + match (opnd0, opnd1) { + (Opnd::Reg(_), Opnd::Reg(_)) => { + (opnd0, opnd1) + }, + (reg_opnd @ Opnd::Reg(_), other_opnd) | + (other_opnd, reg_opnd @ Opnd::Reg(_)) => { + let opnd1 = split_bitmask_immediate(asm, other_opnd, reg_opnd.rm_num_bits()); + (reg_opnd, opnd1) + }, + _ => { + let opnd0 = split_load_operand(asm, opnd0); + let opnd1 = split_bitmask_immediate(asm, opnd1, opnd0.rm_num_bits()); + (opnd0, opnd1) + } + } + } + + /// Returns the operands that should be used for a csel instruction. + fn split_csel_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + let opnd0 = match opnd0 { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd0, + _ => split_load_operand(asm, opnd0) + }; + + let opnd1 = match opnd1 { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd1, + _ => split_load_operand(asm, opnd1) + }; + + (opnd0, opnd1) + } + + fn split_less_than_32_cmp(asm: &mut Assembler, opnd0: Opnd) -> Opnd { + match opnd0 { + Opnd::Reg(_) | Opnd::VReg { .. } => { + match opnd0.rm_num_bits() { + 8 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xff)), + 16 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xffff)), + 32 | 64 => opnd0, + bits => unreachable!("Invalid number of bits. {}", bits) + } + } + _ => opnd0 + } + } + + let mut asm_local = Assembler::new_with_asm(&self); + let mut iterator = self.instruction_iterator(); + let asm = &mut asm_local; + + while let Some((_index, mut insn)) = iterator.next(asm) { + // Here we're going to map the operands of the instruction to load + // any Opnd::Value operands into registers if they are heap objects + // such that only the Op::Load instruction needs to handle that + // case. If the values aren't heap objects then we'll treat them as + // if they were just unsigned integer. + let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); + let is_jump = insn.is_jump(); + + insn.for_each_operand_mut(|opnd| { + if let Opnd::Value(value) = opnd { + if value.special_const_p() { + *opnd = Opnd::UImm(value.as_u64()); + } else if !is_load && !is_jump { + *opnd = asm.load(*opnd); + } + }; + }); + + // We are replacing instructions here so we know they are already + // being used. It is okay not to use their output here. + #[allow(unused_must_use)] + match &mut insn { + Insn::Add { left, right, .. } => { + match (*left, *right) { + // When one operand is a register, legalize the other operand + // into possibly an immdiate and swap the order if necessary. + // Only the rhs of ADD can be an immediate, but addition is commutative. + (reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. }), other_opnd) | + (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. })) => { + *left = reg_opnd; + *right = split_shifted_immediate(asm, other_opnd); + // Now `right` is either a register or an immediate, both can try to + // merge with a subsequent mov. + + asm.push_insn(insn); + } + _ => { + *left = split_load_operand(asm, *left); + *right = split_shifted_immediate(asm, *right); + + asm.push_insn(insn); + } + } + } + Insn::Sub { left, right, .. } => { + *left = split_load_operand(asm, *left); + *right = split_shifted_immediate(asm, *right); + asm.push_insn(insn); + } + Insn::And { left, right, .. } | + Insn::Or { left, right, .. } | + Insn::Xor { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, *left, *right); + *left = opnd0; + *right = opnd1; + + asm.push_insn(insn); + } + /* + // Lower to Joz and Jonz for generating CBZ/CBNZ for compare-with-0-and-branch. + ref insn @ Insn::Cmp { ref left, right: ref right @ (Opnd::UImm(0) | Opnd::Imm(0)) } | + ref insn @ Insn::Test { ref left, right: ref right @ (Opnd::InsnOut { .. } | Opnd::Reg(_)) } if { + let same_opnd_if_test = if let Insn::Test { .. } = insn { + left == right + } else { + true + }; + + same_opnd_if_test && if let Some( + Insn::Jz(target) | Insn::Je(target) | Insn::Jnz(target) | Insn::Jne(target) + ) = iterator.peek() { + matches!(target, Target::SideExit { .. }) + } else { + false + } + } => { + let reg = split_load_operand(asm, *left); + match iterator.peek() { + Some(Insn::Jz(target) | Insn::Je(target)) => asm.push_insn(Insn::Joz(reg, *target)), + Some(Insn::Jnz(target) | Insn::Jne(target)) => asm.push_insn(Insn::Jonz(reg, *target)), + _ => () + } + + iterator.map_insn_index(asm); + iterator.next_unmapped(); // Pop merged jump instruction + } + */ + Insn::Cmp { left, right } => { + let opnd0 = split_load_operand(asm, *left); + let opnd0 = split_less_than_32_cmp(asm, opnd0); + let split_right = split_shifted_immediate(asm, *right); + let opnd1 = match split_right { + Opnd::VReg { .. } if opnd0.num_bits() != split_right.num_bits() => { + split_right.with_num_bits(opnd0.num_bits().unwrap()) + }, + _ => split_right + }; + + asm.cmp(opnd0, opnd1); + }, + Insn::CRet(opnd) => { + match opnd { + // If the value is already in the return register, then + // we don't need to do anything. + Opnd::Reg(C_RET_REG) => {}, + + // If the value is a memory address, we need to first + // make sure the displacement isn't too large and then + // load it into the return register. + Opnd::Mem(_) => { + let split = split_memory_address(asm, *opnd); + asm.load_into(C_RET_OPND, split); + }, + + // Otherwise we just need to load the value into the + // return register. + _ => { + asm.load_into(C_RET_OPND, *opnd); + } + } + asm.cret(C_RET_OPND); + }, + Insn::CSelZ { truthy, falsy, .. } | + Insn::CSelNZ { truthy, falsy, .. } | + Insn::CSelE { truthy, falsy, .. } | + Insn::CSelNE { truthy, falsy, .. } | + Insn::CSelL { truthy, falsy, .. } | + Insn::CSelLE { truthy, falsy, .. } | + Insn::CSelG { truthy, falsy, .. } | + Insn::CSelGE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, *truthy, *falsy); + *truthy = opnd0; + *falsy = opnd1; + asm.push_insn(insn); + }, + Insn::JmpOpnd(opnd) => { + if let Opnd::Mem(_) = opnd { + let opnd0 = split_load_operand(asm, *opnd); + asm.jmp_opnd(opnd0); + } else { + asm.jmp_opnd(*opnd); + } + }, + Insn::Load { opnd, .. } | + Insn::LoadInto { opnd, .. } => { + *opnd = match opnd { + Opnd::Mem(_) => split_memory_address(asm, *opnd), + _ => *opnd + }; + asm.push_insn(insn); + }, + Insn::LoadSExt { opnd, out } => { + match opnd { + // We only want to sign extend if the operand is a + // register, instruction output, or memory address that + // is 32 bits. Otherwise we'll just load the value + // directly since there's no need to sign extend. + Opnd::Reg(Reg { num_bits: 32, .. }) | + Opnd::VReg { num_bits: 32, .. } | + Opnd::Mem(Mem { num_bits: 32, .. }) => { + asm.push_insn(insn); + }, + _ => { + asm.push_insn(Insn::Load { opnd: *opnd, out: *out }); + } + }; + }, + Insn::Mov { dest, src } => { + match (&dest, &src) { + // If we're attempting to load into a memory operand, then + // we'll switch over to the store instruction. + (Opnd::Mem(_), _) => { + let opnd0 = split_memory_address(asm, *dest); + let value = match *src { + // If the first operand is zero, then we can just use + // the zero register. + Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG), + // If the first operand is a memory operand, we're going + // to transform this into a store instruction, so we'll + // need to load this anyway. + Opnd::UImm(_) => asm.load(*src), + // The value that is being moved must be either a + // register or an immediate that can be encoded as a + // bitmask immediate. Otherwise, we'll need to split the + // move into multiple instructions. + _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits()) + }; + + asm.store(opnd0, value); + }, + // If we're loading a memory operand into a register, then + // we'll switch over to the load instruction. + (Opnd::Reg(_) | Opnd::VReg { .. }, Opnd::Mem(_)) => { + let value = split_memory_address(asm, *src); + asm.load_into(*dest, value); + }, + // Otherwise we'll use the normal mov instruction. + (Opnd::Reg(_), _) => { + let value = match *src { + // Unlike other instructions, we can avoid splitting this case, using movz. + Opnd::UImm(uimm) if uimm <= 0xffff => *src, + _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits()), + }; + asm.mov(*dest, value); + }, + _ => unreachable!("unexpected combination of operands in Insn::Mov: {dest:?}, {src:?}") + }; + }, + Insn::Not { opnd, .. } => { + // The value that is being negated must be in a register, so + // if we get anything else we need to load it first. + *opnd = match opnd { + Opnd::Mem(_) => split_load_operand(asm, *opnd), + _ => *opnd + }; + asm.push_insn(insn); + }, + Insn::LShift { opnd, .. } | + Insn::RShift { opnd, .. } | + Insn::URShift { opnd, .. } => { + // The operand must be in a register, so + // if we get anything else we need to load it first. + *opnd = split_load_operand(asm, *opnd); + asm.push_insn(insn); + }, + Insn::Mul { left, right, .. } => { + *left = split_load_operand(asm, *left); + *right = split_load_operand(asm, *right); + asm.push_insn(insn); + }, + Insn::Test { left, right } => { + // The value being tested must be in a register, so if it's + // not already one we'll load it first. + let opnd0 = split_load_operand(asm, *left); + + // The second value must be either a register or an + // unsigned immediate that can be encoded as a bitmask + // immediate. If it's not one of those, we'll need to load + // it first. + let opnd1 = split_bitmask_immediate(asm, *right, opnd0.rm_num_bits()); + asm.test(opnd0, opnd1); + }, + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Split instructions using scratch registers. To maximize the use of the register pool for + /// VRegs, most splits should happen in [`Self::arm64_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so this + /// splits them and uses scratch registers for it. + /// Linearizes all blocks into a single giant block. + fn arm64_scratch_split(self) -> Assembler { + /// If opnd is Opnd::Mem with a too large disp, make the disp smaller using lea. + fn split_large_disp(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(Mem { num_bits, disp, .. }) if !mem_disp_fits_bits(disp) => { + asm.lea_into(scratch_opnd, opnd); + Opnd::mem(num_bits, scratch_opnd, 0) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem with MemBase::Stack, lower it to Opnd::Mem with MemBase::Reg, and split a large disp. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + let opnd = split_only_stack_membase(asm, opnd, scratch_opnd, stack_state); + split_large_disp(asm, opnd, scratch_opnd) + } + + /// split_stack_membase but without split_large_disp. This should be used only by lea, + /// whose lowering already handles large displacements in arm64_emit. + fn split_only_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // Convert MemBase::Stack to MemBase::Reg(NATIVE_BASE_PTR) with the + // correct stack displacement. The stack slot value lives directly at + // [NATIVE_BASE_PTR + stack_disp], so we just adjust the base and + // combine displacements -- no indirection needed. Large + // displacements are handled by split_stack_membase(). + let Mem { base, disp: stack_disp, .. } = stack_state.stack_membase_to_mem(stack_membase); + Opnd::Mem(Mem { base, disp: stack_disp + opnd_disp, num_bits: opnd_num_bits }) + } + Opnd::Mem(Mem { base: MemBase::StackIndirect { stack_idx }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // The spilled value (a pointer) lives at a stack slot. Load it + // into a scratch register, then use the register as the base. + let stack_mem = stack_state.stack_membase_to_mem(MemBase::Stack { stack_idx, num_bits: 64 }); + let stack_opnd = split_large_disp(asm, Opnd::Mem(stack_mem), scratch_opnd); + asm.load_into(scratch_opnd, stack_opnd); + Opnd::Mem(Mem { + base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), + disp: opnd_disp, + num_bits: opnd_num_bits, + }) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem, lower it to scratch_opnd. You should use this when `opnd` is read by the instruction, not written. + fn split_memory_read(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(_) = opnd { + let opnd = split_stack_membase(asm, opnd, scratch_opnd, stack_state); + let scratch_opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + asm.load_into(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for (idx, insn) in linearized_insns.iter().enumerate() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } | + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Mul { left, right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + let reg_out = out.clone(); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + }; + + // If the next instruction is JoMul + if idx + 1 < linearized_insns.len() && matches!(linearized_insns[idx + 1], Insn::JoMul(_)) { + // Produce a register that is all zeros or all ones + // Based on the sign bit of the 64-bit mul result + asm.push_insn(Insn::RShift { out: SCRATCH0_OPND, opnd: reg_out, shift: Opnd::UImm(63) }); + } + } + Insn::LShift { opnd, out, .. } | + Insn::RShift { opnd, out, .. } => { + *opnd = split_memory_read(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + // For compile_exits, support splitting simple return values here + Insn::CRet(opnd) => { + match opnd { + Opnd::Reg(C_RET_REG) => {}, + _ => asm.load_into(C_RET_OPND, *opnd), + } + asm.cret(C_RET_OPND); + } + Insn::Lea { opnd, out } => { + *opnd = split_only_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = split_stack_membase(asm, *out, SCRATCH1_OPND, &stack_state); + + if let Opnd::Mem(_) = out { + // If NATIVE_STACK_PTR is used as a source for Store, it's handled as xzr, storeing zero. + // To save the content of NATIVE_STACK_PTR, we need to load it into another register first. + if *opnd == NATIVE_STACK_PTR { + asm.load_into(SCRATCH0_OPND, NATIVE_STACK_PTR); + *opnd = SCRATCH0_OPND; + } + asm.store(*out, *opnd); + } else { + // If in and out are the same, this is a redundant mov + if opnd != out { + asm.push_insn(insn); + } + } + } + &mut Insn::IncrCounter { mem, value } => { + // Convert Opnd::const_ptr into Opnd::Mem. + // It's split here to support IncrCounter in compile_exits. + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.lea_into(SCRATCH0_OPND, Opnd::mem(64, SCRATCH0_OPND, 0)); + + // Create a local loop to atomically increment a counter using SCRATCH1_OPND to check if it succeeded. + // Note that arm64_emit will peek at the next Cmp to set a status into SCRATCH1_OPND on IncrCounter. + let label = asm.new_label("incr_counter_loop"); + asm.write_label(label.clone()); + asm.incr_counter(SCRATCH0_OPND, value); + asm.cmp(SCRATCH1_OPND, 0.into()); + asm.push_insn(Insn::Jne(label)); + } + Insn::Store { dest, src } => { + *dest = split_stack_membase(asm, *dest, SCRATCH0_OPND, &stack_state); + *src = split_stack_membase(asm, *src, SCRATCH1_OPND, &stack_state); + asm.push_insn(insn); + } + Insn::Mov { dest, src } => { + *src = split_stack_membase(asm, *src, SCRATCH0_OPND, &stack_state); + *dest = split_stack_membase(asm, *dest, SCRATCH1_OPND, &stack_state); + match dest { + Opnd::Reg(_) => asm.load_into(*dest, *src), + Opnd::Mem(_) => asm.store(*dest, *src), + _ => asm.push_insn(insn), + } + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Emit platform-specific machine code + /// Returns a list of GC offsets. Can return failure to signal caller to retry. + fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<CodePtr>, CompileError> { + /// Determine how many instructions it will take to represent moving + /// this value into a register. Note that the return value of this + /// function must correspond to how many instructions are used to + /// represent this load in the emit_load_value function. + fn emit_load_size(value: u64) -> u8 { + if BitmaskImmediate::try_from(value).is_ok() { + return 1; + } + + if value < (1 << 16) { + 1 + } else if value < (1 << 32) { + 2 + } else if value < (1 << 48) { + 3 + } else { + 4 + } + } + + /// Emit a conditional jump instruction to a specific target. This is + /// called when lowering any of the conditional jump instructions. + fn emit_conditional_jump<const CONDITION: u8>(asm: &Assembler, cb: &mut CodeBlock, target: Target) { + fn generate_branch<const CONDITION: u8>(cb: &mut CodeBlock, src_addr: i64, dst_addr: i64) { + let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the jump offset fits into the conditional jump as + // an immediate value and it's properly aligned, then we + // can use the b.cond instruction directly. We're safe + // to use as i32 here since we already checked that it + // fits. + let bytes = (dst_addr - src_addr) as i32; + bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); + + // Here we're going to return 1 because we've only + // written out 1 instruction. + 1 + } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond + // If the jump offset fits into the unconditional jump as + // an immediate value, we can use inverse b.cond + b. + // + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instruction used for branching. + bcond(cb, Condition::inverse(CONDITION), 2.into()); + b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond + + // We've only written out 2 instructions. + 2 + } else { + // Otherwise, we need to load the address into a + // register and use the branch register instruction. + let load_insns: i32 = emit_load_size(dst_addr as u64).into(); + + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instructions used for branching. + bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + + // Here we'll return the number of instructions that it + // took to write out the destination address + 1 for the + // b.cond and 1 for the br. + load_insns + 2 + }; + + // We need to make sure we have at least 6 instructions for + // every kind of jump for invalidation purposes, so we're + // going to write out padding nop instructions here. + assert!(num_insns <= cb.conditional_jump_insns()); + (num_insns..cb.conditional_jump_insns()).for_each(|_| nop(cb)); + } + + let label = match target { + Target::CodePtr(dst_ptr) => { + let dst_addr = dst_ptr.as_offset(); + let src_addr = cb.get_write_ptr().as_offset(); + generate_branch::<CONDITION>(cb, src_addr, dst_addr); + return; + }, + Target::Label(l) => l, + Target::Block(ref edge) => asm.block_label(edge.target), + Target::SideExit { .. } => { + unreachable!("Target::SideExit should have been compiled by compile_exits") + }, + }; + // Try to use a single B.cond instruction + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if bcond_offset_fits_bits(offset) { + bcond(cb, CONDITION, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + } + + /// Emit a CBZ or CBNZ which branches when a register is zero or non-zero + fn emit_cmp_zero_jump(cb: &mut CodeBlock, reg: A64Opnd, branch_if_zero: bool, target: Target) { + if let Target::CodePtr(dst_ptr) = target { + let dst_addr = dst_ptr.as_offset(); + let src_addr = cb.get_write_ptr().as_offset(); + + if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the offset fits in one instruction, generate cbz or cbnz + let bytes = (dst_addr - src_addr) as i32; + if branch_if_zero { + cbz(cb, reg, InstructionOffset::from_bytes(bytes)); + } else { + cbnz(cb, reg, InstructionOffset::from_bytes(bytes)); + } + } else { + // Otherwise, we load the address into a register and + // use the branch register instruction. Note that because + // side exits should always be close, this form should be + // rare or impossible to see. + let dst_addr = dst_ptr.raw_addr(cb) as u64; + let load_insns: i32 = emit_load_size(dst_addr).into(); + + // Write out the inverse condition so that if + // it doesn't match it will skip over the + // instructions used for branching. + if branch_if_zero { + cbnz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); + } else { + cbz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); + } + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr); + br(cb, Assembler::EMIT_OPND); + } + } else { + unreachable!("We should only generate Joz/Jonz with side-exit targets"); + } + } + + /// Do the address calculation of `out_reg = base_reg + disp` + fn load_effective_address(cb: &mut CodeBlock, out: A64Opnd, base_reg_no: u8, disp: i32) { + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + assert_ne!(base_reg_no, out_reg_no, "large displacement need a scratch register"); + + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, out, disp as u64); + add_extended(cb, out, base_reg, out); + }; + } + + /// Load a VALUE to a register and remember it for GC marking and reference updating + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest: A64Opnd, value: VALUE) { + // We dont need to check if it's a special const + // here because we only allow these operands to hit + // this point if they're not a special const. + assert!(!value.special_const_p()); + + // This assumes only load instructions can contain + // references to GC'd Value operands. If the value + // being loaded is a heap object, we'll report that + // back out to the gc_offsets list. + ldr_literal(cb, dest, 2.into()); + b(cb, InstructionOffset::from_bytes(4 + (SIZEOF_VALUE as i32))); + cb.write_bytes(&value.as_u64().to_le_bytes()); + + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } + + /// Push a value to the stack by subtracting from the stack pointer then storing, + /// leaving an 8-byte gap for alignment. + fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) { + str_pre(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + } + + /// Pop a value from the stack by loading `[sp]` then adding to the stack pointer. + fn emit_pop(cb: &mut CodeBlock, opnd: A64Opnd) { + ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); + } + + // List of GC offsets + let mut gc_offsets: Vec<CodePtr> = Vec::new(); + + // Buffered list of PosMarker callbacks to fire if codegen is successful + let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // For each instruction + // NOTE: At this point, the assembler should have been linearized into a single giant block + // by either resolve_parallel_mov_pass() or arm64_scratch_split(). + let mut insn_idx: usize = 0; + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { + match insn { + Insn::Comment(text) => { + cb.add_comment(text); + }, + Insn::Label(target) => { + cb.write_label(target.unwrap_label()); + }, + // Report back the current position in the generated code + Insn::PosMarker(..) => { + pos_markers.push((insn_idx, cb.get_write_ptr())) + } + Insn::BakeString(text) => { + for byte in text.as_bytes() { + cb.write_byte(*byte); + } + + // Add a null-terminator byte for safety (in case we pass + // this to C code) + cb.write_byte(0); + + // Pad out the string to the next 4-byte boundary so that + // it's easy to jump past. + for _ in 0..(4 - ((text.len() + 1) % 4)) { + cb.write_byte(0); + } + }, + &Insn::FrameSetup { preserved, mut slot_count } => { + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + // Preserve X29 and set up frame record + stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); + mov(cb, X29, C_SP_REG); + + for regs in preserved.chunks(2) { + // For the body, store pairs and move SP + if let [reg0, reg1] = regs { + stp_pre(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, C_SP_REG, -16)); + } else if let [reg] = regs { + // For overhang, store but don't move SP. Combine movement with + // movement for slots below. + stur(cb, reg.into(), A64Opnd::new_mem(64, C_SP_REG, -8)); + slot_count += 1; + } else { + unreachable!("chunks(2)"); + } + } + // Align slot_count + if slot_count % 2 == 1 { + slot_count += 1 + } + if slot_count > 0 { + let slot_offset = (slot_count * SIZEOF_VALUE) as u64; + // Bail when asked to reserve too many slots in one instruction. + if ShiftedImmediate::try_from(slot_offset).is_err() { + return Err(CompileError::NativeStackTooLarge); + } + sub(cb, C_SP_REG, C_SP_REG, A64Opnd::new_uimm(slot_offset)); + } + } + Insn::FrameTeardown { preserved } => { + // Restore preserved registers below frame pointer. + let mut base_offset = 0; + for regs in preserved.chunks(2) { + if let [reg0, reg1] = regs { + base_offset -= 16; + ldp(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, X29, base_offset)); + } else if let [reg] = regs { + ldur(cb, reg.into(), A64Opnd::new_mem(64, X29, base_offset - 8)); + } else { + unreachable!("chunks(2)"); + } + } + + // SP = X29 (frame pointer) + mov(cb, C_SP_REG, X29); + ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); + } + Insn::Add { left, right, out } => { + // Usually, we issue ADDS, so you could branch on overflow, but ADDS with + // out=31 refers to out=XZR, which discards the sum. So, instead of ADDS + // (aliased to CMN in this case) we issue ADD instead which writes the sum + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + add(cb, out, left.into(), right.into()); + } else { + adds(cb, out, left.into(), right.into()); + } + }, + Insn::Sub { left, right, out } => { + // Usually, we issue SUBS, so you could branch on overflow, but SUBS with + // out=31 refers to out=XZR, which discards the result. So, instead of SUBS + // (aliased to CMP in this case) we issue SUB instead which writes the diff + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + sub(cb, out, left.into(), right.into()); + } else { + subs(cb, out, left.into(), right.into()); + } + }, + Insn::Mul { left, right, out } => { + // If the next instruction is JoMul with RShift created by arm64_scratch_split + match (insns.get(insn_idx + 1), insns.get(insn_idx + 2)) { + (Some(Insn::RShift { out: out_sign, opnd: out_opnd, shift: out_shift }), Some(Insn::JoMul(_))) => { + // Compute the high 64 bits + smulh(cb, Self::EMIT_OPND, left.into(), right.into()); + + // Compute the low 64 bits + // This may clobber one of the input registers, + // so we do it after smulh + mul(cb, out.into(), left.into(), right.into()); + + // Insert the shift instruction created by arm64_scratch_split + // to prepare the register that has the sign bit of the high 64 bits after mul. + asr(cb, out_sign.into(), out_opnd.into(), out_shift.into()); + insn_idx += 1; // skip the next Insn::RShift + + // If the high 64-bits are not all zeros or all ones, + // matching the sign bit, then we have an overflow + cmp(cb, Self::EMIT_OPND, out_sign.into()); + // Insn::JoMul will emit_conditional_jump::<{Condition::NE}> + } + _ => { + mul(cb, out.into(), left.into(), right.into()); + } + } + }, + Insn::And { left, right, out } => { + and(cb, out.into(), left.into(), right.into()); + }, + Insn::Or { left, right, out } => { + orr(cb, out.into(), left.into(), right.into()); + }, + Insn::Xor { left, right, out } => { + eor(cb, out.into(), left.into(), right.into()); + }, + Insn::Not { opnd, out } => { + mvn(cb, out.into(), opnd.into()); + }, + Insn::RShift { opnd, shift, out } => { + asr(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::URShift { opnd, shift, out } => { + lsr(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::LShift { opnd, shift, out } => { + lsl(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::Store { dest, src } => { + // Split src into EMIT0_OPND if necessary + let src_reg: A64Reg = match src { + Opnd::Reg(reg) => *reg, + // Use zero register when possible + Opnd::UImm(0) | Opnd::Imm(0) => XZR_REG, + // Immediates + &Opnd::Imm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm as u64); + Self::EMIT_REG + } + &Opnd::UImm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm); + Self::EMIT_REG + } + &Opnd::Value(value) => { + emit_load_gc_value(cb, &mut gc_offsets, Self::EMIT_OPND, value); + Self::EMIT_REG + } + src_mem @ &Opnd::Mem(Mem { num_bits: src_num_bits, base: MemBase::Reg(src_base_reg_no), disp: src_disp }) => { + // For mem-to-mem store, load the source into EMIT0_OPND + let src_mem = if mem_disp_fits_bits(src_disp) { + src_mem.into() + } else { + // Split the load address into EMIT0_OPND first if necessary + load_effective_address(cb, Self::EMIT_OPND, src_base_reg_no, src_disp); + A64Opnd::new_mem(dest.rm_num_bits(), Self::EMIT_OPND, 0) + }; + let dst = A64Opnd::Reg(Self::EMIT_REG.with_num_bits(src_num_bits)); + match src_num_bits { + 64 | 32 => ldur(cb, dst, src_mem), + 16 => ldurh(cb, dst, src_mem), + 8 => ldurb(cb, dst, src_mem), + num_bits => panic!("unexpected num_bits: {num_bits}") + }; + Self::EMIT_REG + } + src @ (Opnd::Mem(_) | Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during arm64_emit: {src:?}") + }; + let src = A64Opnd::Reg(src_reg); + + // This order may be surprising but it is correct. The way + // the Arm64 assembler works, the register that is going to + // be stored is first and the address is second. However in + // our IR we have the address first and the register second. + match dest.rm_num_bits() { + 64 | 32 => stur(cb, src, dest.into()), + 16 => sturh(cb, src, dest.into()), + 8 => sturb(cb, src, dest.into()), + num_bits => panic!("unexpected dest num_bits: {num_bits} (src: {src:?}, dest: {dest:?})"), + } + }, + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + match *opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => { + mov(cb, out.into(), opnd.into()); + }, + Opnd::UImm(uimm) => { + emit_load_value(cb, out.into(), uimm); + }, + Opnd::Imm(imm) => { + emit_load_value(cb, out.into(), imm as u64); + }, + Opnd::Mem(_) => { + match opnd.rm_num_bits() { + 64 | 32 => ldur(cb, out.into(), opnd.into()), + 16 => ldurh(cb, out.into(), opnd.into()), + 8 => ldurb(cb, out.into(), opnd.into()), + num_bits => panic!("unexpected num_bits: {num_bits}"), + }; + }, + Opnd::Value(value) => { + emit_load_gc_value(cb, &mut gc_offsets, out.into(), value); + }, + Opnd::None => { + unreachable!("Attempted to load from None operand"); + } + }; + }, + Insn::LoadSExt { opnd, out } => { + match *opnd { + Opnd::Reg(Reg { num_bits: 32, .. }) | + Opnd::VReg { num_bits: 32, .. } => { + sxtw(cb, out.into(), opnd.into()); + }, + Opnd::Mem(Mem { num_bits: 32, .. }) => { + ldursw(cb, out.into(), opnd.into()); + }, + _ => unreachable!() + }; + }, + Insn::Mov { dest, src } => { + // This supports the following two kinds of immediates: + // * The value fits into a single movz instruction + // * It can be encoded with the special bitmask immediate encoding + // arm64_split() should have split other immediates that require multiple instructions. + match src { + Opnd::UImm(uimm) if *uimm <= 0xffff => { + movz(cb, dest.into(), A64Opnd::new_uimm(*uimm), 0); + }, + _ => { + mov(cb, dest.into(), src.into()); + } + } + }, + Insn::Lea { opnd, out } => { + let &Opnd::Mem(Mem { num_bits: _, base: MemBase::Reg(base_reg_no), disp }) = opnd else { + panic!("Unexpected Insn::Lea operand in arm64_emit: {opnd:?}"); + }; + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + + let out = A64Opnd::from(out); + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use a scratch reg for `out += displacement` + let disp_reg = if out_reg_no == base_reg_no { + Self::EMIT_OPND + } else { + out + }; + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, disp_reg, disp as u64); + add_extended(cb, out, base_reg, disp_reg); + } + } + Insn::LeaJumpTarget { out, target, .. } => { + if let Target::Label(label_idx) = target { + // Set output to the raw address of the label + cb.label_ref(*label_idx, 4, |cb, end_addr, dst_addr| { + adr(cb, Self::EMIT_OPND, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + Ok(()) + }); + + mov(cb, out.into(), Self::EMIT_OPND); + } else { + // Set output to the jump target's raw address + let target_code = target.unwrap_code_ptr(); + let target_addr = target_code.raw_addr(cb).as_u64(); + emit_load_value(cb, out.into(), target_addr); + } + }, + Insn::CPush(opnd) => { + emit_push(cb, opnd.into()); + }, + Insn::CPushPair(opnd0, opnd1) => { + let first_push = if let Opnd::UImm(0) | Opnd::Imm(0) = opnd0 { X31 } else { opnd0.into() }; + let second_push = if let Opnd::UImm(0) | Opnd::Imm(0) = opnd1 { X31 } else { opnd1.into() }; + // Second operand ends up at the lower stack address + stp_pre(cb, second_push, first_push, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + }, + Insn::CPop { out } => { + emit_pop(cb, out.into()); + }, + Insn::CPopInto(opnd) => { + emit_pop(cb, opnd.into()); + }, + Insn::CPopPairInto(opnd0, opnd1) => { + let mut first_pop = opnd0.into(); + let second_pop = opnd1.into(); + // Avoid illegal load pair into the same register + // by sinking the first pop to the zero register. + if first_pop == second_pop { + first_pop = X31; + } + // First operand is popped from the lower stack address + ldp_post(cb, first_pop, second_pop, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); + }, + Insn::CCall { fptr, .. } => { + match fptr { + Opnd::UImm(fptr) => { + // The offset to the call target in bytes + let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; + let dst_addr = *fptr as i64; + + // Use BL if the offset is short enough to encode as an immediate. + // Otherwise, use BLR with a register. + if b_offset_fits_bits((dst_addr - src_addr) / 4) { + bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + } else { + emit_load_value(cb, Self::EMIT_OPND, dst_addr as u64); + blr(cb, Self::EMIT_OPND); + } + } + Opnd::Reg(_) => { + blr(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } + }, + Insn::CRet { .. } => { + ret(cb, A64Opnd::None); + }, + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); + }, + Insn::Test { left, right } => { + tst(cb, left.into(), right.into()); + }, + Insn::JmpOpnd(opnd) => { + br(cb, opnd.into()); + }, + Insn::Jmp(target) => { + match *target { + Target::CodePtr(dst_ptr) => { + emit_jmp_ptr(cb, dst_ptr, true); + }, + Target::Label(label_idx) => { + // Reserve space for a single B instruction + cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::Block(ref edge) => { + let label = self.block_label(edge.target); + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::SideExit { .. } => { + unreachable!("Target::SideExit should have been compiled by compile_exits") + }, + }; + }, + Insn::Je(target) | Insn::Jz(target) => { + emit_conditional_jump::<{Condition::EQ}>(self, cb, target.clone()); + }, + Insn::Jne(target) | Insn::Jnz(target) | Insn::JoMul(target) => { + emit_conditional_jump::<{Condition::NE}>(self, cb, target.clone()); + }, + Insn::Jl(target) => { + emit_conditional_jump::<{Condition::LT}>(self, cb, target.clone()); + }, + Insn::Jg(target) => { + emit_conditional_jump::<{Condition::GT}>(self, cb, target.clone()); + }, + Insn::Jge(target) => { + emit_conditional_jump::<{Condition::GE}>(self, cb, target.clone()); + }, + Insn::Jbe(target) => { + emit_conditional_jump::<{Condition::LS}>(self, cb, target.clone()); + }, + Insn::Jb(target) => { + emit_conditional_jump::<{Condition::CC}>(self, cb, target.clone()); + }, + Insn::Jo(target) => { + emit_conditional_jump::<{Condition::VS}>(self, cb, target.clone()); + }, + Insn::Joz(opnd, target) => { + emit_cmp_zero_jump(cb, opnd.into(), true, target.clone()); + }, + Insn::Jonz(opnd, target) => { + emit_cmp_zero_jump(cb, opnd.into(), false, target.clone()); + }, + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in arm64_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + while cb.get_write_pos().saturating_sub(last_patch_pos) < cb.jmp_ptr_bytes() && !cb.has_dropped_bytes() { + nop(cb); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + Insn::IncrCounter { mem, value } => { + // Get the status register allocated by arm64_scratch_split + let Some(Insn::Cmp { + left: status_reg @ Opnd::Reg(_), + right: Opnd::UImm(_) | Opnd::Imm(_), + }) = insns.get(insn_idx + 1) else { + panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", insns.get(insn_idx + 1)); + }; + + // Attempt to increment a counter + ldaxr(cb, Self::EMIT_OPND, mem.into()); + add(cb, Self::EMIT_OPND, Self::EMIT_OPND, value.into()); + + // The status register that gets used to track whether or + // not the store was successful must be 32 bytes. Since we + // store the EMIT registers as their 64-bit versions, we + // need to rewrap it here. + let status = A64Opnd::Reg(status_reg.unwrap_reg().with_num_bits(32)); + stlxr(cb, status, Self::EMIT_OPND, mem.into()); + }, + Insn::Breakpoint => { + brk(cb, A64Opnd::None); + }, + Insn::Abort => { + udf(cb, u16::MAX); + }, + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::EQ); + }, + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::NE); + }, + Insn::CSelL { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LT); + }, + Insn::CSelLE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LE); + }, + Insn::CSelG { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GT); + }, + Insn::CSelGE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); + } + }; + + insn_idx += 1; + } + + // Error if we couldn't write out everything + if cb.has_dropped_bytes() { + Err(CompileError::OutOfMemory) + } else { + // No bytes dropped, so the pos markers point to valid code + for (insn_idx, pos) in pos_markers { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { + callback(pos, cb); + } else { + panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); + } + } + + Ok(gc_offsets) + } + } + + /// Optimize and compile the stored instructions + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_reg = !self.accept_scratch_reg; + asm_dump!(self, init); + + let mut asm = trace_compile_phase("split", || self.arm64_split()); + + asm_dump!(asm, split); + + trace_compile_phase("regalloc", || { + trace_compile_phase("number_instructions", || asm.number_instructions(0)); + + let live_in = trace_compile_phase("analyze_liveness", || asm.analyze_liveness()); + let intervals = trace_compile_phase("build_intervals", || asm.build_intervals(live_in)); + + // Dump live intervals if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::live_intervals) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + } + } + + let preferred_registers = trace_compile_phase("preferred_registers", || asm.preferred_register_assignments(&intervals)); + let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); + + let total_stack_slots = asm.stack_base_idx + num_stack_slots; + if total_stack_slots > Self::MAX_FRAME_STACK_SLOTS { + return Err(CompileError::NativeStackTooLarge); + } + + // Dump vreg-to-physical-register mapping if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::alloc_regs) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + println!("VReg assignments:"); + for (i, alloc) in assignments.iter().enumerate() { + if let Some(alloc) = alloc { + let range = &intervals[i].range; + let alloc_str = match alloc { + Allocation::Reg(n) => format!("{}", regs[*n]), + Allocation::Fixed(reg) => format!("{}", reg), + Allocation::Stack(n) => format!("Stack[{}]", n), + }; + println!(" v{} => {} (range: {:?}..{:?})", i, alloc_str, range.start, range.end); + } + } + } + } + + // Update FrameSetup slot_count to account for: + // 1) stack slots reserved for block params (stack_base_idx), and + // 2) register allocator spills (num_stack_slots). + trace_compile_phase("count_stack_slots", || { + for block in asm.basic_blocks.iter_mut() { + for insn in block.insns.iter_mut() { + if let Insn::FrameSetup { slot_count, .. } = insn { + *slot_count = total_stack_slots; + } + } + } + }); + + trace_compile_phase("resolve_ssa", || { + asm.handle_caller_saved_regs(&intervals, &assignments, &C_ARG_REGREGS); + asm.resolve_ssa(&intervals, &assignments); + }); + + Ok(()) + })?; + asm_dump!(asm, alloc_regs); + + // We are moved out of SSA after resolve_ssa + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + // Exit code is compiled into a separate list of instructions that we append + // to the last reachable block before scratch_split, so it gets linearized and split. + trace_compile_phase("compile_exits", || { + let exit_insns = asm.compile_exits(); + + // Append exit instructions to the last reachable block so they are + // included in linearize_instructions and processed by scratch_split. + if let Some(&last_block) = asm.block_order().last() { + for insn in exit_insns { + asm.basic_blocks[last_block.0].insns.push(insn); + asm.basic_blocks[last_block.0].insn_ids.push(None); + } + } + }); + asm_dump!(asm, compile_exits); + + if use_scratch_reg { + asm = trace_compile_phase("scratch_split", || asm.arm64_scratch_split()); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = trace_compile_phase("resolve_parallel_mov", || asm.resolve_parallel_mov_pass()); + asm_dump!(asm, resolve_parallel_mov); + } + + trace_compile_phase("emit", || { + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.arm64_emit(cb).inspect_err(|_| cb.clear_labels())?; + assert!(!cb.has_dropped_bytes(), "emit should not drop bytes without error"); + + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + + // Invalidate icache for newly written out region so we don't run stale code. + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + + Ok((start_ptr, gc_offsets)) + }) + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "disasm")] + use crate::disasms_with; + use crate::{assert_disasm_snapshot, hexdumps}; + + use super::*; + use insta::assert_snapshot; + + static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; + + fn setup_asm() -> (Assembler, CodeBlock) { + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) + } + + fn setup_asm_with_scratch_reg() -> (Assembler, CodeBlock, Opnd) { + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy(), scratch_reg) + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); + asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.push_insn(Insn::Je(label)); + asm.frame_teardown(JIT_PRESERVED_REGS); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @" + test(): + bb0(): + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, x19, x21, x20 + v0 = Add x19, 0x40 + Store [x21 + 0x10], v0 + Joz Exit(Interrupt), v0 + Mov x0, w0 + Mov x1, [x21 - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[x20 + 0x10], VReg32(v1) + Je bb0 + FrameTeardown x19, x21, x20 + CRet v0 + FrameTeardown x19, x21, x20 + "); + } + + #[test] + fn test_mul_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.mul(Opnd::Reg(TEMP_REGS[1]), 3.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #3 + 0x4: mul x1, x9, x0 + "); + assert_snapshot!(cb.hexdump(), @"600080d2217d009b"); + } + + #[test] + fn test_conditional_branch_to_label() { + let (mut asm, mut cb) = setup_asm(); + let start = asm.new_label("start"); + let forward = asm.new_label("forward"); + + let value = asm.load(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0)); + asm.write_label(start.clone()); + asm.cmp(value, 0.into()); + asm.jg(forward.clone()); + asm.push_insn(Insn::Jl(start.clone())); + asm.write_label(forward); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [sp] + 0x4: cmp x0, #0 + 0x8: b.gt #0x10 + 0xc: b.lt #4 + "); + assert_snapshot!(cb.hexdump(), @"e00340f81f0000f14c000054cbffff54"); + } + + #[test] + fn sp_movements_are_single_instruction() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + let new_sp = asm.add(sp, 0x20.into()); + asm.mov(sp, new_sp); + let new_sp = asm.sub(sp, 0x20.into()); + asm.mov(sp, new_sp); + + asm.compile_with_num_regs(&mut cb, 2); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #0x20 + 0x4: sub sp, sp, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff830091ff8300d1"); + } + + #[test] + fn add_into() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + asm.add_into(sp, 8.into()); + asm.add_into(Opnd::Reg(X20_REG), 0x20.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #8 + 0x4: adds x20, x20, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff230091948200b1"); + } + + #[test] + fn sub_imm_reg() { + let (mut asm, mut cb) = setup_asm(); + + let difference = asm.sub(0x8.into(), Opnd::Reg(X5_REG)); + asm.load_into(Opnd::Reg(X1_REG), difference); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #8 + 0x4: subs x0, x0, x5 + 0x8: mov x1, x0 + "); + assert_snapshot!(cb.hexdump(), @"000180d2000005ebe10300aa"); + } + + #[test] + fn no_dead_mov_from_vreg() { + let (mut asm, mut cb) = setup_asm(); + + let ret_val = asm.load(Opnd::mem(64, C_RET_OPND, 0)); + asm.cret(ret_val); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [x0] + 0x4: ret + "); + assert_snapshot!(cb.hexdump(), @"000040f8c0035fd6"); + } + + #[test] + fn test_emit_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.add(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_regs(&mut cb, vec![X3_REG]).unwrap(); + + // Assert that only 2 instructions were written. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adds x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001ab400000f8"); + } + + #[test] + fn test_emit_bake_string() { + let (mut asm, mut cb) = setup_asm(); + + asm.bake_string("Hello, world!"); + asm.compile_with_num_regs(&mut cb, 0); + + // Testing that we pad the string to the nearest 4-byte boundary to make + // it easier to jump over. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldnp d8, d25, [x10, #-0x140] + 0x4: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x8: .byte 0x6f, 0x72, 0x6c, 0x64 + 0xc: udf #0x21 + "); + assert_snapshot!(cb.hexdump(), @"48656c6c6f2c20776f726c6421000000"); + } + + #[test] + fn test_emit_frame() { + let (mut asm, mut cb) = setup_asm(); + + asm.frame_setup(&[]); + asm.frame_teardown(&[]); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: mov sp, x29 + 0xc: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"fd7bbfa9fd030091bf030091fd7bc1a8"); + } + + #[test] + fn frame_setup_and_teardown() { + const THREE_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; + // Test 3 preserved regs (odd), odd slot_count + let cb1 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 3 preserved regs (odd), even slot_count + let cb2 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 4; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 4 preserved regs (even), odd slot_count + let cb3 = { + static FOUR_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(FOUR_REGS); + asm.frame_teardown(FOUR_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + assert_disasm_snapshot!(disasms_with!("\n", cb1, cb2, cb3), @" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x30 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stp x22, x21, [sp, #-0x10]! + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldp x22, x21, [x29, #-0x20] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(hexdumps!(cb1, cb2, cb3), @" + fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8 + "); + } + + #[test] + fn test_emit_je_fits_into_bcond() { + let (mut asm, mut cb) = setup_asm(); + + let target: CodePtr = cb.get_write_ptr().add_bytes(80); + + asm.push_insn(Insn::Je(Target::CodePtr(target))); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: b.eq #0x50 + 0x4: nop + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"800200541f2003d51f2003d51f2003d51f2003d51f2003d5"); + } + + #[test] + fn test_emit_je_does_not_fit_into_bcond() { + let (mut asm, mut cb) = setup_asm(); + + let offset = 1 << 21; + let target: CodePtr = cb.get_write_ptr().add_bytes(offset); + + asm.push_insn(Insn::Je(Target::CodePtr(target))); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: b.ne #8 + 0x4: b #0x200000 + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"41000054ffff07141f2003d51f2003d51f2003d51f2003d5"); + } + + #[test] + fn test_emit_lea() { + let (mut asm, mut cb) = setup_asm(); + + // Test values that exercise various types of immediates. + // - 9 bit displacement for Load/Store + // - 12 bit ADD/SUB shifted immediate + // - 16 bit MOV family shifted immediates + // - bit mask immediates + for displacement in [i32::MAX, 0x10008, 0x1800, 0x208, -0x208, -0x1800, -0x10008, i32::MIN] { + let mem = Opnd::mem(64, NATIVE_STACK_PTR, displacement); + asm.lea_into(Opnd::Reg(X0_REG), mem); + } + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x7fffffff + 0x4: add x0, sp, x0 + 0x8: mov x0, #8 + 0xc: movk x0, #1, lsl #16 + 0x10: add x0, sp, x0 + 0x14: mov x0, #0x1800 + 0x18: add x0, sp, x0 + 0x1c: add x0, sp, #0x208 + 0x20: sub x0, sp, #0x208 + 0x24: mov x0, #-0x1800 + 0x28: add x0, sp, x0 + 0x2c: mov x0, #0xfff8 + 0x30: movk x0, #0xfffe, lsl #16 + 0x34: movk x0, #0xffff, lsl #32 + 0x38: movk x0, #0xffff, lsl #48 + 0x3c: add x0, sp, x0 + 0x40: mov x0, #-0x80000000 + 0x44: add x0, sp, x0 + "); + assert_snapshot!(cb.hexdump(), @"e07b40b2e063208b000180d22000a0f2e063208b000083d2e063208be0230891e02308d1e0ff8292e063208b00ff9fd2c0ffbff2e0ffdff2e0fffff2e063208be08361b2e063208b"); + } + + #[test] + fn test_load_larg_disp_mem() { + let (mut asm, mut cb) = setup_asm(); + + let extended_ivars = asm.load(Opnd::mem(64, NATIVE_STACK_PTR, 0)); + let result = asm.load(Opnd::mem(VALUE_BITS, extended_ivars, 1000 * SIZEOF_VALUE_I32)); + asm.store(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0), result); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [sp] + 0x4: mov x16, #0x1f40 + 0x8: add x0, x0, x16, uxtx + 0xc: ldur x0, [x0] + 0x10: stur x0, [sp] + "); + assert_snapshot!(cb.hexdump(), @"e00340f810e883d20060308b000040f8e00300f8"); + } + + #[test] + fn test_store() { + let (mut asm, mut cb) = setup_asm(); + + // Large memory offsets in combinations of destination and source + let large_mem = Opnd::mem(64, NATIVE_STACK_PTR, -0x305); + let small_mem = Opnd::mem(64, C_RET_OPND, 0); + asm.store(small_mem, large_mem); + asm.store(large_mem, small_mem); + asm.store(large_mem, large_mem); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: sub x17, sp, #0x305 + 0x4: ldur x16, [x17] + 0x8: stur x16, [x0] + 0xc: sub x15, sp, #0x305 + 0x10: ldur x16, [x0] + 0x14: stur x16, [x15] + 0x18: sub x15, sp, #0x305 + 0x1c: sub x17, sp, #0x305 + 0x20: ldur x16, [x17] + 0x24: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"f1170cd1300240f8100000f8ef170cd1100040f8f00100f8ef170cd1f1170cd1300240f8f00100f8"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + // Side exit code are compiled without the split pass, so we directly call emit here to + // emulate that scenario. + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + let gc_offsets = asm.arm64_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldr x16, #8 + 0x4: b #0x10 + 0x8: udf #0x1000 + 0xc: udf #0 + 0x10: stur x16, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000058030000140010000000000000b00200f8"); + } + + #[test] + fn test_store_with_valid_scratch_reg() { + let (mut asm, mut cb, scratch_reg) = setup_asm_with_scratch_reg(); + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x3902 + 0x4: movk x16, #8, lsl #16 + 0x8: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"502087d21001a0f2f00100f8"); + } + + #[test] + #[should_panic] + fn test_store_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + } + + #[test] + #[should_panic] + fn test_load_into_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.load_into(scratch_reg, 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + } + + #[test] + fn test_emit_lea_label() { + let (mut asm, mut cb) = setup_asm(); + + let label = asm.new_label("label"); + let opnd = asm.lea_jump_target(label.clone()); + + asm.write_label(label); + asm.bake_string("Hello, world!"); + asm.store(Opnd::mem(64, SP, 0), opnd); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adr x16, #8 + 0x4: mov x0, x16 + 0x8: ldnp d8, d25, [x10, #-0x140] + 0xc: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x10: .byte 0x6f, 0x72, 0x6c, 0x64 + 0x14: udf #0x21 + 0x18: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000010e00310aa48656c6c6f2c20776f726c6421000000a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_fits_into_load() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 0)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that two instructions were written: LDUR and STUR. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [x21] + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a00240f8a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_fits_into_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 1 << 10)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that three instructions were written: ADD, LDUR, and STUR. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x21, #0x400 + 0x4: ldur x0, [x0] + 0x8: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a0021091000040f8a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_does_not_fit_into_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 1 << 12 | 1)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that three instructions were written: MOVZ, ADD, LDUR, and STUR. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x1001 + 0x4: add x0, x21, x0, uxtx + 0x8: ldur x0, [x0] + 0xc: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"200082d2a062208b000040f8a00200f8"); + } + + #[test] + fn test_emit_load_value_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::Value(Qnil)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that only two instructions were written since the value is an + // immediate. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #4 + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"800080d2a00200f8"); + } + + #[test] + fn test_emit_load_value_non_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::Value(VALUE(0xCAFECAFECAFE0000))); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that five instructions were written since the value is not an + // immediate and needs to be loaded into a register. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldr x0, #8 + 0x4: b #0x10 + 0x8: eon x0, x0, x30, ror #0 + 0xc: eon x30, x23, x30, ror #50 + 0x10: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"40000058030000140000fecafecafecaa00200f8"); + } + + #[test] + fn test_emit_test_32b_reg_not_bitmask_imm() { + let (mut asm, mut cb) = setup_asm(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); + asm.test(w0, Opnd::UImm(u32::MAX.into())); + // All ones is not encodable with a bitmask immediate, + // so this needs one register + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0xffffffff + 0x4: tst w0, w0 + "); + assert_snapshot!(cb.hexdump(), @"e07f40b21f00006a"); + } + + #[test] + fn test_emit_test_32b_reg_bitmask_imm() { + let (mut asm, mut cb) = setup_asm(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); + asm.test(w0, Opnd::UImm(0x80000001)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"1f040172"); + } + + #[test] + fn test_emit_or() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.or(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: orr x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001aa400000f8"); + } + + #[test] + fn test_emit_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.lshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lsl x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00e87bd3400000f8"); + } + + #[test] + fn test_emit_rshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.rshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: asr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc4593400000f8"); + } + + #[test] + fn test_emit_urshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.urshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lsr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc45d3400000f8"); + } + + #[test] + fn test_emit_test() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); + } + + #[test] + fn test_emit_test_with_encodable_unsigned_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::UImm(7)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); + } + + #[test] + fn test_emit_test_with_unencodable_unsigned_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a load and a test instruction were written. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); + } + + #[test] + fn test_emit_test_with_encodable_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(7)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); + } + + #[test] + fn test_emit_test_with_unencodable_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(5)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a load and a test instruction were written. + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); + } + + #[test] + fn test_emit_test_with_negative_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(-7)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a test instruction is written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #-7"); + assert_snapshot!(cb.hexdump(), @"1ff47df2"); + } + + #[test] + fn test_32_bit_register_with_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); + asm.cmp(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur w0, [x0, #6] + 0x4: mov x1, #0x1001 + 0x8: cmp w0, w1 + "); + assert_snapshot!(cb.hexdump(), @"006040b8210082d21f00016b"); + } + + #[test] + fn test_16_bit_register_store_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(16, Opnd::Reg(X0_REG), 0); + asm.store(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x1001 + 0x4: sturh w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"300082d210000078"); + } + + #[test] + fn test_32_bit_register_store_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); + asm.store(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x1001 + 0x4: stur w16, [x0, #6] + "); + assert_snapshot!(cb.hexdump(), @"300082d2106000b8"); + } + + #[test] + fn test_emit_xor() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.xor(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: eor x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001ca400000f8"); + } + + #[test] + #[cfg(feature = "disasm")] + fn test_simple_disasm() -> std::result::Result<(), capstone::Error> { + // Test drive Capstone with simple input + use capstone::prelude::*; + + let cs = Capstone::new() + .arm64() + .mode(arch::arm64::ArchMode::Arm) + .build()?; + + let insns = cs.disasm_all(&[0x60, 0x0f, 0x80, 0xF2], 0x1000)?; + + match insns.as_ref() { + [insn] => { + assert_eq!(Some("movk"), insn.mnemonic()); + Ok(()) + } + _ => Err(capstone::Error::CustomError( + "expected to disassemble to movk", + )), + } + } + + #[test] + fn test_replace_mov_with_ldur() { + let (mut asm, mut cb) = setup_asm(); + + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::mem(64, CFP, 8)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x1, [x19, #8]"); + assert_snapshot!(cb.hexdump(), @"618240f8"); + } + + #[test] + fn test_not_split_mov() { + let (mut asm, mut cb) = setup_asm(); + + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0xffff)); + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0x10000)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x1, #0xffff + 0x4: mov x1, #0x10000 + "); + assert_snapshot!(cb.hexdump(), @"e1ff9fd2e10370b2"); + } + + #[test] + fn test_merge_csel_mov() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.csel_l(Qtrue.into(), Qfalse.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x14 + 0x4: mov x1, #0 + 0x8: csel x1, x0, x1, lt + "); + assert_snapshot!(cb.hexdump(), @"800280d2010080d201b0819a"); + } + + #[test] + fn test_exceeding_label_branch_generate_bounds() { + // The immediate in a conditional branch is a 19 bit unsigned integer + // which has a max value of 2^18 - 1. + const IMMEDIATE_MAX_VALUE: usize = 2usize.pow(18) - 1; + + // `IMMEDIATE_MAX_VALUE` number of dummy instructions will be generated + // plus a compare, a jump instruction, and a label. + // Adding page_size to avoid OOM on the last page. + let page_size = unsafe { rb_jit_get_page_size() } as usize; + let memory_required = (IMMEDIATE_MAX_VALUE + 8) * 4 + page_size; + + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + let mut cb = CodeBlock::new_dummy_sized(memory_required); + + let far_label = asm.new_label("far"); + + asm.cmp(Opnd::Reg(X0_REG), Opnd::UImm(1)); + asm.push_insn(Insn::Je(far_label.clone())); + + (0..IMMEDIATE_MAX_VALUE).for_each(|_| { + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::Reg(TEMP_REGS[2])); + }); + + asm.write_label(far_label.clone()); + assert_eq!(Err(CompileError::LabelLinkingFailure), asm.compile(&mut cb)); + } + + #[test] + fn test_add_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.add(Opnd::Reg(TEMP_REGS[1]), 1.into()); + let out = asm.add(out, 1_usize.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adds x0, x9, #1 + 0x4: adds x1, x0, #1 + "); + assert_snapshot!(cb.hexdump(), @"200500b1010400b1"); + } + + #[test] + fn test_store_spilled_byte() { + let (mut asm, mut cb) = setup_asm(); + + asm.store(Opnd::mem(8, C_RET_OPND, 0), Opnd::mem(8, C_RET_OPND, 8)); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldurb w16, [x0, #8] + 0x4: sturb w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"1080403810000038"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_no_cycle() { + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov x0, x0 (optimized away) + C_ARG_OPNDS[1], // mov x1, x1 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0 + 0x4: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_single_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[2], // mov x2, x2 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x15 + 0xc: mov x16, #0 + 0x10: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_two_cycles() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle, and x2 and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[3], // mov x2, rcx + C_ARG_OPNDS[2], // mov rcx, x2 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x15 + 0xc: mov x15, x2 + 0x10: mov x2, x3 + 0x14: mov x3, x15 + 0x18: mov x16, #0 + 0x1c: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faaef0302aae20303aae3030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: stp x1, x0, [sp, #-0x10]! + 0x14: stp x3, x2, [sp, #-0x10]! + 0x18: mov x16, #0 + 0x1c: blr x16 + 0x20: ldp x3, x2, [sp], #0x10 + 0x24: ldp x1, x0, [sp], #0x10 + 0x28: adds x0, x0, x1 + 0x2c: adds x0, x2, x3 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2e103bfa9e30bbfa9100080d200023fd6e30bc1a8e103c1a8000001ab400003ab"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, #5 + 0x14: stp x1, x0, [sp, #-0x10]! + 0x18: stp x3, x2, [sp, #-0x10]! + 0x1c: stp xzr, x4, [sp, #-0x10]! + 0x20: mov x16, #0 + 0x24: blr x16 + 0x28: ldp xzr, x4, [sp], #0x10 + 0x2c: ldp x3, x2, [sp], #0x10 + 0x30: ldp x1, x0, [sp], #0x10 + 0x34: adds x0, x0, x1 + 0x38: adds x0, x2, x3 + 0x3c: adds x0, x2, x4 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2a40080d2e103bfa9e30bbfa9ff13bfa9100080d200023fd6ff13c1a8e30bc1a8e103c1a8000001ab400003ab400004ab"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_large_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0, x1, and x2 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[2], // mov x1, x2 + C_ARG_OPNDS[0], // mov x2, x0 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x2 + 0xc: mov x2, x15 + 0x10: mov x16, #0 + 0x14: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae10302aae2030faa100080d200023fd6"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: stp x1, x0, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e103bfa9"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: ldp x0, x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e007c1a8"); + } + + #[test] + fn test_split_spilled_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd_vreg = asm.load(1.into()); + let out_vreg = asm.lshift(opnd_vreg, Opnd::UImm(1)); + asm.mov(C_RET_OPND, out_vreg); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #1 + 0x4: stur x16, [x29, #-8] + 0x8: ldur x15, [x29, #-8] + 0xc: lsl x0, x15, #1 + "); + assert_snapshot!(cb.hexdump(), @"300080d2b0831ff8af835ff8e0f97fd3"); + } + + #[test] + fn test_split_load16_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(16, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldurh w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"0000089100004078"); + } + + #[test] + fn test_split_load32_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(32, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldur w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040b8"); + } + + #[test] + fn test_split_load64_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(64, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldur x0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040f8"); + } +} diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs new file mode 100644 index 0000000000..a417df300a --- /dev/null +++ b/zjit/src/backend/lir.rs @@ -0,0 +1,4471 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fmt; +use std::mem::take; +use std::rc::Rc; +use crate::bitset::BitSet; +use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; +use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::hir::{Invariant, SideExitReason}; +use crate::hir; +use crate::options::{TraceExits, PerfMap, get_option}; +use crate::cruby::VALUE; +use crate::payload::IseqVersionRef; +use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError}; +use crate::virtualmem::CodePtr; +use crate::asm::{CodeBlock, Label}; +use crate::state::{ZJITState, rb_zjit_record_exit_stack}; + +/// LIR Block ID. Unique ID for each block, and also defined in LIR so +/// we can differentiate it from HIR block ids. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct VRegId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl From<VRegId> for usize { + fn from(val: VRegId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "l{}", self.0) + } +} + +impl std::fmt::Display for VRegId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "v{}", self.0) + } +} + +/// Dummy HIR block ID used when creating test or invalid LIR blocks +const DUMMY_HIR_BLOCK_ID: usize = usize::MAX; +/// Dummy RPO index used when creating test or invalid LIR blocks +const DUMMY_RPO_INDEX: usize = usize::MAX; + +/// LIR Instruction ID. Unique ID for each instruction in the LIR. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct InsnId(pub usize); + +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 + } +} + +impl std::fmt::Display for InsnId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "i{}", self.0) + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<Opnd>, +} + +#[derive(Clone, Debug)] +pub struct BasicBlock { + // Unique id for this block + pub id: BlockId, + + // HIR block this LIR block was lowered from. Not injective: multiple LIR blocks may share + // the same hir_block_id because we split HIR blocks into multiple LIR blocks during lowering. + pub hir_block_id: hir::BlockId, + + pub is_entry: bool, + + // Instructions in this basic block + pub insns: Vec<Insn>, + + // Instruction IDs for each instruction (same length as insns) + pub insn_ids: Vec<Option<InsnId>>, + + // Input parameters for this block + pub parameters: Vec<Opnd>, + + // RPO position of the source HIR block + pub rpo_index: usize, + + // Range of instruction IDs in this block + pub from: InsnId, + pub to: InsnId, +} + +pub struct EdgePair(Option<BranchEdge>, Option<BranchEdge>); + +impl BasicBlock { + fn new(id: BlockId, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> Self { + Self { + id, + hir_block_id, + is_entry, + insns: vec![], + insn_ids: vec![], + parameters: vec![], + rpo_index, + from: InsnId(0), + to: InsnId(0), + } + } + + pub fn is_dummy(&self) -> bool { + self.hir_block_id == hir::BlockId(DUMMY_HIR_BLOCK_ID) + } + + pub fn add_parameter(&mut self, param: Opnd) { + self.parameters.push(param); + } + + pub fn push_insn(&mut self, insn: Insn) { + self.insns.push(insn); + self.insn_ids.push(None); + } + + pub fn edges(&self) -> EdgePair { + // Stub blocks (from new_block_without_id) have no real CFG structure. + if self.rpo_index == DUMMY_RPO_INDEX { + return EdgePair(None, None); + } + assert!(self.insns.last().unwrap().is_terminator()); + let extract_edge = |insn: &Insn| -> Option<BranchEdge> { + if let Some(Target::Block(edge)) = insn.target() { + Some(edge.clone()) + } else { + None + } + }; + + match self.insns.as_slice() { + [] => panic!("empty block"), + [.., second_last, last] => { + EdgePair(extract_edge(second_last), extract_edge(last)) + }, + [.., last] => { + EdgePair(extract_edge(last), None) + } + } + } + + /// Sort key for scheduling blocks in code layout order + pub fn sort_key(&self) -> (usize, usize) { + (self.rpo_index, self.id.0) + } + + pub fn successors(&self) -> Vec<BlockId> { + let EdgePair(edge1, edge2) = self.edges(); + let mut succs = Vec::new(); + if let Some(edge) = edge1 { + succs.push(edge.target); + } + if let Some(edge) = edge2 { + succs.push(edge.target); + } + succs + } + + /// Get the output VRegs for this block. + /// These are VRegs referenced by operands passed to successor blocks via block edges. + /// This function is used for live range calculations and should _not_ + /// be used for parallel moves between blocks + pub fn out_vregs(&self) -> Vec<VRegId> { + let EdgePair(edge1, edge2) = self.edges(); + let mut out_vregs = Vec::new(); + if let Some(edge) = edge1 { + for arg in &edge.args { + for idx in arg.vreg_ids() { + out_vregs.push(idx); + } + } + } + if let Some(edge) = edge2 { + for arg in &edge.args { + for idx in arg.vreg_ids() { + out_vregs.push(idx); + } + } + } + out_vregs + } +} + +pub use crate::backend::current::{ + mem_base_reg, + Reg, + EC, CFP, SP, + NATIVE_BASE_PTR, + C_ARG_OPNDS, C_RET_OPND, +}; + +pub static JIT_PRESERVED_REGS: &[Opnd] = &[CFP, SP, EC]; + +// Memory operand base +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Ord, PartialOrd)] +pub enum MemBase +{ + /// Register: Every Opnd::Mem should have MemBase::Reg as of emit. + Reg(u8), + /// Virtual register: Lowered to MemBase::Reg or MemBase::Stack during register assignment. + VReg(VRegId), + /// Stack slot: a direct stack access. `stack_membase_to_mem()` turns this + /// into `[NATIVE_BASE_PTR + disp]`, so scratch splitting can use it as a + /// normal memory operand without first loading a pointer from the stack. + Stack { stack_idx: usize, num_bits: u8 }, + /// A pointer stored in a stack slot, used as a memory base. + /// Unlike Stack, this first loads the pointer value from the stack slot + /// into a scratch register, then uses that register as the base for the + /// memory access with the Mem's displacement. + /// Created when a VReg used as MemBase is spilled to the stack. + StackIndirect { stack_idx: usize }, +} + +// Memory location +#[derive(Copy, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct Mem +{ + // Base register number or instruction index + pub base: MemBase, + + // Offset relative to the base pointer + pub disp: i32, + + // Size in bits + pub num_bits: u8, +} + +impl fmt::Display for Mem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.num_bits != 64 { + write!(f, "Mem{}", self.num_bits)?; + } + write!(f, "[")?; + match self.base { + MemBase::Reg(reg_no) => write!(f, "{}", mem_base_reg(reg_no))?, + MemBase::VReg(idx) => write!(f, "{idx}")?, + MemBase::Stack { stack_idx, num_bits } if num_bits == 64 => write!(f, "Stack[{stack_idx}]")?, + MemBase::StackIndirect { stack_idx } => write!(f, "*Stack[{stack_idx}]")?, + MemBase::Stack { stack_idx, num_bits } => write!(f, "Stack{num_bits}[{stack_idx}]")?, + } + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(f, " {sign} ")?; + if self.disp.abs() >= 10 { + write!(f, "0x")?; + } + write!(f, "{:x}", self.disp.abs())?; + } + write!(f, "]") + } +} + +impl fmt::Debug for Mem { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Mem{}[{:?}", self.num_bits, self.base)?; + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(fmt, " {sign} {}", self.disp.abs())?; + } + + write!(fmt, "]") + } +} + +/// Operand to an IR instruction +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum Opnd +{ + None, // For insns with no output + + // Immediate Ruby value, may be GC'd, movable + Value(VALUE), + + /// Virtual register. Lowered to Reg or Mem during register assignment. + VReg{ idx: VRegId, num_bits: u8 }, + + // Low-level operands, for lowering + Imm(i64), // Raw signed immediate + UImm(u64), // Raw unsigned immediate + Mem(Mem), // Memory location + Reg(Reg), // Machine register +} + +impl PartialOrd for Opnd { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for Opnd { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn case_order(opnd: &Opnd) -> u8 { + match opnd { + Opnd::None => 0, + Opnd::Value(_) => 1, + Opnd::VReg { .. } => 2, + Opnd::Imm(_) => 3, + Opnd::UImm(_) => 4, + Opnd::Mem(_) => 5, + Opnd::Reg(_) => 6, + } + } + match (self, other) { + (Opnd::None, Opnd::None) => std::cmp::Ordering::Equal, + (Opnd::Value(l), Opnd::Value(r)) => l.0.cmp(&r.0), + (Opnd::VReg { idx: lidx, num_bits: lnum_bits }, Opnd::VReg { idx: ridx, num_bits: rnum_bits }) => (lidx, lnum_bits).cmp(&(ridx, rnum_bits)), + (Opnd::Imm(l), Opnd::Imm(r)) => l.cmp(&r), + (Opnd::UImm(l), Opnd::UImm(r)) => l.cmp(&r), + (Opnd::Mem(l), Opnd::Mem(r)) => l.cmp(&r), + (Opnd::Reg(l), Opnd::Reg(r)) => l.cmp(&r), + (l, r) => { + case_order(l).cmp(&case_order(r)) + } + } + } +} + +impl fmt::Display for Opnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Opnd::*; + match self { + None => write!(f, "None"), + Value(VALUE(value)) if *value < 10 => write!(f, "Value({value:x})"), + Value(VALUE(value)) => write!(f, "Value(0x{value:x})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(f, "{idx}"), + VReg { idx, num_bits } => write!(f, "VReg{num_bits}({idx})"), + Imm(value) if value.abs() < 10 => write!(f, "Imm({value:x})"), + Imm(value) => write!(f, "Imm(0x{value:x})"), + UImm(value) if *value < 10 => write!(f, "{value:x}"), + UImm(value) => write!(f, "0x{value:x}"), + Mem(mem) => write!(f, "{mem}"), + Reg(reg) => write!(f, "{reg}"), + } + } +} + +impl fmt::Debug for Opnd { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + use Opnd::*; + match self { + Self::None => write!(fmt, "None"), + Value(val) => write!(fmt, "Value({val:?})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(fmt, "VReg({})", idx.0), + VReg { idx, num_bits } => write!(fmt, "VReg{num_bits}({})", idx.0), + Imm(signed) => write!(fmt, "{signed:x}_i64"), + UImm(unsigned) => write!(fmt, "{unsigned:x}_u64"), + // Say Mem and Reg only once + Mem(mem) => write!(fmt, "{mem:?}"), + Reg(reg) => write!(fmt, "{reg:?}"), + } + } +} + +impl Opnd +{ + /// Returns true if this operand is a virtual register + pub fn is_vreg(&self) -> bool { + matches!(self, Opnd::VReg { .. }) + } + + /// Convenience constructor for memory operands + pub fn mem(num_bits: u8, base: Opnd, disp: i32) -> Self { + match base { + Opnd::Reg(base_reg) => { + assert!(base_reg.num_bits == 64); + Opnd::Mem(Mem { + base: MemBase::Reg(base_reg.reg_no), + disp, + num_bits, + }) + }, + + Opnd::VReg{idx, num_bits: out_num_bits } => { + assert!(num_bits <= out_num_bits); + Opnd::Mem(Mem { + base: MemBase::VReg(idx), + disp, + num_bits, + }) + }, + + _ => unreachable!("memory operand with non-register base: {base:?}") + } + } + + /// Constructor for constant pointer operand + pub fn const_ptr<T>(ptr: *const T) -> Self { + Opnd::UImm(ptr as u64) + } + + /// Unwrap a register operand + pub fn unwrap_reg(&self) -> Reg { + match self { + Opnd::Reg(reg) => *reg, + _ => unreachable!("trying to unwrap {:?} into reg", self) + } + } + + /// Unwrap the index of a VReg + pub fn vreg_idx(&self) -> VRegId { + match self { + Opnd::VReg { idx, .. } => *idx, + _ => unreachable!("trying to unwrap {self:?} into VReg"), + } + } + + /// Extract VReg indices from this operand, including memory base VRegs. + /// Returns an iterator over all VRegIds referenced by this operand. + pub fn vreg_ids(&self) -> impl Iterator<Item = VRegId> { + let mut ids = [None, None]; + match self { + Opnd::VReg { idx, .. } => { ids[0] = Some(*idx); } + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { ids[0] = Some(*idx); } + _ => {} + } + ids.into_iter().flatten() + } + + /// Get the size in bits for this operand if there is one. + pub fn num_bits(&self) -> Option<u8> { + match *self { + Opnd::Reg(Reg { num_bits, .. }) => Some(num_bits), + Opnd::Mem(Mem { num_bits, .. }) => Some(num_bits), + Opnd::VReg { num_bits, .. } => Some(num_bits), + _ => None + } + } + + /// Return Opnd with a given num_bits if self has num_bits. Panic otherwise. + #[track_caller] + pub fn with_num_bits(&self, num_bits: u8) -> Opnd { + assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); + match *self { + Opnd::Reg(reg) => Opnd::Reg(reg.with_num_bits(num_bits)), + Opnd::Mem(Mem { base, disp, .. }) => Opnd::Mem(Mem { base, disp, num_bits }), + Opnd::VReg { idx, .. } => Opnd::VReg { idx, num_bits }, + _ => unreachable!("with_num_bits should not be used for: {self:?}"), + } + } + + /// Get the size in bits for register/memory operands. + pub fn rm_num_bits(&self) -> u8 { + self.num_bits().unwrap() + } + + /// Maps the indices from a previous list of instructions to a new list of + /// instructions. + pub fn map_index(self, indices: &[usize]) -> Opnd { + match self { + Opnd::VReg { idx, num_bits } => { + Opnd::VReg { idx: VRegId(indices[idx.0]), num_bits } + } + Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { + Opnd::Mem(Mem { base: MemBase::VReg(VRegId(indices[idx.0])), disp, num_bits }) + }, + _ => self + } + } + + /// When there aren't any operands to check against, this is the number of + /// bits that should be used for any given output variable. + const DEFAULT_NUM_BITS: u8 = 64; + + /// Determine the size in bits from the iterator of operands. If any of them + /// are different sizes this will panic. + pub fn match_num_bits_iter<'a>(opnds: impl Iterator<Item = &'a Opnd>) -> u8 { + let mut value: Option<u8> = None; + + for opnd in opnds { + if let Some(num_bits) = opnd.num_bits() { + match value { + None => { + value = Some(num_bits); + }, + Some(value) => { + assert_eq!(value, num_bits, "operands of incompatible sizes"); + } + }; + } + } + + value.unwrap_or(Self::DEFAULT_NUM_BITS) + } + + /// Determine the size in bits of the slice of the given operands. If any of + /// them are different sizes this will panic. + pub fn match_num_bits(opnds: &[Opnd]) -> u8 { + Self::match_num_bits_iter(opnds.iter()) + } +} + +impl From<usize> for Opnd { + fn from(value: usize) -> Self { + Opnd::UImm(value.try_into().unwrap()) + } +} + +impl From<u64> for Opnd { + fn from(value: u64) -> Self { + Opnd::UImm(value) + } +} + +impl From<i64> for Opnd { + fn from(value: i64) -> Self { + Opnd::Imm(value) + } +} + +impl From<i32> for Opnd { + fn from(value: i32) -> Self { + Opnd::Imm(value.into()) + } +} + +impl From<u32> for Opnd { + fn from(value: u32) -> Self { + Opnd::UImm(value as u64) + } +} + +impl From<VALUE> for Opnd { + fn from(value: VALUE) -> Self { + Opnd::Value(value) + } +} + +/// Context for a side exit. If `SideExit` matches, it reuses the same code. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExit { + pub pc: Opnd, + pub stack: Vec<Opnd>, + pub locals: Vec<Opnd>, + pub iseq: IseqPtr, + /// If set, the side exit will call the recompile function with these arguments + /// to profile the send and invalidate the ISEQ for recompilation. + pub recompile: Option<SideExitRecompile>, +} + +/// Arguments for the recompile callback on side exit. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExitRecompile { + pub iseq: Opnd, + pub insn_idx: u32, + pub strategy: hir::Recompile, +} + +/// Branch target (something that we can jump to) +/// for branch instructions +#[derive(Clone)] +pub enum Target +{ + /// Pointer to a piece of ZJIT-generated code + CodePtr(CodePtr), + /// A label within the generated code + Label(Label), + /// An LIR branch edge + Block(BranchEdge), + /// Side exit to the interpreter + SideExit { + /// Context used for compiling the side exit + exit: SideExit, + /// We use this to increment exit counters + reason: SideExitReason, + }, +} + +impl fmt::Debug for Target { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Target::CodePtr(ptr) => write!(f, "CodePtr({:?})", ptr), + Target::Label(label) => write!(f, "Label({:?})", label), + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, "Block({:?})", edge.target) + } else { + write!(f, "Block({:?}(", edge.target)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}", arg)?; + } + write!(f, "))") + } + } + Target::SideExit { exit, reason } => { + write!(f, "SideExit {{ exit: {:?}, reason: {:?} }}", exit, reason) + } + } + } +} + +impl Target +{ + pub fn unwrap_label(&self) -> Label { + match self { + Target::Label(label) => *label, + _ => unreachable!("trying to unwrap {:?} into label", self) + } + } + + pub fn unwrap_code_ptr(&self) -> CodePtr { + match self { + Target::CodePtr(ptr) => *ptr, + _ => unreachable!("trying to unwrap {:?} into code ptr", self) + } + } +} + +impl From<CodePtr> for Target { + fn from(code_ptr: CodePtr) -> Self { + Target::CodePtr(code_ptr) + } +} + +type PosMarkerFn = Rc<dyn Fn(CodePtr, &CodeBlock)>; + +/// ZJIT Low-level IR instruction +#[derive(Clone)] +pub enum Insn { + /// Add two operands together, and return the result as a new operand. + Add { left: Opnd, right: Opnd, out: Opnd }, + + /// This is the same as the OP_ADD instruction, except that it performs the + /// binary AND operation. + And { left: Opnd, right: Opnd, out: Opnd }, + + /// Bake a string directly into the instruction stream. + BakeString(String), + + // Trigger a debugger breakpoint + #[allow(dead_code)] + Breakpoint, + + // Abort the process + Abort, + + /// Add a comment into the IR at the point that this instruction is added. + /// It won't have any impact on that actual compiled code. + Comment(String), + + /// Compare two operands + Cmp { left: Opnd, right: Opnd }, + + /// Pop a register from the C stack + CPop { out: Opnd }, + + /// Pop a register from the C stack and store it into another register + CPopInto(Opnd), + + /// Pop a pair of registers from the C stack and store it into a pair of registers. + /// The registers are popped from left to right. + CPopPairInto(Opnd, Opnd), + + /// Push a register onto the C stack + CPush(Opnd), + + /// Push a pair of registers onto the C stack. + /// The registers are pushed from left to right. + CPushPair(Opnd, Opnd), + + // C function call with N arguments (variadic) + CCall { + opnds: Vec<Opnd>, + /// The function pointer to be called. This should be Opnd::const_ptr + /// (Opnd::UImm) in most cases. gen_entry_trampoline() uses Opnd::Reg. + fptr: Opnd, + /// Optional PosMarker to remember the start address of the C call. + /// It's embedded here to insert the PosMarker after push instructions + /// that are split from this CCall during register assignment. + start_marker: Option<PosMarkerFn>, + /// Optional PosMarker to remember the end address of the C call. + /// It's embedded here to insert the PosMarker before pop instructions + /// that are split from this CCall during register assignment. + end_marker: Option<PosMarkerFn>, + out: Opnd, + }, + + // C function return + CRet(Opnd), + + /// Conditionally select if equal + CSelE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater + CSelG { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater or equal + CSelGE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if less + CSelL { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if less or equal + CSelLE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if not equal + CSelNE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if not zero + CSelNZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if zero + CSelZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Set up the frame stack as necessary per the architecture. + FrameSetup { preserved: &'static [Opnd], slot_count: usize }, + + /// Tear down the frame stack as necessary per the architecture. + FrameTeardown { preserved: &'static [Opnd], }, + + // Atomically increment a counter + // Input: memory operand, increment value + // Produces no output + IncrCounter { mem: Opnd, value: Opnd }, + + /// Jump if below or equal (unsigned) + Jbe(Target), + + /// Jump if below (unsigned) + Jb(Target), + + /// Jump if equal + Je(Target), + + /// Jump if lower + Jl(Target), + + /// Jump if greater + Jg(Target), + + /// Jump if greater or equal + Jge(Target), + + // Unconditional jump to a branch target + Jmp(Target), + + // Unconditional jump which takes a reg/mem address operand + JmpOpnd(Opnd), + + /// Jump if not equal + Jne(Target), + + /// Jump if not zero + Jnz(Target), + + /// Jump if overflow + Jo(Target), + + /// Jump if overflow in multiplication + JoMul(Target), + + /// Jump if zero + Jz(Target), + + /// Jump if operand is zero (only used during lowering at the moment) + Joz(Opnd, Target), + + /// Jump if operand is non-zero (only used during lowering at the moment) + Jonz(Opnd, Target), + + // Add a label into the IR at the point that this instruction is added. + Label(Target), + + /// Get the code address of a jump target + LeaJumpTarget { target: Target, out: Opnd }, + + // Load effective address + Lea { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register. + Load { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a specified register. + LoadInto { dest: Opnd, opnd: Opnd }, + + // A low-level instruction that loads a value into a register and + // sign-extends it to a 64-bit value. + LoadSExt { opnd: Opnd, out: Opnd }, + + /// Shift a value left by a certain amount. + LShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // A low-level mov instruction. It accepts two operands. + Mov { dest: Opnd, src: Opnd }, + + // Perform the NOT operation on an individual operand, and return the result + // as a new operand. This operand can then be used as the operand on another + // instruction. + Not { opnd: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary OR operation. + Or { left: Opnd, right: Opnd, out: Opnd }, + + /// Patch point that will be rewritten to a jump to a side exit on invalidation. + PatchPoint { target: Target, invariant: Invariant, version: IseqVersionRef }, + + /// Make sure the last PatchPoint has enough space to insert a jump. + /// We insert this instruction at the end of each block so that the jump + /// will not overwrite the next block or a side exit. + PadPatchPoint, + + // Mark a position in the generated code + PosMarker(PosMarkerFn), + + /// Shift a value right by a certain amount (signed). + RShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // Low-level instruction to store a value to memory. + Store { dest: Opnd, src: Opnd }, + + // This is the same as the add instruction, except for subtraction. + Sub { left: Opnd, right: Opnd, out: Opnd }, + + // Integer multiplication + Mul { left: Opnd, right: Opnd, out: Opnd }, + + // Bitwise AND test instruction + Test { left: Opnd, right: Opnd }, + + /// Shift a value right by a certain amount (unsigned). + URShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary XOR operation. + Xor { left: Opnd, right: Opnd, out: Opnd } +} + +macro_rules! target_for_each_operand_impl { + ($self:expr, $visit_many:ident) => { + match $self { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + visit_many!(stack); + visit_many!(locals); + } + Target::Block(edge) => { + visit_many!(edge.args); + } + Target::CodePtr(_) | Target::Label(_) => {} + } + } +} + +/// Macro that enumerates all operands of an Insn, dispatching to caller-provided `$visit_one` +/// macro for a single `Opnd` field and `$visit_many` macro for a slice/`Vec` of `Opnd`s. Used by +/// both `for_each_operand` and `for_each_operand_mut`. +macro_rules! for_each_operand_impl { + ($self:expr, $visit_one:ident, $visit_many:ident $(, $const:expr)?) => { + match $self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + target_for_each_operand_impl!(target, $visit_many); + } + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + visit_one!(opnd); + target_for_each_operand_impl!(target, $visit_many); + } + + Insn::BakeString(_) | + Insn::Breakpoint | Insn::Abort | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::PadPatchPoint | + Insn::PosMarker(_) => {}, + + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { + visit_one!(opnd); + } + Insn::Add { left: opnd0, right: opnd1, .. } | + Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | + Insn::Cmp { left: opnd0, right: opnd1 } | + Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | + Insn::LoadInto { dest: opnd0, opnd: opnd1 } | + Insn::LShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Mov { dest: opnd0, src: opnd1 } | + Insn::Or { left: opnd0, right: opnd1, .. } | + Insn::RShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Store { dest: opnd0, src: opnd1 } | + Insn::Sub { left: opnd0, right: opnd1, .. } | + Insn::Mul { left: opnd0, right: opnd1, .. } | + Insn::Test { left: opnd0, right: opnd1 } | + Insn::URShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Xor { left: opnd0, right: opnd1, .. } => { + visit_one!(opnd0); + visit_one!(opnd1); + } + Insn::CCall { opnds, .. } => { + visit_many!(opnds); + } + // only iterate over preserved in the const iterator + #[allow(unused_variables)] + Insn::FrameSetup { preserved, .. } | + Insn::FrameTeardown { preserved } => { + $( + visit_many!(preserved); + $const; + )? + } + } + } +} + +impl Insn { + pub fn opnd_count(&self) -> usize { + let mut count = 0; + self.for_each_operand(|_| count += 1); + count + } + + /// Call `f` on each operand (Opnd) of this instruction. + pub fn for_each_operand(&self, mut f: impl FnMut(Opnd)) { + macro_rules! visit_one { ($id:expr) => { f(*$id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id) } }; } + // Extra () is a throw-away parameter to avoid iterating over FrameSetup/FrameTeardown + // preserved in the mutable iterator. + for_each_operand_impl!(self, visit_one, visit_many, ()); + } + + /// Call `f` on a mutable reference to each operand (Opnd) of this instruction. + pub fn for_each_operand_mut(&mut self, mut f: impl FnMut(&mut Opnd)) { + macro_rules! visit_one { ($id:expr) => { f($id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter_mut() { f(id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); + } + + /// Call `f` on each operand, short-circuiting on the first error. + pub fn try_for_each_operand<E>(&self, mut f: impl FnMut(Opnd) -> Result<(), E>) -> Result<(), E> { + macro_rules! visit_one { ($id:expr) => { f(*$id)? }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id)? } }; } + for_each_operand_impl!(self, visit_one, visit_many, ()); + Ok(()) + } + + /// Get a mutable reference to a Target if it exists. + pub(super) fn target_mut(&mut self) -> Option<&mut Target> { + match self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Joz(_, target) | + Insn::Jonz(_, target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + Some(target) + } + _ => None, + } + } + + /// Returns a string that describes which operation this instruction is + /// performing. This is used for debugging. + fn op(&self) -> &'static str { + match self { + Insn::Add { .. } => "Add", + Insn::And { .. } => "And", + Insn::BakeString(_) => "BakeString", + Insn::Breakpoint => "Breakpoint", + Insn::Abort => "Abort", + Insn::Comment(_) => "Comment", + Insn::Cmp { .. } => "Cmp", + Insn::CPop { .. } => "CPop", + Insn::CPopInto(_) => "CPopInto", + Insn::CPopPairInto(_, _) => "CPopPairInto", + Insn::CPush(_) => "CPush", + Insn::CPushPair(_, _) => "CPushPair", + Insn::CCall { .. } => "CCall", + Insn::CRet(_) => "CRet", + Insn::CSelE { .. } => "CSelE", + Insn::CSelG { .. } => "CSelG", + Insn::CSelGE { .. } => "CSelGE", + Insn::CSelL { .. } => "CSelL", + Insn::CSelLE { .. } => "CSelLE", + Insn::CSelNE { .. } => "CSelNE", + Insn::CSelNZ { .. } => "CSelNZ", + Insn::CSelZ { .. } => "CSelZ", + Insn::FrameSetup { .. } => "FrameSetup", + Insn::FrameTeardown { .. } => "FrameTeardown", + Insn::IncrCounter { .. } => "IncrCounter", + Insn::Jbe(_) => "Jbe", + Insn::Jb(_) => "Jb", + Insn::Je(_) => "Je", + Insn::Jl(_) => "Jl", + Insn::Jg(_) => "Jg", + Insn::Jge(_) => "Jge", + Insn::Jmp(_) => "Jmp", + Insn::JmpOpnd(_) => "JmpOpnd", + Insn::Jne(_) => "Jne", + Insn::Jnz(_) => "Jnz", + Insn::Jo(_) => "Jo", + Insn::JoMul(_) => "JoMul", + Insn::Jz(_) => "Jz", + Insn::Joz(..) => "Joz", + Insn::Jonz(..) => "Jonz", + Insn::Label(_) => "Label", + Insn::LeaJumpTarget { .. } => "LeaJumpTarget", + Insn::Lea { .. } => "Lea", + Insn::Load { .. } => "Load", + Insn::LoadInto { .. } => "LoadInto", + Insn::LoadSExt { .. } => "LoadSExt", + Insn::LShift { .. } => "LShift", + Insn::Mov { .. } => "Mov", + Insn::Not { .. } => "Not", + Insn::Or { .. } => "Or", + Insn::PatchPoint { .. } => "PatchPoint", + Insn::PadPatchPoint => "PadPatchPoint", + Insn::PosMarker(_) => "PosMarker", + Insn::RShift { .. } => "RShift", + Insn::Store { .. } => "Store", + Insn::Sub { .. } => "Sub", + Insn::Mul { .. } => "Mul", + Insn::Test { .. } => "Test", + Insn::URShift { .. } => "URShift", + Insn::Xor { .. } => "Xor" + } + } + + /// Return a non-mutable reference to the out operand for this instruction + /// if it has one. + pub fn out_opnd(&self) -> Option<&Opnd> { + match self { + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaJumpTarget { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::Mul { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Return a mutable reference to the out operand for this instruction if it + /// has one. + pub fn out_opnd_mut(&mut self) -> Option<&mut Opnd> { + match self { + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaJumpTarget { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::Mul { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Returns the target for this instruction if there is one. + pub fn target(&self) -> Option<&Target> { + match self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Joz(_, target) | + Insn::Jonz(_, target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => Some(target), + _ => None + } + } + + /// Returns the text associated with this instruction if there is some. + pub fn text(&self) -> Option<&String> { + match self { + Insn::BakeString(text) | + Insn::Comment(text) => Some(text), + _ => None + } + } + + /// Returns true if this instruction is a terminator (ends a basic block). + pub fn is_terminator(&self) -> bool { + self.is_jump() || + match self { + Insn::CRet(_) => true, + _ => false + } + } + + /// Returns true if this instruction is a jump. + pub fn is_jump(&self) -> bool { + match self { + Insn::Jbe(_) | + Insn::Jb(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jg(_) | + Insn::Jge(_) | + Insn::Jmp(_) | + Insn::JmpOpnd(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::JoMul(_) | + Insn::Jz(_) | + Insn::Joz(..) | + Insn::Jonz(..) => true, + _ => false + } + } +} + +impl fmt::Debug for Insn { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "{}(", self.op())?; + + if let Insn::FrameSetup { slot_count, .. } = self { + write!(fmt, "{slot_count}")?; + } + // Print list of operands + let mut sep = ""; + self.for_each_operand(|opnd| { + write!(fmt, "{sep}{opnd:?}").unwrap(); + sep = ", "; + }); + write!(fmt, ")")?; + + // Print text, target, and pos if they are present + if let Some(text) = self.text() { + write!(fmt, " {text:?}")? + } + if let Some(target) = self.target() { + write!(fmt, " target={target:?}")?; + } + + write!(fmt, " -> {:?}", self.out_opnd().unwrap_or(&Opnd::None)) + } +} + +/// Live range of a VReg +/// TODO: Consider supporting lifetime holes +#[derive(Clone, Debug, PartialEq)] +pub struct LiveRange { + /// Index of the first instruction that used the VReg + pub start: Option<usize>, + /// Index of the last instruction that used the VReg + pub end: Option<usize>, +} + +impl LiveRange { + /// Shorthand for self.start.unwrap() + pub fn start(&self) -> usize { + self.start.unwrap() + } + + /// Shorthand for self.end.unwrap() + pub fn end(&self) -> usize { + self.end.unwrap() + } +} + +/// Live Interval of a VReg +#[derive(Clone)] +pub struct Interval { + pub range: LiveRange, + pub id: usize, +} + +impl Interval { + /// Create a new Interval with no range + pub fn new(i: usize) -> Self { + Self { + range: LiveRange { + start: None, + end: None, + }, + id: i, + } + } + + /// Check if the interval is alive at position x + /// Panics if the range is not set + pub fn survives(&self, x: usize) -> bool { + assert!(self.range.start.is_some() && self.range.end.is_some(), "survives called on interval with no range"); + let start = self.range.start.unwrap(); + let end = self.range.end.unwrap(); + start < x && end > x + } + + pub fn born_at(&self, x:usize) -> bool { + let start = self.range.start.unwrap(); + start == x + } + + pub fn dies_at(&self, x:usize) -> bool { + let end = self.range.end.unwrap(); + end == x + } + + pub fn has_bounds(&self) -> bool { + self.range.start.is_some() && self.range.end.is_some() + } + + /// Add a range to the interval, extending it if necessary + pub fn add_range(&mut self, from: usize, to: usize) { + if to <= from { + panic!("Invalid range: {} to {}", from, to); + } + + if self.range.start.is_none() { + self.range.start = Some(from); + self.range.end = Some(to); + return; + } + + // Extend the range to cover both the existing range and the new range + self.range.start = Some(self.range.start.unwrap().min(from)); + self.range.end = Some(self.range.end.unwrap().max(to)); + } + + /// Set the start of the range + pub fn set_from(&mut self, from: usize) { + let end = self.range.end.unwrap_or(from); + self.range.start = Some(from); + self.range.end = Some(end); + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Allocation { + Reg(usize), + Fixed(Reg), + Stack(usize), +} + +impl Allocation { + fn assigned_reg(self) -> Option<Reg> { + use crate::backend::current::ALLOC_REGS; + + match self { + Allocation::Reg(n) => Some(ALLOC_REGS[n]), + Allocation::Fixed(reg) => Some(reg), + Allocation::Stack(_) => None, + } + } + + fn alloc_pool_index(self, num_registers: usize) -> Option<usize> { + match self { + Allocation::Reg(n) => Some(n), + Allocation::Fixed(reg) => { + use crate::backend::current::ALLOC_REGS; + + ALLOC_REGS + .iter() + .take(num_registers) + .position(|candidate| candidate.reg_no == reg.reg_no) + } + Allocation::Stack(_) => None, + } + } +} + +/// StackState converts abstract stack slots into concrete stack addresses. +pub struct StackState { + /// Copy of Assembler::stack_base_idx. Used for calculating stack slot offsets. + stack_base_idx: usize, +} + +impl StackState { + /// Initialize a stack allocator + pub(super) fn new(stack_base_idx: usize) -> Self { + StackState { stack_base_idx } + } + + /// Convert a stack index to the `disp` of the stack slot + fn stack_idx_to_disp(&self, stack_idx: usize) -> i32 { + (self.stack_base_idx + stack_idx + 1) as i32 * -SIZEOF_VALUE_I32 + } + /// Convert MemBase::Stack to Mem + pub(super) fn stack_membase_to_mem(&self, membase: MemBase) -> Mem { + match membase { + MemBase::Stack { stack_idx, num_bits } => { + let disp = self.stack_idx_to_disp(stack_idx); + Mem { base: MemBase::Reg(NATIVE_BASE_PTR.unwrap_reg().reg_no), disp, num_bits } + } + _ => unreachable!(), + } + } +} + +/// Initial capacity for asm.insns vector +const ASSEMBLER_INSNS_CAPACITY: usize = 256; + +/// Object into which we assemble instructions to be +/// optimized and lowered +#[derive(Clone)] +pub struct Assembler { + pub basic_blocks: Vec<BasicBlock>, + + /// The block to which new instructions are added. Used during HIR to LIR lowering to + /// determine which LIR block we should add instructions to. Set by `set_current_block()` + /// and automatically set to new entry blocks created by `new_block()`. + current_block_id: BlockId, + + /// Number of VRegs allocated + pub(super) num_vregs: usize, + + /// Names of labels + pub(super) label_names: Vec<String>, + + /// If true, `push_insn` is allowed to use scratch registers. + /// On `compile`, it also disables the backend's use of them. + pub(super) accept_scratch_reg: bool, + + /// The maximum number of stack slots that have been reserved + /// by Assembler::alloc_stack(). + pub stack_base_idx: usize, + + /// If Some, the next ccall should verify its leafness + leaf_ccall_stack_size: Option<usize>, + + /// Current instruction index, incremented for each instruction pushed + idx: usize, +} + +impl Assembler +{ + /// Create an Assembler with defaults + pub fn new() -> Self { + Self { + label_names: Vec::default(), + accept_scratch_reg: false, + stack_base_idx: 0, + leaf_ccall_stack_size: None, + basic_blocks: Vec::default(), + current_block_id: BlockId(0), + num_vregs: 0, + idx: 0, + } + } + + /// Create an Assembler, reserving a specified number of stack slots + pub fn new_with_stack_slots(stack_base_idx: usize) -> Self { + Self { stack_base_idx, ..Self::new() } + } + + /// Create an Assembler that allows the use of scratch registers. + /// This should be called only through [`Self::new_with_scratch_reg`]. + pub(super) fn new_with_accept_scratch_reg(accept_scratch_reg: bool) -> Self { + Self { accept_scratch_reg, ..Self::new() } + } + + /// Create an Assembler with parameters of another Assembler and empty instructions. + /// Compiler passes build a next Assembler with this API and insert new instructions to it. + pub(super) fn new_with_asm(old_asm: &Assembler) -> Self { + let mut asm = Self { + label_names: old_asm.label_names.clone(), + accept_scratch_reg: old_asm.accept_scratch_reg, + stack_base_idx: old_asm.stack_base_idx, + ..Self::new() + }; + + // Initialize basic blocks from the old assembler, preserving hir_block_id and entry flag + // but with empty instruction lists + for old_block in &old_asm.basic_blocks { + asm.new_block_from_old_block(&old_block); + } + + // Initialize num_vregs to match the old assembler's size + // This allows reusing VRegs from the old assembler + asm.num_vregs = old_asm.num_vregs; + + asm + } + + // Create a new LIR basic block. Returns the newly created block ID + pub fn new_block(&mut self, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, hir_block_id, is_entry, rpo_index); + self.basic_blocks.push(lir_bb); + if is_entry { + self.set_current_block(bb_id); + } + bb_id + } + + // Create a new LIR basic block from an old one. This should only be used + // when creating new assemblers during passes when we want to translate + // one assembler to a new one. + pub fn new_block_from_old_block(&mut self, old_block: &BasicBlock) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let mut lir_bb = BasicBlock::new(bb_id, old_block.hir_block_id, old_block.is_entry, old_block.rpo_index); + lir_bb.parameters = old_block.parameters.clone(); + self.basic_blocks.push(lir_bb); + bb_id + } + + // Create a LIR basic block without a valid HIR block ID (for testing or internal use). + pub fn new_block_without_id(&mut self, name: &str) -> BlockId { + let bb_id = self.new_block(hir::BlockId(DUMMY_HIR_BLOCK_ID), true, DUMMY_RPO_INDEX); + let label = self.new_label(name); + self.write_label(label); + bb_id + } + + pub fn set_current_block(&mut self, block_id: BlockId) { + self.current_block_id = block_id; + } + + pub fn current_block(&mut self) -> &mut BasicBlock { + &mut self.basic_blocks[self.current_block_id.0] + } + + /// Return basic blocks sorted by RPO index, then by block ID. + /// TODO: Use a more advanced scheduling algorithm + pub fn sorted_blocks(&self) -> Vec<&BasicBlock> { + let mut sorted: Vec<&BasicBlock> = self.basic_blocks.iter().collect(); + sorted.sort_by_key(|block| block.sort_key()); + sorted + } + + /// Validate that jump instructions only appear as the last two instructions in each block. + /// This is a CFG invariant that ensures proper control flow structure. + /// Only active in debug builds. + pub fn validate_jump_positions(&self) { + for block in &self.basic_blocks { + let insns = &block.insns; + let len = insns.len(); + + // Check all instructions except the last two + for (i, insn) in insns.iter().enumerate() { + debug_assert!( + !insn.is_terminator() || i >= len.saturating_sub(2), + "Invalid jump position in block {:?}: {:?} at position {} (block has {} instructions). \ + Jumps must only appear in the last two positions.", + block.id, insn.op(), i, len + ); + } + } + } + + /// Return true if `opnd` is or depends on `reg` + pub fn has_reg(opnd: Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(opnd_reg) => opnd_reg == reg, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => reg_no == reg.reg_no, + _ => false, + } + } + + pub fn instruction_iterator(&mut self) -> InsnIter { + let mut blocks = take(&mut self.basic_blocks); + blocks.sort_by_key(|block| block.sort_key()); + + let mut iter = InsnIter { + blocks, + current_block_idx: 0, + current_insn_iter: vec![].into_iter(), // Will be replaced immediately + peeked: None, + index: 0, + }; + + // Set up first block's iterator + if !iter.blocks.is_empty() { + iter.current_insn_iter = take(&mut iter.blocks[0].insns).into_iter(); + } + + iter + } + + pub fn linearize_instructions(&self) -> Vec<Insn> { + // Emit instructions with labels, expanding branch parameters + let mut insns = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); + + let block_ids = self.block_order(); + let num_blocks = block_ids.len(); + + for (i, block_id) in block_ids.iter().enumerate() { + let block = &self.basic_blocks[block_id.0]; + // Entry blocks shouldn't ever be preceded by something that can + // stomp on this block. + if !block.is_entry { + insns.push(Insn::PadPatchPoint); + } + + // Process each instruction, expanding branch params if needed + for insn in &block.insns { + self.expand_branch_insn(insn, &mut insns); + } + + // Eliminate redundant jumps: if the last instruction is an + // unconditional jump to the next block in the linear order, + // remove it and let execution fall through. + if let Some(next_block_id) = block_ids.get(i + 1) { + let next_label = self.block_label(*next_block_id); + if let Some(Insn::Jmp(Target::Label(label))) = insns.last() { + if *label == next_label { + insns.pop(); + } + } + } + + // Make sure we don't stomp on the next function + if block_id.0 == num_blocks - 1 { + insns.push(Insn::PadPatchPoint); + } + } + insns + } + + /// Expand and linearize a branch instruction: + /// 1. If the branch has Target::Block with arguments, insert a ParallelMov first + /// 2. Convert Target::Block to Target::Label + /// 3. Push the converted instruction + fn expand_branch_insn(&self, insn: &Insn, insns: &mut Vec<Insn>) { + // Helper to process branch arguments and return the label target + let process_edge = |edge: &BranchEdge| -> Label { + self.block_label(edge.target) + }; + + // Convert Target::Block to Target::Label, processing args if needed + let stripped_insn = match insn { + Insn::Jmp(Target::Block(edge)) => Insn::Jmp(Target::Label(process_edge(edge))), + Insn::Jz(Target::Block(edge)) => Insn::Jz(Target::Label(process_edge(edge))), + Insn::Jnz(Target::Block(edge)) => Insn::Jnz(Target::Label(process_edge(edge))), + Insn::Je(Target::Block(edge)) => Insn::Je(Target::Label(process_edge(edge))), + Insn::Jne(Target::Block(edge)) => Insn::Jne(Target::Label(process_edge(edge))), + Insn::Jl(Target::Block(edge)) => Insn::Jl(Target::Label(process_edge(edge))), + Insn::Jg(Target::Block(edge)) => Insn::Jg(Target::Label(process_edge(edge))), + Insn::Jge(Target::Block(edge)) => Insn::Jge(Target::Label(process_edge(edge))), + Insn::Jbe(Target::Block(edge)) => Insn::Jbe(Target::Label(process_edge(edge))), + Insn::Jb(Target::Block(edge)) => Insn::Jb(Target::Label(process_edge(edge))), + Insn::Jo(Target::Block(edge)) => Insn::Jo(Target::Label(process_edge(edge))), + Insn::JoMul(Target::Block(edge)) => Insn::JoMul(Target::Label(process_edge(edge))), + Insn::Joz(opnd, Target::Block(edge)) => Insn::Joz(*opnd, Target::Label(process_edge(edge))), + Insn::Jonz(opnd, Target::Block(edge)) => Insn::Jonz(*opnd, Target::Label(process_edge(edge))), + _ => insn.clone() + }; + + // Push the stripped instruction + insns.push(stripped_insn); + } + + // Get the label for a given block by extracting it from the first instruction. + pub(super) fn block_label(&self, block_id: BlockId) -> Label { + let block = &self.basic_blocks[block_id.0]; + match block.insns.first() { + Some(Insn::Label(Target::Label(label))) => *label, + other => panic!("Expected first instruction of block {:?} to be a Label, but found: {:?}", block_id, other), + } + } + + pub fn expect_leaf_ccall(&mut self, stack_size: usize) { + self.leaf_ccall_stack_size = Some(stack_size); + } + + fn set_stack_canary(&mut self) -> Option<Opnd> { + if cfg!(feature = "runtime_checks") { + if let Some(stack_size) = self.leaf_ccall_stack_size.take() { + let canary_addr = self.lea(Opnd::mem(64, SP, (stack_size as i32) * SIZEOF_VALUE_I32)); + let canary_opnd = Opnd::mem(64, canary_addr, 0); + self.mov(canary_opnd, vm_stack_canary().into()); + return Some(canary_opnd) + } + } + None + } + + fn clear_stack_canary(&mut self, canary_opnd: Option<Opnd>){ + if let Some(canary_opnd) = canary_opnd { + self.store(canary_opnd, 0.into()); + }; + } + + /// Build an Opnd::VReg + pub fn new_vreg(&mut self, num_bits: u8) -> Opnd { + let vreg = Opnd::VReg { idx: VRegId(self.num_vregs), num_bits }; + self.num_vregs += 1; + vreg + } + + /// Build an Opnd::VReg for use as a block parameter. + pub fn new_block_param(&mut self, num_bits: u8) -> Opnd { + self.new_vreg(num_bits) + } + + /// Append an instruction onto the current list of instructions and update + /// the live ranges of any instructions whose outputs are being used as + /// operands to this instruction. + pub fn push_insn(&mut self, insn: Insn) { + // If this Assembler should not accept scratch registers, assert no use of them. + if !self.accept_scratch_reg { + insn.for_each_operand(|opnd| { + assert!(!Self::has_scratch_reg(opnd), "should not use scratch register: {opnd:?}"); + }); + } + + self.idx += 1; + + self.current_block().push_insn(insn); + } + + /// Create a new label instance that we can jump to + pub fn new_label(&mut self, name: &str) -> Target + { + assert!(!name.contains(' '), "use underscores in label names, not spaces"); + + let label = Label(self.label_names.len()); + self.label_names.push(name.to_string()); + Target::Label(label) + } + + // Shuffle register moves, sometimes adding extra moves using scratch_reg, + // so that they will not rewrite each other before they are used. + pub fn resolve_parallel_moves(old_moves: &[(Opnd, Opnd)], scratch_opnd: Option<Opnd>) -> Option<Vec<(Opnd, Opnd)>> { + // Return the index of a move whose destination is not used as a source if any. + fn find_safe_move(moves: &[(Opnd, Opnd)]) -> Option<usize> { + moves.iter().enumerate().find(|&(_, &(dst, src))| { + // Check if `dst` is used in other moves. If `dst` is not used elsewhere, it's safe to write into `dst` now. + moves.iter().filter(|&&other_move| other_move != (dst, src)).all(|&(other_dst, other_src)| + match dst { + Opnd::Reg(reg) => !Assembler::has_reg(other_dst, reg) && !Assembler::has_reg(other_src, reg), + _ => other_dst != dst && other_src != dst, + } + ) + }).map(|(index, _)| index) + } + + // Remove moves whose source and destination are the same + let mut old_moves: Vec<(Opnd, Opnd)> = old_moves.iter().copied() + .filter(|&(dst, src)| dst != src).collect(); + + let mut new_moves = vec![]; + while !old_moves.is_empty() { + // Keep taking safe moves + while let Some(index) = find_safe_move(&old_moves) { + new_moves.push(old_moves.remove(index)); + } + + // No safe move. Load the source of one move into scratch_opnd, and + // then load scratch_opnd into the destination when it's safe. + if !old_moves.is_empty() { + // If scratch_opnd is None, return None and leave it to *_split_with_scratch_regs to resolve it. + let scratch_opnd = scratch_opnd?; + let scratch_reg = scratch_opnd.unwrap_reg(); + // Make sure it's safe to use scratch_reg + assert!(old_moves.iter().all(|&(dst, src)| !Self::has_reg(dst, scratch_reg) && !Self::has_reg(src, scratch_reg))); + + // Move scratch_opnd <- src, and delay dst <- scratch_opnd + let (dst, src) = old_moves.remove(0); + new_moves.push((scratch_opnd, src)); + old_moves.push((dst, scratch_opnd)); + } + } + Some(new_moves) + } + + /// Discover vregs that should preferentially reuse a physical register, + /// such as a newborn vreg immediately moved into a preg in the next instruction. + pub fn preferred_register_assignments(&self, intervals: &[Interval]) -> Vec<Option<Reg>> { + let mut preferred = vec![None; self.num_vregs]; + + for block in &self.basic_blocks { + let mut prev_insn: Option<(InsnId, &Insn)> = None; + + for (insn, insn_id) in block.insns.iter().zip(block.insn_ids.iter()) { + let Some(insn_id) = insn_id else { continue; }; + + if !matches!(insn, Insn::Label(_)) { + if let ( + Some((prev_id, prev)), + Insn::Mov { + dest: Opnd::Reg(dest_reg), + src: Opnd::VReg { idx, .. }, + }, + ) = (prev_insn, insn) + { + if let Some(Opnd::VReg { idx: out_idx, .. }) = prev.out_opnd() { + if out_idx == idx + && intervals[idx.0].born_at(prev_id.0) + && intervals[idx.0].dies_at(insn_id.0) + { + preferred[idx.0].get_or_insert(*dest_reg); + } + } + } + + prev_insn = Some((*insn_id, insn)); + } + } + } + + preferred + } + + // TODO: We want to make the following refactoring so that we DON'T have + // to parcopy in to entry blocks + // + // * Move Allocation to Interval + // * Pre-allocate pinned regs + // * Update linear scan to handle pinned LRs + // + pub fn linear_scan( + &self, + intervals: Vec<Interval>, + num_registers: usize, + preferred_registers: &[Option<Reg>], + ) -> (Vec<Option<Allocation>>, usize) { + assert_eq!(preferred_registers.len(), intervals.len()); + + let mut free_registers: BTreeSet<usize> = (0..num_registers).collect(); + let mut active: Vec<&Interval> = Vec::new(); // vreg indices sorted by increasing end point + let mut assignment: Vec<Option<Allocation>> = vec![None; intervals.len()]; + let mut num_stack_slots: usize = 0; + + // Collect vreg indices that have valid ranges, sorted by start point + let mut sorted_intervals: Vec<Interval> = intervals.iter() + .filter(|i| i.range.start.is_some() && i.range.end.is_some()) + .cloned() + .collect(); + sorted_intervals.sort_by_key(|i| i.range.start.unwrap()); + + for interval in &sorted_intervals { + // Expire old intervals + active.retain(|&active_interval| { + if active_interval.range.end.unwrap() > interval.range.start.unwrap() { + true + } else { + if let Some(allocation) = assignment[active_interval.id] { + if let Some(reg) = allocation.alloc_pool_index(num_registers) { + assert!( + free_registers.insert(reg), + "attempted to return allocator register {:?} to the free pool more than once", + allocation.assigned_reg().unwrap(), + ); + } else { + assert!( + allocation.assigned_reg().is_none_or(|reg| { + crate::backend::current::ALLOC_REGS + .iter() + .take(num_registers) + .all(|candidate| candidate.reg_no != reg.reg_no) + }), + "attempted to return non-allocatable register {:?} to the allocator pool", + allocation.assigned_reg().unwrap(), + ); + } + } + false + } + }); + + let preferred_reg = preferred_registers[interval.id]; + let preferred_taken = preferred_reg.is_some_and(|reg| { + active.iter().any(|active_interval| { + assignment[active_interval.id] + .and_then(|alloc| alloc.assigned_reg()) + .is_some_and(|active_reg| active_reg.reg_no == reg.reg_no) + }) + }); + + if let Some(preferred_reg) = preferred_reg.filter(|_| !preferred_taken) { + if let Some(reg_idx) = Allocation::Fixed(preferred_reg).alloc_pool_index(num_registers) { + if free_registers.remove(®_idx) { + assignment[interval.id] = Some(Allocation::Fixed(preferred_reg)); + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + continue; + } + } else { + assignment[interval.id] = Some(Allocation::Fixed(preferred_reg)); + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + continue; + } + } + + if free_registers.is_empty() { + // Spill: pick the longest-lived active interval (last in sorted active) + // but only from the allocatable register pool. Fixed register + // assignments represent preferred/pinned physical registers + // (for example SP) and should not be selected as spill victims. + let spill = active.iter().rev().copied().find(|active_interval| { + matches!(assignment[active_interval.id], Some(Allocation::Reg(_))) + }); + let slot = Allocation::Stack(num_stack_slots); + num_stack_slots += 1; + + if let Some(spill) = spill.filter(|spill| spill.range.end.unwrap() > interval.range.end.unwrap()) { + // Spill the last active interval; give its register to current + assignment[interval.id] = assignment[spill.id]; + assignment[spill.id] = Some(slot); + let spill_idx = active.iter().position(|active_interval| active_interval.id == spill.id).unwrap(); + active.remove(spill_idx); + // Insert current into sorted active + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + } else { + // Spill the current interval + assignment[interval.id] = Some(slot); + } + } else { + // Allocate lowest free register + let reg = *free_registers.iter().min().unwrap(); + free_registers.remove(®); + assignment[interval.id] = Some(Allocation::Reg(reg)); + // Insert into sorted active + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + } + } + + (assignment, num_stack_slots) + } + + /// Resolve SSA block parameters by inserting sequentialized move instructions + /// at block boundaries. This is SSA deconstruction: after linear_scan assigns + /// registers/stack slots, we lower block parameter passing to explicit moves. + pub fn resolve_ssa(&mut self, _intervals: &[Interval], assignments: &[Option<Allocation>]) { + use crate::backend::parcopy; + use crate::backend::current::SCRATCH_REG; + + // Count predecessors for each block + let mut num_predecessors: HashMap<BlockId, usize> = HashMap::new(); + for block_id in self.block_order() { + for succ in self.basic_blocks[block_id.0].successors() { + *num_predecessors.entry(succ).or_insert(0) += 1; + } + } + + // Collect block order upfront so we don't borrow self while mutating + let block_order = self.block_order(); + + // This code is iterating over each block in our CFG and inserting + // copy instructions at each edge. + for &pred_id in &block_order { + let pred_hir_block_id = self.basic_blocks[pred_id.0].hir_block_id; + let pred_rpo_index = self.basic_blocks[pred_id.0].rpo_index; + let EdgePair(edge1, edge2) = self.basic_blocks[pred_id.0].edges(); + + let edges: Vec<BranchEdge> = [edge1, edge2].into_iter().flatten().collect(); + let num_successors = edges.len(); + + for edge in edges { + let successor = edge.target; + let params = self.basic_blocks[successor.0].parameters.clone(); + + // Build the list of register-to-register copies and immediate moves. + // Rewrite VRegs to physical registers BEFORE sequentialization so + // the parcopy algorithm can see real physical register conflicts. + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = edge.args + .iter() + .zip(params.iter()) + .filter(|(_arg, param)| assignments[param.vreg_idx().0].is_some() ) + .map(|(arg, param)| parcopy::RegisterCopy::<Opnd> { + destination: Self::rewritten_opnd(*param, assignments), + source: Self::rewritten_opnd(*arg, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + // Sequentialize register copies. + // Copies must use physical registers, not VRegs, so the + // parcopy algorithm can detect physical register conflicts. + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + let moves: Vec<Insn> = sequentialized + .iter() + .map(|copy| match copy.source { + Opnd::Value(_) => Insn::LoadInto { dest: copy.destination, opnd: copy.source }, + _ => Insn::Mov { dest: copy.destination, src: copy.source }, + }) + .collect(); + + if moves.is_empty() { + continue; + } + + let num_preds = *num_predecessors.get(&successor).unwrap_or(&0); + if num_preds > 1 && num_successors > 1 { + // Critical edge: create interstitial block + let new_block_id = self.new_block(pred_hir_block_id, false, pred_rpo_index); + let label = self.new_label("split"); + self.basic_blocks[new_block_id.0].push_insn(Insn::Label(label)); + for mov in moves { + self.basic_blocks[new_block_id.0].push_insn(mov); + } + self.basic_blocks[new_block_id.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: successor, + args: vec![], + }))); + + // Redirect predecessor's branch to the new block + let pred_insns = &mut self.basic_blocks[pred_id.0].insns; + for insn in pred_insns.iter_mut() { + if let Some(target) = insn.target_mut() { + if let Target::Block(e) = target { + if e.target == successor { + e.target = new_block_id; + e.args = vec![]; + break; + } + } + } + } + } else if num_successors > 1 { + // Multi-succ: insert at start of successor (after Label) + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[successor.0].insns.insert(1 + i, mov); + self.basic_blocks[successor.0].insn_ids.insert(1 + i, None); + } + } else { + assert_eq!(num_successors, 1); + // Single-succ: insert at end of predecessor before terminator + let len = self.basic_blocks[pred_id.0].insns.len(); + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[pred_id.0].insns.insert(len - 1 + i, mov); + self.basic_blocks[pred_id.0].insn_ids.insert(len - 1 + i, None); + } + } + } + } + + // Handle entry block parameters: move from calling-convention registers + // to their allocated locations, just like inter-block edge moves above. + for &block_id in &block_order { + if !self.basic_blocks[block_id.0].is_entry { continue; } + if self.basic_blocks[block_id.0].is_dummy() { continue; } + let params = self.basic_blocks[block_id.0].parameters.clone(); + + // JIT-to-JIT entries that would need more argument registers should + // be unreachable because can_direct_send() refuses to call them. + // Keep compiling the function body, but make the unsupported entry + // abort if control ever reaches it. TODO: Remove this (Shopify/ruby#916) + if params.len() > C_ARG_OPNDS.len() { + let insert_pos = self.basic_blocks[block_id.0].insns.iter() + .position(|insn| matches!(insn, Insn::FrameSetup { .. })) + .or_else(|| self.basic_blocks[block_id.0].insns.iter().position(|insn| matches!(insn, Insn::Label(_))).map(|idx| idx + 1)) + .unwrap_or(0); + self.basic_blocks[block_id.0].insns.insert(insert_pos, Insn::Abort); + self.basic_blocks[block_id.0].insn_ids.insert(insert_pos, None); + continue; + } + + // Rewrite VRegs to physical registers before sequentialization + // so the parcopy algorithm can detect physical register conflicts. + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = params.iter().enumerate() + .map(|(i, param)| parcopy::RegisterCopy::<Opnd> { + source: C_ARG_OPNDS[i], + destination: Self::rewritten_opnd(*param, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + let moves: Vec<Insn> = sequentialized + .iter() + .map(|copy| match copy.source { + Opnd::Value(_) => Insn::LoadInto { + dest: copy.destination, + opnd: copy.source, + }, + _ => Insn::Mov { + dest: copy.destination, + src: copy.source, + }, + }) + .collect(); + + // Find the position after FrameSetup to insert moves + let insert_pos = self.basic_blocks[block_id.0].insns.iter() + .position(|insn| matches!(insn, Insn::FrameSetup { .. })) + .or_else(|| self.basic_blocks[block_id.0].insns.iter().position(|insn| matches!(insn, Insn::Label(_))).map(|idx| idx + 1)) + .unwrap_or(0); + + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[block_id.0].insns.insert(insert_pos + i, mov); + self.basic_blocks[block_id.0].insn_ids.insert(insert_pos + i, None); + } + } + + // Clear edge args on all branch instructions since the moves have been + // materialized as explicit Mov instructions. This prevents + // linearize_instructions from generating redundant ParallelMov instructions. + for block_id in &block_order { + for insn in &mut self.basic_blocks[block_id.0].insns { + if let Some(Target::Block(edge)) = insn.target_mut() { + edge.args.clear(); + } + } + } + + self.rewrite_instructions(assignments); + } + + /// Handle caller-saved registers around CCall instructions. + /// For each CCall, push live caller-saved registers, set up arguments + /// in C calling convention registers, and pop saved registers after. + pub fn handle_caller_saved_regs( + &mut self, + intervals: &[Interval], + assignments: &[Option<Allocation>], + regs: &[Reg], + ) { + use crate::backend::parcopy; + use crate::backend::current::{C_RET_OPND, SCRATCH_REG, ALLOC_REGS}; + + for block_id in self.block_order() { + let block = &mut self.basic_blocks[block_id.0]; + let old_insns = take(&mut block.insns); + let old_ids = take(&mut block.insn_ids); + + let mut new_insns = Vec::with_capacity(old_insns.len()); + let mut new_ids = Vec::with_capacity(old_ids.len()); + + for (insn, insn_id) in old_insns.into_iter().zip(old_ids.into_iter()) { + if let Insn::CCall { opnds, out, start_marker, end_marker, fptr } = insn { + let insn_number = insn_id.map(|id| id.0).unwrap_or(0); + // Do we have a case where a ccall is emitted, but nobody + // uses the result? + let call_result_live = out.is_vreg() + && intervals[out.vreg_idx().0] + .range + .end + .is_some_and(|end| end > insn_number); + + // Find survivors: intervals that survive this Call instruction + // We need to preserve the "surviving" registers past the ccall, + // so we're going to push them all on the stack, then pop + // after we make the ccall + let survivors: Vec<usize> = intervals.iter() + .filter(|interval| { + interval.has_bounds() + && interval.survives(insn_number) + && assignments[interval.id].and_then(|alloc| alloc.alloc_pool_index(ALLOC_REGS.len())).is_some() + }) + .map(|interval| interval.id) + .collect(); + + let survivor_regs: Vec<Opnd> = survivors.iter() + .map(|&s| match assignments[s].unwrap() { + Allocation::Reg(n) => Opnd::Reg(ALLOC_REGS[n]), + Allocation::Fixed(reg) => Opnd::Reg(reg), + _ => unreachable!(), + }) + .collect(); + + // Push all survivors on the stack, pairing adjacent pushes when possible. + for group in survivor_regs.chunks(2) { + match group { + &[left, right] => new_insns.push(Insn::CPushPair(left, right)), + &[reg] => new_insns.push(Insn::CPushPair(reg, 0.into())), + _ => unreachable!(), + } + new_ids.push(None); + } + // Extract arguments from CCall, clear opnds + + assert!(opnds.len() <= regs.len()); + + // Sequentialize argument moves: each arg goes to regs[i] + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = opnds + .iter() + .zip(regs.iter()) + .map(|(arg, param)| parcopy::RegisterCopy::<Opnd> { + destination: Opnd::Reg(*param), + source: Self::rewritten_opnd(*arg, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + + for copy in sequentialized { + new_insns.push(match copy.source { + Opnd::Value(_) => Insn::LoadInto { dest: copy.destination, opnd: copy.source }, + _ => Insn::Mov { dest: copy.destination, src: copy.source }, + }); + new_ids.push(None); + } + + // Extract PosMarkers from the CCall so they get emitted + // as separate instructions at the right code positions. + // Emit start_marker PosMarker before the CCall + if let Some(marker) = start_marker { + new_insns.push(Insn::PosMarker(marker)); + new_ids.push(None); + } + + // The CCall itself + new_insns.push(Insn::CCall { + out: C_RET_OPND, + opnds: vec![], // We've moved everything in to ccall regs, so this should + // be empty now + start_marker: None, + end_marker: None, + fptr + }); + new_ids.push(insn_id); + + // Emit end_marker PosMarker after the CCall + if let Some(marker) = end_marker { + new_insns.push(Insn::PosMarker(marker)); + new_ids.push(None); + } + + if survivors.is_empty() { + if call_result_live { + // No survivors to restore -- move result directly to output. + let out = Self::rewritten_opnd(out, assignments); + new_insns.push(Insn::Mov { dest: out, src: C_RET_OPND }); + new_ids.push(None); + } + } else { + if call_result_live { + // Save CCall result to scratch immediately, before pops + // can clobber either C_RET or the output register. + new_insns.push(Insn::Mov { dest: Opnd::Reg(SCRATCH_REG), src: C_RET_OPND }); + new_ids.push(None); + } + + // Restore all survivors in reverse stack order, pairing adjacent pops when possible. + for group in survivor_regs.chunks(2).rev() { + match group { + &[reg] => new_insns.push(Insn::CPopPairInto(reg, reg)), + &[left, right] => new_insns.push(Insn::CPopPairInto(right, left)), + _ => unreachable!(), + } + new_ids.push(None); + } + + if call_result_live { + // Move result from scratch to output AFTER all pops. + let out = Self::rewritten_opnd(out, assignments); + new_insns.push(Insn::Mov { dest: out, src: Opnd::Reg(SCRATCH_REG) }); + new_ids.push(None); + } + } + } else { + new_insns.push(insn); + new_ids.push(insn_id); + } + } + + let block = &mut self.basic_blocks[block_id.0]; + block.insns = new_insns; + block.insn_ids = new_ids; + } + } + + /// Walk every instruction and replace VReg operands with the physical + /// register (or stack slot) from the allocation assignments. + fn rewrite_instructions(&mut self, assignments: &[Option<Allocation>]) { + for block_id in self.block_order() { + for insn in self.basic_blocks[block_id.0].insns.iter_mut() { + insn.for_each_operand_mut(|opnd| { + Self::rewrite_opnd(opnd, assignments); + }); + if let Some(out) = insn.out_opnd_mut() { + Self::rewrite_opnd(out, assignments); + } + } + } + } + + fn rewritten_opnd(mut opnd: Opnd, assignments: &[Option<Allocation>]) -> Opnd { + Self::rewrite_opnd(&mut opnd, assignments); + opnd + } + + fn rewrite_opnd(opnd: &mut Opnd, assignments: &[Option<Allocation>]) { + use crate::backend::current::ALLOC_REGS; + let regs = &ALLOC_REGS; + + match opnd { + Opnd::VReg { idx, num_bits } => { + if let Some(assignment) = assignments[idx.0] { + match assignment { + Allocation::Reg(n) => { + let mut reg = regs[n]; + reg.num_bits = *num_bits; + *opnd = Opnd::Reg(reg); + } + Allocation::Fixed(mut reg) => { + reg.num_bits = *num_bits; + *opnd = Opnd::Reg(reg); + } + Allocation::Stack(n) => { + let num_bits = *num_bits; + *opnd = Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx: n, num_bits }, + disp: 0, + num_bits, + }); + } + } + } else { + panic!("Expected assignment for {opnd}"); + } + } + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { + match assignments[idx.0].unwrap() { + Allocation::Reg(n) => { + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::Reg(regs[n].reg_no); + } + } + Allocation::Fixed(reg) => { + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::Reg(reg.reg_no); + } + } + Allocation::Stack(n) => { + // The VReg used as a memory base was spilled to a stack slot. + // Mark it as StackIndirect so arm64_scratch_split can load + // the pointer from the stack into a scratch register. + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::StackIndirect { stack_idx: n }; + } + } + } + } + _ => {} + } + } + + /// Compile the instructions down to machine code. + /// Can fail due to lack of code memory and inopportune code placement, among other reasons. + pub fn compile(self, cb: &mut CodeBlock) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + #[cfg(feature = "disasm")] + let start_addr = cb.get_write_ptr(); + let alloc_regs = Self::get_alloc_regs(); + let had_dropped_bytes = cb.has_dropped_bytes(); + let ret = self.compile_with_regs(cb, alloc_regs).inspect_err(|err| { + // If we use too much memory to compile the Assembler, it would set cb.dropped_bytes = true. + // To avoid failing future compilation by cb.has_dropped_bytes(), attempt to reset dropped_bytes with + // the current zjit_alloc_bytes() which may be decreased after self is dropped in compile_with_regs(). + if *err == CompileError::OutOfMemory && !had_dropped_bytes { + cb.update_dropped_bytes(); + } + }); + + #[cfg(feature = "disasm")] + if let Some(dump_disasm) = crate::options::get_option_ref!(dump_disasm).filter(|_| ret.is_ok()) { + let end_addr = cb.get_write_ptr(); + crate::disasm::dump_disasm_addr_range(cb, start_addr, end_addr, dump_disasm); + } + ret + } + + /// Compile with a limited number of registers. Used only for unit tests. + #[cfg(test)] + pub fn compile_with_num_regs(self, cb: &mut CodeBlock, num_regs: usize) -> (CodePtr, Vec<CodePtr>) { + let mut alloc_regs = Self::get_alloc_regs(); + let alloc_regs = alloc_regs.drain(0..num_regs).collect(); + self.compile_with_regs(cb, alloc_regs).unwrap() + } + + /// Compile Target::SideExit and convert it into Target::Label for all instructions. + /// Returns the exit code as a list of instructions to be appended after the main + /// code is linearized and split. + pub fn compile_exits(&mut self) -> Vec<Insn> { + /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. + fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { + let SideExit { pc, stack, locals, iseq, .. } = exit; + + // Side exit blocks are not part of the CFG at the moment, + // so we need to manually ensure that patchpoints get padded + // so that nobody stomps on us + asm.pad_patch_point(); + + asm_comment!(asm, "save cfp->pc"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); + + asm_comment!(asm, "save cfp->sp"); + asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + + asm_comment!(asm, "save cfp->iseq"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); + + // cfp->block_code and cfp->jit_return are cleared by the materialize_exit trampoline + + if !stack.is_empty() { + asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); + for (idx, &opnd) in stack.iter().enumerate() { + asm.store(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + } + + if !locals.is_empty() { + asm_comment!(asm, "write locals: {}", join_opnds(&locals, ", ")); + for (idx, &opnd) in locals.iter().enumerate() { + asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); + } + } + } + + /// Tear down the JIT frame and return to the interpreter. + fn compile_exit_return(asm: &mut Assembler) { + asm_comment!(asm, "exit to the interpreter"); + asm.jmp(Target::CodePtr(ZJITState::get_materialize_exit_trampoline())); + } + + fn compile_exit_recompile(asm: &mut Assembler, exit: &SideExit) { + if let Some(recompile) = &exit.recompile { + + use crate::codegen::exit_recompile; + asm_comment!(asm, "profile and maybe recompile"); + asm_ccall!(asm, exit_recompile, + EC, + recompile.iseq, + Opnd::UImm(recompile.insn_idx as u64), + Opnd::Imm(match recompile.strategy { + hir::Recompile::ProfileSend { argc } => argc as i64, + hir::Recompile::ProfileSelf => -1, + }) + ); + } + } + + /// Compile the main side-exit code. The side exit will optionally record a traced exit + /// stack, optionally trigger recompilation, and then return to the interpreter. Shared + /// exits pass no trace reason so they can still be deduplicated by SideExit. + /// IOW, we should never pass a trace reason if we expect the exit to be + /// deduplicated. + fn compile_exit(asm: &mut Assembler, exit: &SideExit, trace_reason: Option<SideExitReason>) { + // Save VM state before the ccall so that + // rb_profile_frames sees valid cfp->pc and the + // ccall doesn't clobber caller-saved registers + // holding stack/local operands. + compile_exit_save_state(asm, exit); + if trace_reason.is_some() || exit.recompile.is_some() { + // Clear cfp->jit_return to prepare for a C call. Normally, cfp->jit_return + // is cleared by the materialize_exit trampoline, but if we're about to + // make a C call, we need to clear any stale JITFrame. + asm_comment!(asm, "clear cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + } + if let Some(reason) = trace_reason { + // Leak a CString with the reason so it's available at runtime + let reason_cstr = std::ffi::CString::new(reason.to_string()) + .unwrap_or_else(|_| std::ffi::CString::new("unknown").unwrap()); + let reason_ptr = reason_cstr.into_raw() as *const u8; + asm_ccall!(asm, rb_zjit_record_exit_stack, Opnd::const_ptr(reason_ptr)); + } + compile_exit_recompile(asm, exit); + compile_exit_return(asm); + } + + fn join_opnds(opnds: &Vec<Opnd>, delimiter: &str) -> String { + opnds.iter().map(|opnd| format!("{opnd}")).collect::<Vec<_>>().join(delimiter) + } + + // Extract targets first so that we can update instructions while referencing part of them. + let mut targets = HashMap::new(); + + for block_id in self.block_order() { + let block = &self.basic_blocks[block_id.0]; + for (idx, insn) in block.insns.iter().enumerate() { + if let Some(target @ Target::SideExit { .. }) = insn.target() { + targets.insert((block_id.0, idx), target.clone()); + } + } + } + + // Create a dedicated block for exit code. This block is not part of the + // CFG (DUMMY_RPO_INDEX), so it won't be included in block_order() or + // linearize_instructions(). Its instructions are returned to the caller + // for appending after scratch_split. + let saved_block = self.current_block_id; + let exit_block = self.new_block_without_id("side_exits"); + + // Map from SideExit to compiled Label. This table is used to deduplicate side exit code. + let mut compiled_exits: HashMap<SideExit, Label> = HashMap::new(); + + // Start a new perf range for side exits + let perf_symbol = if get_option!(perf) == Some(PerfMap::HIR) { + Some(perf_symbol_range_start(self, "side exit")) + } else { + None + }; + + // Mark the start of side-exit code so we can measure its size + if !targets.is_empty() { + self.pos_marker(move |start_pos, cb| { + let end_pos = cb.get_write_ptr(); + let size = end_pos.as_offset() - start_pos.as_offset(); + crate::stats::incr_counter_by(crate::stats::Counter::side_exit_size, size as u64); + }); + } + + // Measure time spent compiling side-exit LIR + let side_exit_start = std::time::Instant::now(); + + for ((block_id, idx), target) in targets { + // Compile a side exit. Note that this is past register assignment, + // so you can't use an instruction that returns a VReg. + if let Target::SideExit { exit: exit @ SideExit { pc, .. }, reason } = target { + // Only record the exit if `trace_side_exits` is defined and the counter is either the one specified + let should_record_exit = get_option!(trace_side_exits).map(|trace| match trace { + TraceExits::All => true, + TraceExits::Counter(counter) if counter == side_exit_counter(reason) => true, + _ => false, + }).unwrap_or(false); + + // If enabled, instrument exits first, and then jump to a shared exit. + let counted_exit = if get_option!(stats) || should_record_exit || cfg!(test) { + let counted_exit = self.new_label("counted_exit"); + self.write_label(counted_exit.clone()); + asm_comment!(self, "Counted Exit: {reason}"); + + if get_option!(stats) || cfg!(test) { + asm_comment!(self, "increment a side exit counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr(reason)), 1.into()); + + if let SideExitReason::UnhandledYARVInsn(opcode) = reason { + asm_comment!(self, "increment an unhandled YARV insn counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr_for_opcode(opcode)), 1.into()); + } + } + + if should_record_exit { + compile_exit(self, &exit, Some(reason)); + } else { + // If the side exit has already been compiled, jump to it. + // Otherwise, let it fall through and compile the exit next. + if let Some(&exit_label) = compiled_exits.get(&exit) { + self.jmp(Target::Label(exit_label)); + } + } + Some(counted_exit) + } else { + None + }; + + // Compile the shared side exit if not compiled yet + let compiled_exit = if let Some(&compiled_exit) = compiled_exits.get(&exit) { + Target::Label(compiled_exit) + } else { + let new_exit = self.new_label("side_exit"); + self.write_label(new_exit.clone()); + asm_comment!(self, "Exit: {pc}"); + compile_exit(self, &exit, None); + compiled_exits.insert(exit, new_exit.unwrap_label()); + new_exit + }; + + *self.basic_blocks[block_id].insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); + } + } + + // Measure time spent compiling side-exit LIR + if !compiled_exits.is_empty() { + let nanos = side_exit_start.elapsed().as_nanos(); + crate::stats::incr_counter_by(crate::stats::Counter::compile_side_exit_time_ns, nanos as u64); + crate::stats::incr_counter_by(crate::stats::Counter::compiled_side_exit_count, compiled_exits.len() as u64); + } + + // Close the current perf range for side exits + if let Some(perf_symbol) = &perf_symbol { + perf_symbol_range_end(self, perf_symbol); + } + + // Extract exit instructions and restore the previous current block + let exit_insns = take(&mut self.basic_blocks[exit_block.0].insns); + self.set_current_block(saved_block); + exit_insns + } + + /// Return a traversal of the block graph in reverse post-order. + pub fn reverse_post_order(&self) -> Vec<BlockId> { + let entry_blocks: Vec<BlockId> = self.basic_blocks.iter() + .filter(|block| block.is_entry) + .map(|block| block.id) + .collect(); + let mut result = self.po_from(entry_blocks); + result.reverse(); + result + } + + /// Compute postorder traversal starting from the given blocks. + /// Outbound edges are extracted from the last 0, 1, or 2 instructions (jumps). + fn po_from(&self, starts: Vec<BlockId>) -> Vec<BlockId> { + #[derive(PartialEq)] + enum Action { + VisitEdges, + VisitSelf, + } + let mut result = vec![]; + let mut seen = HashSet::with_capacity(self.basic_blocks.len()); + let mut stack: Vec<_> = starts.iter().map(|&start| (start, Action::VisitEdges)).collect(); + while let Some((block, action)) = stack.pop() { + if action == Action::VisitSelf { + result.push(block); + continue; + } + if !seen.insert(block) { continue; } + stack.push((block, Action::VisitSelf)); + let EdgePair(edge1, edge2) = self.basic_blocks[block.0].edges(); + // Push edge2 before edge1 so that edge1 is popped first from the + // LIFO stack, matching the visit order of a recursive DFS. + if let Some(edge) = edge2 { + stack.push((edge.target, Action::VisitEdges)); + } + if let Some(edge) = edge1 { + stack.push((edge.target, Action::VisitEdges)); + } + } + result + } + + /// Number all instructions in the LIR in reverse postorder. + /// This assigns a unique InsnId to each instruction across all blocks, skipping labels. + /// Also sets the from/to range on each block. + /// Returns the next available instruction ID after numbering. + pub fn number_instructions(&mut self, start: usize) -> usize { + let block_ids = self.block_order(); + let mut insn_id = start; + for block_id in block_ids { + let block = &mut self.basic_blocks[block_id.0]; + let block_start = insn_id; + insn_id += 2; + for (insn, id_slot) in block.insns.iter().zip(block.insn_ids.iter_mut()) { + if matches!(insn, Insn::Label(_)) { + *id_slot = Some(InsnId(block_start)); + } else { + *id_slot = Some(InsnId(insn_id)); + insn_id += 2; + } + } + block.from = InsnId(block_start); + block.to = InsnId(insn_id); + } + insn_id + } + + /// Iterate over all instructions mutably with their block ID, instruction ID, and instruction index within the block. + /// Returns an iterator of (BlockId, `Option<InsnId>`, usize, &mut Insn). + pub fn iter_insns_mut(&mut self) -> impl Iterator<Item = (BlockId, Option<InsnId>, usize, &mut Insn)> { + self.basic_blocks.iter_mut().flat_map(|block| { + let block_id = block.id; + block.insns.iter_mut() + .zip(block.insn_ids.iter().copied()) + .enumerate() + .map(move |(idx, (insn, insn_id))| (block_id, insn_id, idx, insn)) + }) + } + + /// Compute initial liveness sets (kill and gen) for the given blocks. + /// Returns (kill_sets, gen_sets) where each is indexed by block ID. + /// - kill: VRegs defined (written) in the block + /// - gen: VRegs used (read) in the block before being defined + pub fn compute_initial_liveness_sets(&self, block_ids: &[BlockId]) -> (Vec<BitSet<usize>>, Vec<BitSet<usize>>) { + let num_blocks = self.basic_blocks.len(); + let num_vregs = self.num_vregs; + + let mut kill_sets: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + let mut gen_sets: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + + for &block_id in block_ids { + let block = &self.basic_blocks[block_id.0]; + let kill_set = &mut kill_sets[block_id.0]; + let gen_set = &mut gen_sets[block_id.0]; + + // Iterate over instructions in reverse + for insn in block.insns.iter().rev() { + // If the instruction has an output that is a VReg, add to kill set + if let Some(out) = insn.out_opnd() { + if let Opnd::VReg { idx, .. } = out { + kill_set.insert(idx.0); + } + } + + // For all input operands that are VRegs (including memory base VRegs), add to gen set + insn.for_each_operand(|opnd| { + for idx in opnd.vreg_ids() { + assert!(!kill_set.get(idx.0)); + gen_set.insert(idx.0); + } + }); + } + + // Add block parameters to kill set + for param in &block.parameters { + if let Opnd::VReg { idx, .. } = param { + kill_set.insert(idx.0); + } + } + + } + + (kill_sets, gen_sets) + } + + pub fn block_order(&self) -> Vec<BlockId> { + self.reverse_post_order() + } + + /// Calculate live intervals for each VReg. + pub fn build_intervals(&self, live_in: Vec<BitSet<usize>>) -> Vec<Interval> { + let num_vregs = self.num_vregs; + let mut intervals: Vec<Interval> = (0..num_vregs) + .map(|i| Interval::new(i)) + .collect(); + + let blocks = self.block_order(); + + for block_id in blocks { + let block = &self.basic_blocks[block_id.0]; + + // live = union of successor.liveIn for each successor + let mut live = BitSet::with_capacity(num_vregs); + for succ_id in block.successors() { + live.union_with(&live_in[succ_id.0]); + } + + // Add out_vregs to live set + for idx in block.out_vregs() { + live.insert(idx.0); + } + + // For each live vreg, add entire block range + // block.to is the first instruction of the next block + for idx in live.iter_set_bits() { + intervals[idx].add_range(block.from.0, block.to.0); + } + + // Iterate instructions in reverse + for (insn_id, insn) in block.insn_ids.iter().zip(&block.insns).rev() { + // TODO(max): Remove labels, which are not numbered, in favor of blocks + let Some(insn_id) = insn_id else { continue; }; + // If instruction has VReg output, set_from + if let Some(out) = insn.out_opnd() { + if let Opnd::VReg { idx, .. } = out { + intervals[idx.0].set_from(insn_id.0); + } + } + + // For each VReg input (including memory base VRegs), add_range from block start to insn + insn.for_each_operand(|opnd| { + for idx in opnd.vreg_ids() { + intervals[idx.0].add_range(block.from.0, insn_id.0); + } + }); + } + } + + intervals + } + + /// Analyze liveness for all blocks using a fixed-point algorithm. + /// Returns live_in sets for each block, indexed by block ID. + /// A VReg is live-in to a block if it may be used before being defined. + pub fn analyze_liveness(&self) -> Vec<BitSet<usize>> { + // Get blocks in postorder + let po_blocks = { + let entry_blocks: Vec<BlockId> = self.basic_blocks.iter() + .filter(|block| block.is_entry) + .map(|block| block.id) + .collect(); + self.po_from(entry_blocks) + }; + + // Compute initial gen/kill sets + let (kill_sets, gen_sets) = self.compute_initial_liveness_sets(&po_blocks); + + let num_blocks = self.basic_blocks.len(); + let num_vregs = self.num_vregs; + + // Initialize live_in sets + let mut live_in: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + + // Fixed-point iteration + let mut changed = true; + while changed { + changed = false; + + // Iterate over blocks in postorder + for &block_id in &po_blocks { + let block = &self.basic_blocks[block_id.0]; + + // block_live = union of live_in[succ] for all successors + let mut block_live = BitSet::with_capacity(num_vregs); + for succ_id in block.successors() { + block_live.union_with(&live_in[succ_id.0]); + } + + // block_live |= gen[block] + block_live.union_with(&gen_sets[block_id.0]); + + // block_live &= ~kill[block] + block_live.difference_with(&kill_sets[block_id.0]); + + // Update live_in if changed + if !live_in[block_id.0].equals(&block_live) { + live_in[block_id.0] = block_live; + changed = true; + } + } + } + + live_in + } +} + +/// Return a result of fmt::Display for Assembler without escape sequence +pub fn lir_string(asm: &Assembler) -> String { + use crate::ttycolors::TTY_TERMINAL_COLOR; + format!("{asm}").replace(TTY_TERMINAL_COLOR.bold_begin, "").replace(TTY_TERMINAL_COLOR.bold_end, "") +} + +/// Format live intervals as a grid showing which VRegs are alive at each instruction +pub fn lir_intervals_string(asm: &Assembler, intervals: &[Interval]) -> String { + let mut output = String::new(); + let num_vregs = intervals.len(); + + let vreg_header = |output: &mut String| { + output.push_str(" "); + for i in 0..num_vregs { + output.push_str(&format!(" v{:<2}", i)); + } + output.push('\n'); + + output.push_str(" "); + for _ in 0..num_vregs { + output.push_str(" ---"); + } + output.push('\n'); + }; + + // Collect all numbered instruction positions in RPO order + let mut first = true; + for block_id in asm.block_order() { + let block = &asm.basic_blocks[block_id.0]; + + // Print VReg header before each block + if !first { output.push('\n'); } + first = false; + vreg_header(&mut output); + + // Print basic block label header with parameters + let label = asm.block_label(block_id); + if block.parameters.is_empty() { + output.push_str(&format!("{}():\n", asm.label_names[label.0])); + } else { + output.push_str(&format!("{}(", asm.label_names[label.0])); + for (idx, param) in block.parameters.iter().enumerate() { + if idx > 0 { + output.push_str(", "); + } + output.push_str(&format!("{param}")); + } + output.push_str("):\n"); + } + + for (insn, insn_id) in block.insns.iter().zip(&block.insn_ids) { + // Skip labels (they're not numbered) + let Some(insn_id) = insn_id else { panic!("{insn:?}"); }; + + // Print instruction ID + output.push_str(&format!("i{:<6}: ", insn_id.0)); + + // For each VReg, check if it's alive at this position + for vreg_idx in 0..num_vregs { + let is_alive = intervals[vreg_idx].range.start.is_some() && + intervals[vreg_idx].range.end.is_some() && + intervals[vreg_idx].survives(insn_id.0); + + let has_range = intervals[vreg_idx].range.start.is_some(); + if has_range && intervals[vreg_idx].born_at(insn_id.0) { + output.push_str(" v "); + } else if has_range && intervals[vreg_idx].dies_at(insn_id.0) { + output.push_str(" ^ "); + } else if is_alive { + output.push_str(" █ "); + } else { + output.push_str(" . "); + } + } + + if let Insn::Label(_) = insn { + output.push('\n'); + continue; + } + + // Show the instruction text using compact formatting + output.push_str(" "); + + if let Insn::Comment(comment) = insn { + output.push_str(&format!("# {}", comment)); + } else { + // Print output operand if any + if let Some(out) = insn.out_opnd() { + output.push_str(&format!("{out} = ")); + } + + // Use the helper function to format instruction (reuses Display logic) + output.push_str(&format_insn_compact(asm, insn)); + } + + output.push('\n'); + } + } + + output +} + +/// Format live intervals as a grid showing which VRegs are alive at each instruction +pub fn debug_intervals(asm: &Assembler, intervals: &[Interval]) -> String { + lir_intervals_string(asm, intervals) +} + +/// Helper function to format a single instruction (without the output part, which is already printed) +/// Returns a string formatted like: "OpName target operand1, operand2, ..." +fn format_insn_compact(asm: &Assembler, insn: &Insn) -> String { + let mut output = String::new(); + + // Print the instruction name + output.push_str(insn.op()); + + // Print target (before operands, to match --zjit-dump-lir format) + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => output.push_str(&format!(" {code_ptr:?}")), + Target::Label(Label(label_idx)) => output.push_str(&format!(" {}", asm.label_names[*label_idx])), + Target::SideExit { reason, .. } => output.push_str(&format!(" Exit({reason})")), + Target::Block(edge) => { + let label = asm.block_label(edge.target); + let name = &asm.label_names[label.0]; + if edge.args.is_empty() { + output.push_str(&format!(" {name}")); + } else { + output.push_str(&format!(" {name}(")); + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + output.push_str(", "); + } + output.push_str(&format!("{}", arg)); + } + output.push_str(")"); + } + } + } + } + + // Print operands (but skip branch args since they're already printed with target) + if let Some(Target::SideExit { .. }) = insn.target() { + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + output.push_str(&format!(", {opnd}")); + } + _ => {} + } + } else if let Some(Target::Block(_)) = insn.target() { + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + output.push_str(&format!(", {opnd}")); + } + _ => {} + } + } else if insn.opnd_count() > 0 { + let mut sep = ""; + insn.for_each_operand(|opnd| { + output.push_str(&format!("{sep}{opnd}")); + sep = ", "; + }); + } + + output +} + +impl fmt::Display for Assembler { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Count the number of duplicated label names to disambiguate them if needed + let mut label_counts: HashMap<&String, usize> = HashMap::new(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + for label_name in self.label_names.iter() { + let counter = label_counts.entry(label_name).or_insert(0); + *counter += 1; + } + + /// Return a label name String. Suffix "_{label_idx}" if the label name is used multiple times. + fn label_name(asm: &Assembler, label_idx: usize, label_counts: &HashMap<&String, usize>) -> String { + let label_name = &asm.label_names[label_idx]; + let label_count = label_counts.get(&label_name).unwrap_or(&0); + if *label_count > 1 { + format!("{label_name}_{label_idx}") + } else { + label_name.to_string() + } + } + + // Use sorted_blocks() instead of block_order() because block_order() + // calls rpo() -> edges() which requires all blocks end with terminators. + // After arm64_scratch_split, blocks may not have terminators. + for bb in self.sorted_blocks() { + let params = &bb.parameters; + for (insn_id, insn) in bb.insn_ids.iter().zip(&bb.insns) { + if let Some(id) = insn_id { + write!(f, "{id}: ")?; + } else { + write!(f, " ")?; + } + match insn { + Insn::Comment(comment) => { + writeln!(f, " {bold_begin}# {comment}{bold_end}")?; + } + Insn::Label(target) => { + let Target::Label(Label(label_idx)) = target else { + panic!("unexpected target for Insn::Label: {target:?}"); + }; + write!(f, " {}(", label_name(self, *label_idx, &label_counts))?; + for (idx, param) in params.iter().enumerate() { + if idx > 0 { + write!(f, ", ")?; + } + write!(f, "{param}")?; + } + writeln!(f, "):")?; + } + _ => { + write!(f, " ")?; + + // Print output operand if any + if let Some(out) = insn.out_opnd() { + write!(f, "{out} = ")?; + } + + // Print the instruction name + write!(f, "{}", insn.op())?; + + // Show slot_count for FrameSetup + if let Insn::FrameSetup { slot_count, preserved } = insn { + write!(f, " {slot_count}")?; + if !preserved.is_empty() { + write!(f, ",")?; + } + } + + // Print target + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => write!(f, " {code_ptr:?}")?, + Target::Label(Label(label_idx)) => write!(f, " {}", label_name(self, *label_idx, &label_counts))?, + Target::SideExit { reason, .. } => write!(f, " Exit({reason})")?, + Target::Block(edge) => { + let label = self.block_label(edge.target); + let name = label_name(self, label.0, &label_counts); + if edge.args.is_empty() { + write!(f, " {name}")?; + } else { + write!(f, " {name}(")?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + write!(f, ")")?; + } + } + } + } + + // Print list of operands + if let Some(Target::SideExit { .. }) = insn.target() { + // If the instruction has a SideExit, avoid using opnd_iter(), which has stack/locals. + // Here, only handle instructions that have both Opnd and Target. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if let Some(Target::Block(_)) = insn.target() { + // If the instruction has a Block target, avoid using opnd_iter() for branch args + // since they're already printed inline with the target. Only print non-target operands. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if insn.opnd_count() > 0 { + let mut sep = " "; + insn.try_for_each_operand(|opnd| { + let result = write!(f, "{sep}{opnd}"); + sep = ", "; + result + })?; + } + + write!(f, "\n")?; + } + } + } + } + Ok(()) + } +} + +impl fmt::Debug for Assembler { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + writeln!(fmt, "Assembler")?; + + for (idx, insn) in self.linearize_instructions().iter().enumerate() { + writeln!(fmt, " {idx:03} {insn:?}")?; + } + + Ok(()) + } +} + +pub struct InsnIter { + blocks: Vec<BasicBlock>, + current_block_idx: usize, + current_insn_iter: std::vec::IntoIter<Insn>, + peeked: Option<(usize, Insn)>, + index: usize, +} + +impl InsnIter { + // We're implementing our own peek() because we don't want peek to + // cross basic blocks as we're iterating. + pub fn peek(&mut self) -> Option<&(usize, Insn)> { + // If we don't have a peeked value, get one + if self.peeked.is_none() { + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + self.peeked = Some((idx, insn)); + } + // Return a reference to the peeked value + self.peeked.as_ref() + } + + // Get the next instruction, advancing to the next block when current block is exhausted. + // Sets the current block on new_asm when moving to a new block. + pub fn next(&mut self, new_asm: &mut Assembler) -> Option<(usize, Insn)> { + // If we have a peeked value, return it + if let Some(item) = self.peeked.take() { + return Some(item); + } + + // Try to get the next instruction from current block + if let Some(insn) = self.current_insn_iter.next() { + let idx = self.index; + self.index += 1; + return Some((idx, insn)); + } + + // Current block is exhausted, move to next block + self.current_block_idx += 1; + if self.current_block_idx >= self.blocks.len() { + return None; + } + + // Set up the next block + let next_block = &mut self.blocks[self.current_block_idx]; + new_asm.set_current_block(next_block.id); + + self.current_insn_iter = take(&mut next_block.insns).into_iter(); + + // Get first instruction from the new block + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + Some((idx, insn)) + } +} + +impl Assembler { + #[must_use] + pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Add { left, right, out }); + out + } + + pub fn add_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of add_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Add { left, right, out: left }); + } + + #[must_use] + pub fn and(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::And { left, right, out }); + out + } + + pub fn bake_string(&mut self, text: &str) { + self.push_insn(Insn::BakeString(text.to_string())); + } + + pub fn is_ruby_code(&self) -> bool { + self.basic_blocks.len() > 1 || !self.basic_blocks[0].is_dummy() + } + + #[allow(dead_code)] + pub fn breakpoint(&mut self) { + self.push_insn(Insn::Breakpoint); + } + + #[allow(dead_code)] + pub fn abort(&mut self) { + self.push_insn(Insn::Abort); + } + + /// Call a C function without PosMarkers + pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { + let canary_opnd = self.set_stack_canary(); + let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + let fptr = Opnd::const_ptr(fptr); + self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + self.clear_stack_canary(canary_opnd); + out + } + + /// Call a C function into an explicit output operand without allocating a + /// new vreg for the result. + pub fn ccall_into(&mut self, out: Opnd, fptr: *const u8, opnds: Vec<Opnd>) { + let fptr = Opnd::const_ptr(fptr); + self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + } + + /// Call a C function stored in a register + pub fn ccall_reg(&mut self, fptr: Opnd, num_bits: u8) -> Opnd { + assert!(matches!(fptr, Opnd::Reg(_)), "ccall_reg must be called with Opnd::Reg: {fptr:?}"); + let out = self.new_vreg(num_bits); + self.push_insn(Insn::CCall { fptr, opnds: vec![], start_marker: None, end_marker: None, out }); + out + } + + /// Call a C function with PosMarkers. This is used for recording the start and end + /// addresses of the C call and rewriting it with a different function address later. + pub fn ccall_with_pos_markers( + &mut self, + fptr: *const u8, + opnds: Vec<Opnd>, + start_marker: impl Fn(CodePtr, &CodeBlock) + 'static, + end_marker: impl Fn(CodePtr, &CodeBlock) + 'static, + ) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + self.push_insn(Insn::CCall { + fptr: Opnd::const_ptr(fptr), + opnds, + start_marker: Some(Rc::new(start_marker)), + end_marker: Some(Rc::new(end_marker)), + out, + }); + out + } + + pub fn count_call_to(&mut self, fn_name: &str) { + // We emit ccalls while initializing the JIT. Unfortunately, we skip those because + // otherwise we have no counter pointers to read. + if crate::state::ZJITState::has_instance() && get_option!(stats) { + let ccall_counter_pointers = crate::state::ZJITState::get_ccall_counter_pointers(); + let counter_ptr = ccall_counter_pointers.entry(fn_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + self.incr_counter(Opnd::const_ptr(counter_ptr), 1.into()); + } + } + + pub fn cmp(&mut self, left: Opnd, right: Opnd) { + self.push_insn(Insn::Cmp { left, right }); + } + + #[must_use] + pub fn cpop(&mut self) -> Opnd { + let out = self.new_vreg(Opnd::DEFAULT_NUM_BITS); + self.push_insn(Insn::CPop { out }); + out + } + + pub fn cpop_into(&mut self, opnd: Opnd) { + assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); + self.push_insn(Insn::CPopInto(opnd)); + } + + #[track_caller] + pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); + } + + pub fn cpush(&mut self, opnd: Opnd) { + self.push_insn(Insn::CPush(opnd)); + } + + #[track_caller] + pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPushPair(opnd0, opnd1)); + } + + pub fn cret(&mut self, opnd: Opnd) { + self.push_insn(Insn::CRet(opnd)); + } + + #[must_use] + pub fn csel_e(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_g(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelG { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_ge(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelGE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_l(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelL { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_le(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelLE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_ne(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelNE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_nz(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelNZ { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_z(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelZ { truthy, falsy, out }); + out + } + + pub fn frame_setup(&mut self, preserved_regs: &'static [Opnd]) { + let slot_count = self.stack_base_idx; + self.push_insn(Insn::FrameSetup { preserved: preserved_regs, slot_count }); + } + + /// The inverse of [Self::frame_setup] used before return. `reserve_bytes` + /// not necessary since we use a base pointer register. + pub fn frame_teardown(&mut self, preserved_regs: &'static [Opnd]) { + self.push_insn(Insn::FrameTeardown { preserved: preserved_regs }); + } + + pub fn incr_counter(&mut self, mem: Opnd, value: Opnd) { + self.push_insn(Insn::IncrCounter { mem, value }); + } + + pub fn jb(&mut self, target: Target) { + self.push_insn(Insn::Jb(target)); + } + + #[allow(dead_code)] + pub fn jg(&mut self, target: Target) { + self.push_insn(Insn::Jg(target)); + } + + pub fn jmp(&mut self, target: Target) { + self.push_insn(Insn::Jmp(target)); + } + + pub fn jmp_opnd(&mut self, opnd: Opnd) { + self.push_insn(Insn::JmpOpnd(opnd)); + } + + + #[must_use] + pub fn lea(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Lea { opnd, out }); + out + } + + pub fn lea_into(&mut self, out: Opnd, opnd: Opnd) { + assert!(matches!(out, Opnd::Reg(_) | Opnd::Mem(_)), "Destination of lea_into must be a register or memory, got: {out:?}"); + self.push_insn(Insn::Lea { opnd, out }); + } + + #[must_use] + pub fn lea_jump_target(&mut self, target: Target) -> Opnd { + let out = self.new_vreg(Opnd::DEFAULT_NUM_BITS); + self.push_insn(Insn::LeaJumpTarget { target, out }); + out + } + + #[must_use] + pub fn load(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Load { opnd, out }); + out + } + + pub fn load_into(&mut self, dest: Opnd, opnd: Opnd) { + assert!(matches!(dest, Opnd::Reg(_)), "Destination of load_into must be a register, got: {dest:?}"); + match (dest, opnd) { + (Opnd::Reg(dest), Opnd::Reg(opnd)) if dest == opnd => {}, // skip if noop + _ => self.push_insn(Insn::LoadInto { dest, opnd }), + } + } + + #[must_use] + pub fn load_sext(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::LoadSExt { opnd, out }); + out + } + + #[must_use] + pub fn lshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::LShift { opnd, shift, out }); + out + } + + pub fn mov(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of mov must not be Opnd::VReg, got: {dest:?}"); + self.push_insn(Insn::Mov { dest, src }); + } + + #[must_use] + pub fn not(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Not { opnd, out }); + out + } + + #[must_use] + pub fn or(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Or { left, right, out }); + out + } + + pub fn patch_point(&mut self, target: Target, invariant: Invariant, version: IseqVersionRef) { + self.push_insn(Insn::PatchPoint { target, invariant, version }); + } + + pub fn pad_patch_point(&mut self) { + self.push_insn(Insn::PadPatchPoint); + } + + pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr, &CodeBlock) + 'static) { + self.push_insn(Insn::PosMarker(Rc::new(marker_fn))); + } + + #[must_use] + pub fn rshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::RShift { opnd, shift, out }); + out + } + + pub fn store(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of store must not be Opnd::VReg, got: {dest:?}"); + self.push_insn(Insn::Store { dest, src }); + } + + #[must_use] + pub fn sub(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Sub { left, right, out }); + out + } + + pub fn sub_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of sub_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Sub { left, right, out: left }); + } + + #[must_use] + pub fn mul(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Mul { left, right, out }); + out + } + + pub fn test(&mut self, left: Opnd, right: Opnd) { + self.push_insn(Insn::Test { left, right }); + } + + #[must_use] + #[allow(dead_code)] + pub fn urshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::URShift { opnd, shift, out }); + out + } + + /// Add a label at the current position + pub fn write_label(&mut self, target: Target) { + assert!(target.unwrap_label().0 < self.label_names.len()); + self.push_insn(Insn::Label(target)); + } + + #[must_use] + pub fn xor(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Xor { left, right, out }); + out + } + + /// This is used for trampolines that don't allow scratch registers. + /// Linearizes all blocks into a single giant block. + pub fn resolve_parallel_mov_pass(self) -> Assembler { + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = self.accept_scratch_reg; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // TODO: Aaron, this could be better. We don't need to do this, FIXME + // Process each linearized instruction + for insn in linearized_insns { + match insn { + Insn::Mov { dest, src } => { + if src != dest { + asm_local.push_insn(insn); + } + }, + _ => { + asm_local.push_insn(insn); + } + } + } + + asm_local + } +} + +/// Macro to use format! for Insn::Comment, which skips a format! call +/// when not dumping disassembly. +macro_rules! asm_comment { + ($asm:expr, $($fmt:tt)*) => { + // If --zjit-dump-disasm or --zjit-dump-lir is given, enrich them with comments. + // Also allow --zjit-debug on dev builds to enable comments since dev builds dump LIR on panic. + let enable_comment = $crate::options::get_option_ref!(dump_disasm).is_some() || + $crate::options::get_option!(dump_lir).is_some() || + (cfg!(debug_assertions) && $crate::options::get_option!(debug)); + if enable_comment { + $asm.push_insn(crate::backend::lir::Insn::Comment(format!($($fmt)*))); + } + }; +} +pub(crate) use asm_comment; + +/// Convenience macro over [`Assembler::ccall`] that also adds a comment with the function name. +macro_rules! asm_ccall { + [$asm: ident, $fn_name:ident, $($args:expr),* ] => {{ + $crate::backend::lir::asm_comment!($asm, concat!("call ", stringify!($fn_name))); + $asm.count_call_to(stringify!($fn_name)); + $asm.ccall($fn_name as *const u8, vec![$($args),*]) + }}; +} +pub(crate) use asm_ccall; + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + use crate::backend::current::NATIVE_STACK_PTR; + + fn scratch_reg() -> Opnd { + Assembler::new_with_scratch_reg().1 + } + + #[test] + fn test_for_each_operand() { + let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; + + let mut result = vec![]; + insn.for_each_operand(|opnd| result.push(opnd)); + assert_eq!(result, vec![Opnd::None, Opnd::None]); + } + + #[test] + fn test_for_each_operand_mut() { + let mut insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; + + let mut counter = 0; + insn.for_each_operand_mut(|opnd| { + *opnd = Opnd::Imm(counter); + counter += 1; + }); + assert!(matches!(insn, Insn::Add { left: Opnd::Imm(0), right: Opnd::Imm(1), out: Opnd::None })); + let mut result = vec![]; + insn.for_each_operand(|opnd| result.push(opnd)); + assert_eq!(result, vec![Opnd::Imm(0), Opnd::Imm(1)]); + } + + #[test] + #[should_panic] + fn load_into_memory_is_invalid() { + let mut asm = Assembler::new(); + let mem = Opnd::mem(64, SP, 0); + asm.load_into(mem, mem); + } + + #[test] + fn test_resolve_parallel_moves_reorder_registers() { + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, Some(vec![ + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + fn test_resolve_parallel_moves_give_up_register_cycle() { + // If scratch_opnd is not given, it cannot break cycles. + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, None); + } + + #[test] + fn test_resolve_parallel_moves_break_register_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_memory_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_register_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_reorder_memory_destination() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_register() { + Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[0], CFP), + ], Some(scratch_reg())); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_memory() { + Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg())); + } + + // Helper function to convert a BitSet to a list of vreg indices + fn bitset_to_vreg_indices(bitset: &BitSet<usize>, num_vregs: usize) -> Vec<usize> { + (0..num_vregs) + .filter(|&idx| bitset.get(idx)) + .collect() + } + + struct TestFunc { + asm: Assembler, + r10: Opnd, + r11: Opnd, + r12: Opnd, + r13: Opnd, + r14: Opnd, + r15: Opnd, + b1: BlockId, + b2: BlockId, + b3: BlockId, + b4: BlockId, + } + + fn build_func() -> TestFunc { + let mut asm = Assembler::new(); + + // Create virtual registers - these will be parameters + let r10 = asm.new_vreg(64); + let r11 = asm.new_vreg(64); + let r12 = asm.new_vreg(64); + let r13 = asm.new_vreg(64); + + // Create blocks + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + let b3 = asm.new_block(hir::BlockId(2), false, 2); + let b4 = asm.new_block(hir::BlockId(3), false, 3); + + // Build b1: define(r10, r11) { jump(edge(b2, [imm(1), r11])) } + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + asm.basic_blocks[b1.0].add_parameter(r10); + asm.basic_blocks[b1.0].add_parameter(r11); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![Opnd::UImm(1), r11], + }))); + + // Build b2: define(r12, r13) { cmp(r13, imm(1)); blt(...) } + asm.set_current_block(b2); + let label_b2 = asm.new_label("bb1"); + asm.write_label(label_b2); + asm.basic_blocks[b2.0].add_parameter(r12); + asm.basic_blocks[b2.0].add_parameter(r13); + asm.basic_blocks[b2.0].push_insn(Insn::Cmp { left: r13, right: Opnd::UImm(1) }); + asm.basic_blocks[b2.0].push_insn(Insn::Jl(Target::Block(BranchEdge { target: b4, args: vec![] }))); + asm.basic_blocks[b2.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![] }))); + + // Build b3: r14 = mul(r12, r13); r15 = sub(r13, imm(1)); jump(edge(b2, [r14, r15])) + asm.set_current_block(b3); + let label_b3 = asm.new_label("bb2"); + asm.write_label(label_b3); + let r14 = asm.new_vreg(64); + let r15 = asm.new_vreg(64); + asm.basic_blocks[b3.0].push_insn(Insn::Mul { left: r12, right: r13, out: r14 }); + asm.basic_blocks[b3.0].push_insn(Insn::Sub { left: r13, right: Opnd::UImm(1), out: r15 }); + asm.basic_blocks[b3.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![r14, r15], + }))); + + // Build b4: out = add(r10, r12); ret out + asm.set_current_block(b4); + let label_b4 = asm.new_label("bb3"); + asm.write_label(label_b4); + let out = asm.new_vreg(64); + asm.basic_blocks[b4.0].push_insn(Insn::Add { left: r10, right: r12, out }); + asm.basic_blocks[b4.0].push_insn(Insn::CRet(out)); + + TestFunc { asm, r10, r11, r12, r13, r14, r15, b1, b2, b3, b4 } + } + + #[test] + fn test_live_in() { + let TestFunc { asm, r10, r12, r13, b1, b2, b3, b4, .. } = build_func(); + + let num_vregs = asm.num_vregs; + let live_in = asm.analyze_liveness(); + + // b1: [] - entry block, no variables are live-in + assert_eq!(bitset_to_vreg_indices(&live_in[b1.0], num_vregs), vec![]); + + // b2: [r10] - r10 is live-in (used in b4 which is reachable) + assert_eq!(bitset_to_vreg_indices(&live_in[b2.0], num_vregs), vec![r10.vreg_idx().0]); + + // b3: [r10, r12, r13] - all are live-in + assert_eq!( + bitset_to_vreg_indices(&live_in[b3.0], num_vregs), + vec![r10.vreg_idx().0, r12.vreg_idx().0, r13.vreg_idx().0] + ); + + // b4: [r10, r12] - both are live-in + assert_eq!( + bitset_to_vreg_indices(&live_in[b4.0], num_vregs), + vec![r10.vreg_idx().0, r12.vreg_idx().0] + ); + } + + #[test] + fn test_lir_debug_output() { + let TestFunc { asm, .. } = build_func(); + + // Test the LIR string output + let output = lir_string(&asm); + + assert_snapshot!(output, @" + bb0(v0, v1): + Jmp bb1(1, v1) + bb1(v2, v3): + Cmp v3, 1 + Jl bb3 + Jmp bb2 + bb2(): + v4 = Mul v2, v3 + v5 = Sub v3, 1 + Jmp bb1(v4, v5) + bb3(): + v6 = Add v0, v2 + CRet v6 + "); + } + + #[test] + fn test_out_vregs() { + let TestFunc { asm, r11, r14, r15, b1, b2, b3, b4, .. } = build_func(); + + // b1 has one edge to b2 with args [imm(1), r11] + // Only r11 is a VReg, so we should only get that + let out_b1 = asm.basic_blocks[b1.0].out_vregs(); + assert_eq!(out_b1.len(), 1); + assert_eq!(out_b1[0], r11.vreg_idx()); + + // b2 has two edges: one to b4 (no args) and one to b3 (no args) + let out_b2 = asm.basic_blocks[b2.0].out_vregs(); + assert_eq!(out_b2.len(), 0); + + // b3 has one edge to b2 with args [r14, r15] + let out_b3 = asm.basic_blocks[b3.0].out_vregs(); + assert_eq!(out_b3.len(), 2); + assert_eq!(out_b3[0], r14.vreg_idx()); + assert_eq!(out_b3[1], r15.vreg_idx()); + + // b4 has no edges (terminates with CRet) + let out_b4 = asm.basic_blocks[b4.0].out_vregs(); + assert_eq!(out_b4.len(), 0); + } + + #[test] + fn test_out_vregs_includes_memory_base_vregs() { + let mut asm = Assembler::new(); + + let base = asm.new_vreg(64); + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![Opnd::mem(64, base, 8)], + }))); + + let out_vregs = asm.basic_blocks[b1.0].out_vregs(); + assert_eq!(out_vregs, vec![base.vreg_idx()]); + } + + #[test] + fn test_interval_add_range() { + let mut interval = Interval::new(1); + + // Add range to empty interval + interval.add_range(5, 10); + assert_eq!(interval.range.start, Some(5)); + assert_eq!(interval.range.end, Some(10)); + + // Extend range backward + interval.add_range(3, 7); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(10)); + + // Extend range forward + interval.add_range(8, 15); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(15)); + } + + #[test] + fn test_interval_survives() { + let mut interval = Interval::new(1); + interval.add_range(3, 10); + + assert!(!interval.survives(2)); // Before range + assert!(!interval.survives(3)); // At start (exclusive) + assert!(interval.survives(5)); // Inside range + assert!(!interval.survives(10)); // At end (exclusive) + assert!(!interval.survives(11)); // After range + } + + #[test] + fn test_interval_set_from() { + let mut interval = Interval::new(1); + + // With no range, sets both start and end + interval.set_from(10); + assert_eq!(interval.range.start, Some(10)); + assert_eq!(interval.range.end, Some(10)); + + // With existing range, updates start but keeps end + interval.add_range(5, 20); + interval.set_from(3); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(20)); + } + + #[test] + #[should_panic(expected = "Invalid range")] + fn test_interval_add_range_invalid() { + let mut interval = Interval::new(1); + interval.add_range(10, 5); + } + + #[test] + #[should_panic(expected = "survives called on interval with no range")] + fn test_interval_survives_panics_without_range() { + let interval = Interval::new(1); + interval.survives(5); + } + + #[test] + fn test_build_intervals() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + // Analyze liveness + let live_in = asm.analyze_liveness(); + + // Number instructions (starting from 16 to match Ruby test) + asm.number_instructions(16); + + // Build intervals + let intervals = asm.build_intervals(live_in); + + // Extract vreg indices + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + // Assert expected ranges + // Note: Rust CFG differs from Ruby due to conditional branches requiring two instructions (Jl + Jmp) + assert_eq!(intervals[r10_idx.0].range.start, Some(16)); + assert_eq!(intervals[r10_idx.0].range.end, Some(38)); + + assert_eq!(intervals[r11_idx.0].range.start, Some(16)); + assert_eq!(intervals[r11_idx.0].range.end, Some(20)); + + assert_eq!(intervals[r12_idx.0].range.start, Some(20)); + assert_eq!(intervals[r12_idx.0].range.end, Some(38)); + + assert_eq!(intervals[r13_idx.0].range.start, Some(20)); + assert_eq!(intervals[r13_idx.0].range.end, Some(32)); + + assert_eq!(intervals[r14_idx.0].range.start, Some(30)); + assert_eq!(intervals[r14_idx.0].range.end, Some(36)); + + assert_eq!(intervals[r15_idx.0].range.start, Some(32)); + assert_eq!(intervals[r15_idx.0].range.end, Some(36)); + } + + #[test] + fn test_linear_scan_no_spill() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + // Analyze liveness + let live_in = asm.analyze_liveness(); + + // Number instructions (starting from 16 to match Ruby test) + asm.number_instructions(16); + + // Build intervals + let intervals = asm.build_intervals(live_in); + + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 5, &preferred_registers); + + // Extract vreg indices + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + // 5 registers is enough for all intervals, no spills needed + assert_eq!(num_stack_slots, 0); + + // Verify register assignments + // r10: [16,42) gets Reg(0) (first allocated) + // r11: [16,20) gets Reg(1) + // r12: [20,36) gets Reg(1) (r11 expired, reuses its register) + // r13: [20,38) gets Reg(2) + // r14: [36,42) gets Reg(1) (r12 expired, reuses its register) + // r15: [38,42) gets Reg(2) (r13 expired, reuses its register) + assert_eq!(assignments[r10_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(2))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Reg(3))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(2))); + } + + #[test] + fn test_linear_scan_spill_less() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + + // 3 registers -- only r10 needs to spill + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 3, &preferred_registers); + + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + assert_eq!(num_stack_slots, 1); + assert_eq!(assignments[r10_idx.0], Some(Allocation::Stack(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(2))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(2))); + } + + #[test] + fn test_linear_scan_spill() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + + // Only 1 register available -- forces spills + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 1, &preferred_registers); + + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + assert_eq!(num_stack_slots, 3); + assert_eq!(assignments[r10_idx.0], Some(Allocation::Stack(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Stack(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Stack(2))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(0))); + } + + #[test] + fn test_preferred_register_assignment_for_newborn_mov_source() { + let mut asm = Assembler::new(); + let block = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(block); + let label = asm.new_label("bb0"); + asm.write_label(label); + + let sp = NATIVE_STACK_PTR; + let new_sp = asm.add(sp, 0x20.into()); + asm.mov(sp, new_sp); + asm.cret(sp); + + asm.number_instructions(0); + let live_in = asm.analyze_liveness(); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + + let vreg_idx = new_sp.vreg_idx(); + assert_eq!(preferred_registers[vreg_idx.0], Some(sp.unwrap_reg())); + + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 0, &preferred_registers); + assert_eq!(num_stack_slots, 0); + assert_eq!(assignments[vreg_idx.0], Some(Allocation::Fixed(sp.unwrap_reg()))); + } + + #[test] + fn test_debug_intervals() { + let TestFunc { mut asm, .. } = build_func(); + + // Number instructions + asm.number_instructions(16); + + // Get the debug output + let live_in = asm.analyze_liveness(); + let intervals = asm.build_intervals(live_in); + let output = debug_intervals(&asm, &intervals); + + // Verify it contains the grid structure + assert!(output.contains("v0")); // Header with vreg names + assert!(output.contains("---")); // Separator + assert!(output.contains("█")); // Live marker + assert!(output.contains(".")); // Dead marker + } + + #[test] + fn test_resolve_ssa() { + let TestFunc { mut asm, b1, b3, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + asm.resolve_ssa(&intervals, &assignments); + + use crate::backend::current::ALLOC_REGS; + let regs = &ALLOC_REGS[..5]; + + // Edge b1->b2 (single succ): args=[UImm(1), v1], params=[v2, v3] + // v1->Reg(1), v2->Reg(1), v3->Reg(2) + // Reg copy: Reg(1)->Reg(2) -> Mov(regs[2], regs[1]) + // Imm move: Mov(regs[1], UImm(1)) + // Inserted in b1 before Jmp: [Label, Mov, Mov, Jmp] + let b1_insns = &asm.basic_blocks[b1.0].insns; + assert_eq!(b1_insns.len(), 4); + assert!(matches!(&b1_insns[1], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[2]) && *src == Opnd::Reg(regs[1]))); + assert!(matches!(&b1_insns[2], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[1]) && *src == Opnd::UImm(1))); + + // Edge b3->b2 (single succ): args=[v4, v5], params=[v2, v3] + // v4->Reg(3), v5->Reg(2), v2->Reg(1), v3->Reg(2) + // Reg copy: Reg(3)->Reg(1) -> Mov(regs[1], regs[3]) + // Reg(2)->Reg(2) is self-move, filtered + // Inserted in b3 before Jmp: [Label, Mul, Sub, Mov, Jmp] + let b3_insns = &asm.basic_blocks[b3.0].insns; + assert_eq!(b3_insns.len(), 5); + assert!(matches!(&b3_insns[3], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[1]) && *src == Opnd::Reg(regs[3]))); + + // Verify original instructions in b3 are rewritten to physical registers. + // b3: Mul { left: r12, right: r13, out: r14 }, Sub { left: r13, right: UImm(1), out: r15 } + // r12->Reg(1), r13->Reg(2), r14->Reg(3), r15->Reg(2) + assert!(matches!(&b3_insns[1], Insn::Mul { left, right, out } + if *left == Opnd::Reg(regs[1]) && *right == Opnd::Reg(regs[2]) && *out == Opnd::Reg(regs[3]))); + assert!(matches!(&b3_insns[2], Insn::Sub { left, right, out } + if *left == Opnd::Reg(regs[2]) && *right == Opnd::UImm(1) && *out == Opnd::Reg(regs[2]))); + } + + #[test] + fn test_resolve_ssa_entry_params() { + let TestFunc { mut asm, b1, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + // Entry block b1 has parameters [v0, v1]. + // With 5 registers: v0 -> Reg(0) = regs[0], arrival = C_ARG_OPNDS[0] = regs[0] -> self-move, filtered + // v1 -> Reg(1) = regs[1], arrival = C_ARG_OPNDS[1] = regs[1] -> self-move, filtered + // Before resolve_ssa, b1 has: [Label, Jmp] = 2 insns + assert_eq!(asm.basic_blocks[b1.0].insns.len(), 2); + + asm.resolve_ssa(&intervals, &assignments); + + // After resolve_ssa, b1 should still have the same number of insns + // (plus any edge moves, but no entry param moves since they're all self-moves). + // Edge b1->b2 inserts 2 moves before Jmp: [Label, Mov, Mov, Jmp] = 4 insns + // No additional entry param moves. + let b1_insns = &asm.basic_blocks[b1.0].insns; + assert_eq!(b1_insns.len(), 4); + // Verify the moves are edge moves (not entry param moves) + assert!(matches!(&b1_insns[1], Insn::Mov { .. })); + assert!(matches!(&b1_insns[2], Insn::Mov { .. })); + + // After resolve_ssa, edge args are cleared since the moves have been + // materialized as explicit Mov instructions. + if let Insn::Jmp(Target::Block(edge)) = &b1_insns[3] { + assert!(edge.args.is_empty(), "Edge args should be cleared after resolve_ssa"); + } else { + panic!("Expected Jmp at end of b1"); + } + } + + #[test] + fn test_resolve_ssa_entry_params_too_many_abort() { + let mut asm = Assembler::new(); + let block = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(block); + let label = asm.new_label("bb0"); + asm.write_label(label); + + for _ in 0..=C_ARG_OPNDS.len() { + let param = asm.new_vreg(64); + asm.basic_blocks[block.0].add_parameter(param); + } + asm.basic_blocks[block.0].push_insn(Insn::CRet(Opnd::UImm(0))); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(0); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + asm.resolve_ssa(&intervals, &assignments); + + assert!(matches!(asm.basic_blocks[block.0].insns[1], Insn::Abort)); + } + + fn build_critical_edge() -> (Assembler, Opnd, Opnd, Opnd, Opnd, Opnd, BlockId, BlockId, BlockId) { + let mut asm = Assembler::new(); + + // Create blocks + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + let b3 = asm.new_block(hir::BlockId(2), false, 2); + + // b1: v0 = Add(123, 0), v1 = Add(v0, 456), Cmp(v1, 0), Jl(b2, [v0]), Jmp(b3, [v1]) + // v0 is live across b1->b2 edge AND v1 is live across b1->b3 edge + // This forces v0 and v1 to have overlapping live ranges -> different registers + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + let v0 = asm.new_vreg(64); + let v1 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: Opnd::UImm(123), right: Opnd::UImm(0), out: v0 }); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v0, right: Opnd::UImm(456), out: v1 }); + asm.basic_blocks[b1.0].push_insn(Insn::Cmp { left: v1, right: Opnd::UImm(0) }); + asm.basic_blocks[b1.0].push_insn(Insn::Jl(Target::Block(BranchEdge { target: b2, args: vec![v0] }))); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![v1] }))); + + // b2(v2): v3 = Add(v2, 789), Jmp(b3, [v3]) + asm.set_current_block(b2); + let label_b2 = asm.new_label("bb1"); + asm.write_label(label_b2); + let v2 = asm.new_block_param(64); + asm.basic_blocks[b2.0].add_parameter(v2); + let v3 = asm.new_vreg(64); + asm.basic_blocks[b2.0].push_insn(Insn::Add { left: v2, right: Opnd::UImm(789), out: v3 }); + asm.basic_blocks[b2.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![v3] }))); + + // b3(v4): CRet(v4) + asm.set_current_block(b3); + let label_b3 = asm.new_label("bb2"); + asm.write_label(label_b3); + let v4 = asm.new_block_param(64); + asm.basic_blocks[b3.0].add_parameter(v4); + asm.basic_blocks[b3.0].push_insn(Insn::CRet(v4)); + + (asm, v0, v1, v2, v3, v4, b1, b2, b3) + } + + #[test] + fn test_resolve_critical_edge() { + let (mut asm, _v0, v1, _v2, v3, v4, b1, b2, b3) = build_critical_edge(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let num_regs = 5; + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), num_regs, &preferred_registers); + + assert_eq!(asm.basic_blocks.len(), 3); + + // Verify v1 and v4 have different allocations (so moves are needed) + let v1_alloc = assignments[v1.vreg_idx().0].unwrap(); + let v4_alloc = assignments[v4.vreg_idx().0].unwrap(); + assert_ne!(v1_alloc, v4_alloc, "Test setup: v1 and v4 should have different allocations"); + + asm.resolve_ssa(&intervals, &assignments); + + // A new interstitial block should have been created for the critical edge b1->b3 + // b1->b3 is critical because b1 has 2 successors and b3 has 2 predecessors + assert_eq!(asm.basic_blocks.len(), 4); + let split_block_id = BlockId(3); + + // b1's Jmp should now target the split block instead of b3 + let b1_insns = &asm.basic_blocks[b1.0].insns; + let last_insn = b1_insns.last().unwrap(); + if let Insn::Jmp(Target::Block(edge)) = last_insn { + assert_eq!(edge.target, split_block_id); + } else { + panic!("Expected Jmp at end of b1"); + } + + // The split block should contain: Label, Mov(s), Jmp(b3) + let split_insns = &asm.basic_blocks[split_block_id.0].insns; + assert!(matches!(&split_insns[0], Insn::Label(_))); + let split_last = split_insns.last().unwrap(); + if let Insn::Jmp(Target::Block(edge)) = split_last { + assert_eq!(edge.target, b3); + assert!(edge.args.is_empty()); + } else { + panic!("Expected Jmp(b3) at end of split block"); + } + + // The split block should have a Mov for v1->v4 + let has_mov = split_insns.iter().any(|insn| matches!(insn, Insn::Mov { .. })); + assert!(has_mov, "Expected Mov in split block for v1->v4"); + + // b2->b3 is not a critical edge (b2 has single succ), so moves go before Jmp in b2 + let v3_alloc = assignments[v3.vreg_idx().0].unwrap(); + let b2_insns = &asm.basic_blocks[b2.0].insns; + if v3_alloc != v4_alloc { + // Check that a Mov was inserted before the Jmp in b2 + let second_last = &b2_insns[b2_insns.len() - 2]; + assert!(matches!(second_last, Insn::Mov { .. }), "Expected Mov before Jmp in b2"); + } + } + + #[test] + fn test_call() { + use crate::backend::current::ALLOC_REGS; + + let mut asm = Assembler::new(); + + // Single entry block + let b1 = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(b1); + let label = asm.new_label("bb0"); + asm.write_label(label); + + // v0 = param (entry block parameter) + let v0 = asm.new_block_param(64); + asm.basic_blocks[b1.0].add_parameter(v0); + + // v1 = Load(UImm(5)) + let v1 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Load { opnd: Opnd::UImm(5), out: v1 }); + + // v2 = Add(v1, UImm(1)) + let v2 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v1, right: Opnd::UImm(1), out: v2 }); + + // v3 = CCall { fptr: UImm(0xF00), opnds: [v2] } + let v3 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::CCall { + opnds: vec![v2], + fptr: Opnd::UImm(0xF00), + start_marker: None, + end_marker: None, + out: v3, + }); + + // v4 = Add(v3, v1) + let v4 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v3, right: v1, out: v4 }); + + // v5 = Add(v0, v4) + let v5 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v0, right: v4, out: v5 }); + + // CRet(v5) + asm.basic_blocks[b1.0].push_insn(Insn::CRet(v5)); + + // Run liveness + numbering + intervals + linear scan with 2 registers + let live_in = asm.analyze_liveness(); + asm.number_instructions(0); + let intervals = asm.build_intervals(live_in); + let num_regs = 2; + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), num_regs, &preferred_registers); + + let regs = &ALLOC_REGS[..num_regs]; + + // v0 should be spilled (long-lived, only 2 regs) + assert!(matches!(assignments[v0.vreg_idx().0], Some(Allocation::Stack(_))), + "v0 should be spilled to stack"); + // v1 should be in a register + assert!(matches!(assignments[v1.vreg_idx().0], Some(Allocation::Reg(_))), + "v1 should be in a register"); + + // Run the pipeline: handle_caller_saved_regs then resolve_ssa + asm.handle_caller_saved_regs(&intervals, &assignments, regs); + asm.resolve_ssa(&intervals, &assignments); + + let insns = &asm.basic_blocks[b1.0].insns; + + // Find CPush and CPopInto - they should be balanced. + let pushes: Vec<_> = insns.iter().filter(|i| matches!(i, Insn::CPushPair(..))).collect(); + let pops: Vec<_> = insns.iter().filter(|i| matches!(i, Insn::CPopPairInto(..))).collect(); + assert_eq!(pushes.len(), pops.len(), "CPush/CPopInto should be balanced"); + assert!(!pushes.is_empty(), "Expected at least one saved register across CCall"); + + // The survivor register should match v1's allocation + let v1_reg = match assignments[v1.vreg_idx().0].unwrap() { + Allocation::Reg(n) => Opnd::Reg(regs[n]), + Allocation::Fixed(reg) => Opnd::Reg(reg), + _ => unreachable!(), + }; + let pushed_v1 = pushes.iter().any(|insn| matches!(**insn, Insn::CPushPair(first, second) if first == v1_reg || second == v1_reg)); + let popped_v1 = pops.iter().any(|insn| matches!(**insn, Insn::CPopPairInto(first, second) if first == v1_reg || second == v1_reg)); + assert!(pushed_v1, "CPushPair should save v1's register"); + assert!(popped_v1, "CPopPairInto should restore v1's register"); + + // The CCall should have empty opnds and out = C_RET_OPND (rewritten to regs[0]) + let ccall = insns.iter().find(|i| matches!(i, Insn::CCall { .. })).unwrap(); + if let Insn::CCall { opnds, .. } = ccall { + assert!(opnds.is_empty(), "CCall opnds should be empty after handle_caller_saved_regs"); + } + + // v0 should be rewritten to a Stack operand + // Find an Add that uses a Stack operand (the v0+v4 add) + let has_stack_opnd = insns.iter().any(|i| { + if let Insn::Add { left: Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }), .. } = i { + true + } else { + false + } + }); + assert!(has_stack_opnd, "v0 should be rewritten to a Stack memory operand"); + } +} diff --git a/zjit/src/backend/mod.rs b/zjit/src/backend/mod.rs new file mode 100644 index 0000000000..f9a7e60a6b --- /dev/null +++ b/zjit/src/backend/mod.rs @@ -0,0 +1,19 @@ +//! A multi-platform assembler generation backend. + +#[cfg(target_arch = "x86_64")] +pub mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub mod arm64; + +#[cfg(target_arch = "x86_64")] +pub use x86_64 as current; + +#[cfg(target_arch = "aarch64")] +pub use arm64 as current; + +#[cfg(test)] +mod tests; + +pub mod lir; +pub mod parcopy; diff --git a/zjit/src/backend/parcopy.rs b/zjit/src/backend/parcopy.rs new file mode 100644 index 0000000000..614bf68dcd --- /dev/null +++ b/zjit/src/backend/parcopy.rs @@ -0,0 +1,368 @@ +// This file came from here: https://github.com/bboissin/thesis_bboissin/blob/main/src/algorithm13.rs +// +// Copyright (c) 2025 bboissin +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// It's also Apache-2.0 licensed +use std::hash::Hash; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct Register(pub u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct RegisterCopy<T> { + pub source: T, + pub destination: T, +} + +// Algorithm 13 in Boissin's thesis: parallel copy sequentialization +// +// Takes a list of parallel copies, return a list of sequential copy operations +// such that each output register contains the same value as if the copies were +// parallel. +// The `spare` register may be used to break cycles and should not be contained +// in `parallel_copies`. The value of `spare` is undefined after the function +// returns. +// +// Varies slightly from the original algorithm as it splits the copies between +// pending and available to reduce state tracking. +pub fn sequentialize_register<T: PartialEq + Eq + Hash + Ord + std::fmt::Debug + Clone + Copy>(parallel_copies: &[RegisterCopy<T>], spare: T) -> Vec<RegisterCopy<T>> { + let mut sequentialized = Vec::new(); + // `resource` in the original code, this point to the current register + // holding a particular initial value. + // If a given Register is no longer needed, the value might be inaccurate. + let mut current_holder = std::collections::HashMap::new(); + // Copies that are pending, indexed by destination register. + // Use btree map to stay deterministic. + let mut pending = std::collections::BTreeMap::new(); + // If a copy can be materialized (nothing depends on the destination), we + // move it from pending into available. + let mut available = Vec::new(); + + for copy in parallel_copies { + if copy.source == spare || copy.destination == spare { + panic!("Spare register cannot be a source or destination of a copy"); + } + if let Some(_old_value) = pending.insert(copy.destination, copy) { + panic!( + "Destination register {:?} has multiple copies.", + copy.destination + ); + } + current_holder.insert(copy.source, copy.source); + } + for copy in parallel_copies { + // If we didn't record it, this means nothing depends on that register. + if !current_holder.contains_key(©.destination) { + pending.remove(©.destination); + available.push(copy); + } + } + while !pending.is_empty() || !available.is_empty() { + while let Some(copy) = available.pop() { + if let Some(source) = current_holder.get_mut(©.source) { + // Materialize the copy. + sequentialized.push(RegisterCopy { + source: source.clone(), + destination: copy.destination, + }); + if let Some(available_copy) = pending.remove(source) { + available.push(available_copy); + // Point to the new destination. + *source = copy.destination; + } else if *source == spare { + // Also point to new destination if we were copying from a + // spare, this lets us reuse spare for the next cycle. + *source = copy.destination; + } + } else { + panic!("No holder for source register {:?}", copy.source); + } + } + // If we have anything left, break the cycle by using the spare register + // on the first pending entry. + if let Some((destination, copy)) = pending.iter().next() { + sequentialized.push(RegisterCopy { + source: copy.destination, + destination: spare, + }); + current_holder.insert(copy.destination, spare); + available.push(copy); + let to_remove = *destination; + pending.remove(&to_remove); + } else { + // nothing pending. + break; + } + } + sequentialized +} + +#[cfg(test)] +mod tests { + use rand::RngExt; + use std::collections::HashMap; + + use super::*; + + // Assumes that each register initially contains the value matching its id. + fn execute_sequential(copies: &[RegisterCopy<Register>]) -> HashMap<Register, u32> { + let mut register_values = HashMap::new(); + // Initialize registers with their own ids as values. + for copy in copies { + register_values.insert(copy.source, copy.source.0); + } + for copy in copies { + let source_value = *register_values.get(©.source).unwrap(); + register_values.insert(copy.destination, source_value); + } + register_values + } + + fn execute_parallel(copies: &[RegisterCopy<Register>]) -> HashMap<Register, u32> { + let mut register_values = HashMap::new(); + // Initialize registers with their own ids as values. + for copy in copies { + register_values.insert(copy.source, copy.source.0); + } + // Execute copies. + for copy in copies { + register_values.insert(copy.destination, copy.source.0); + } + register_values + } + + #[test] + fn test_execute_sequential() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(3), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(4), + }, + RegisterCopy { + source: Register(2), + destination: Register(1), + }, + RegisterCopy { + source: Register(5), + destination: Register(3), + }, + ]; + let result = execute_sequential(&copies); + let expected: HashMap<Register, u32> = vec![ + (Register(1), 3), + (Register(2), 3), + (Register(3), 5), + (Register(4), 3), + (Register(5), 5), + ] + .into_iter() + .collect(); + assert_eq!(result, expected); + } + + #[test] + fn test_execute_sequential_2() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(4), + }, + RegisterCopy { + source: Register(3), + destination: Register(1), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + ]; + let result = execute_sequential(&copies); + assert_eq!( + result, + Vec::from_iter([ + (Register(1), 3), + (Register(2), 3), + (Register(3), 2), + (Register(4), 1), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_register_simple() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(3), + destination: Register(4), + }, + ]; + + let spare = Register(5); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([ + (Register(1), 1), + (Register(2), 1), + (Register(3), 2), + (Register(4), 3), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_cycle() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(3), + destination: Register(1), + }, + ]; + let spare = Register(4); + let result = sequentialize_register(&copies, spare); + let mut sequential_result = execute_sequential(&result); + assert!(matches!(sequential_result.remove(&spare), Some(_))); + assert_eq!( + sequential_result, + Vec::from_iter([(Register(2), 1), (Register(3), 2), (Register(1), 3),]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_no_pending() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(3), + destination: Register(4), + }, + ]; + let spare = Register(5); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([ + (Register(1), 1), + (Register(2), 1), + (Register(3), 3), + (Register(4), 3), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_with_fanin() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(1), + destination: Register(3), + }, + RegisterCopy { + source: Register(2), + destination: Register(1), + }, + ]; + let spare = Register(4); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([(Register(2), 1), (Register(3), 1), (Register(1), 2)]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_rand() { + let mut rng = rand::rng(); + for _ in 0..1000 { + let num_copies = 100; + let mut copies = Vec::new(); + for i in 0..num_copies { + let dest = Register(i); + let src = Register(rng.random_range(0..num_copies)); + if src == dest { + continue; // Skip self-copies + } + copies.push(RegisterCopy { + source: src, + destination: dest, + }); + } + // shuffle the copies. + use rand::seq::SliceRandom; + + copies.shuffle(&mut rng); + let spare = Register(num_copies); + let result = sequentialize_register(&copies, spare); + let mut sequential_result = execute_sequential(&result); + // remove the spare register from the result. + sequential_result.remove(&spare); + assert_eq!(sequential_result, execute_parallel(&copies)); + } + } +} diff --git a/zjit/src/backend/tests.rs b/zjit/src/backend/tests.rs new file mode 100644 index 0000000000..7174ac4c80 --- /dev/null +++ b/zjit/src/backend/tests.rs @@ -0,0 +1,261 @@ +use crate::asm::CodeBlock; +use crate::backend::lir::*; +use crate::cruby::*; +use crate::codegen::c_callable; +use crate::options::rb_zjit_prepare_options; + +#[test] +fn test_add() { + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + let out = asm.add(SP, Opnd::UImm(1)); + let _ = asm.add(out, Opnd::UImm(2)); +} + +fn setup_asm() -> (Assembler, CodeBlock) { + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) +} + +// Test full codegen pipeline +#[test] +fn test_compile() +{ + let (mut asm, mut cb) = setup_asm(); + let regs = Assembler::get_alloc_regs(); + + let out = asm.add(Opnd::Reg(regs[0]), Opnd::UImm(2)); + let out2 = asm.add(out, Opnd::UImm(2)); + asm.store(Opnd::mem(64, SP, 0), out2); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test memory-to-memory move +#[test] +fn test_mov_mem2mem() +{ + let (mut asm, mut cb) = setup_asm(); + + asm_comment!(asm, "check that comments work too"); + asm.mov(Opnd::mem(64, SP, 0), Opnd::mem(64, SP, 8)); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test load of register into new register +#[test] +fn test_load_reg() +{ + let (mut asm, mut cb) = setup_asm(); + + let out = asm.load(SP); + asm.mov(Opnd::mem(64, SP, 0), out); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test load of a GC'd value +#[test] +fn test_load_value() +{ + let (mut asm, mut cb) = setup_asm(); + + let gcd_value = VALUE(0xFFFFFFFFFFFF00); + assert!(!gcd_value.special_const_p()); + + let out = asm.load(Opnd::Value(gcd_value)); + asm.mov(Opnd::mem(64, SP, 0), out); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Multiple registers needed and register reuse +#[test] +fn test_reuse_reg() +{ + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.add(Opnd::mem(64, SP, 0), Opnd::UImm(1)); + let v1 = asm.add(Opnd::mem(64, SP, 8), Opnd::UImm(1)); + + let v2 = asm.add(v1, Opnd::UImm(1)); // Reuse v1 register + let v3 = asm.add(v0, v2); + + asm.store(Opnd::mem(64, SP, 0), v2); + asm.store(Opnd::mem(64, SP, 8), v3); + + asm.compile_with_num_regs(&mut cb, 2); +} + +// 64-bit values can't be written directly to memory, +// need to be split into one or more register movs first +#[test] +fn test_store_u64() +{ + let (mut asm, mut cb) = setup_asm(); + asm.store(Opnd::mem(64, SP, 0), u64::MAX.into()); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Use instruction output as base register for memory operand +#[test] +fn test_base_insn_out() +{ + let (mut asm, mut cb) = setup_asm(); + + // Forced register to be reused + // This also causes the insn sequence to change length + asm.mov( + Opnd::mem(64, SP, 8), + Opnd::mem(64, SP, 0) + ); + + // Load the pointer into a register + let ptr_opnd = Opnd::const_ptr(4351776248 as *const u8); + + // Increment and store the updated value + asm.incr_counter(ptr_opnd, 1.into()); + + asm.compile_with_num_regs(&mut cb, 2); +} + +#[test] +fn test_c_call() +{ + c_callable! { + fn dummy_c_fun(_v0: usize, _v1: usize) {} + } + + let (mut asm, mut cb) = setup_asm(); + + let ret_val = asm.ccall( + dummy_c_fun as *const u8, + vec![Opnd::mem(64, SP, 0), Opnd::UImm(1)] + ); + + // Make sure that the call's return value is usable + asm.mov(Opnd::mem(64, SP, 0), ret_val); + + asm.compile(&mut cb).unwrap(); +} + +#[test] +fn test_alloc_ccall_regs() { + let (mut asm, mut cb) = setup_asm(); + let out1 = asm.ccall(std::ptr::null::<u8>(), vec![]); + let out2 = asm.ccall(std::ptr::null::<u8>(), vec![out1]); + asm.mov(EC, out2); + asm.compile_with_regs(&mut cb, Assembler::get_alloc_regs()).unwrap(); +} + +#[test] +fn test_lea_ret() +{ + let (mut asm, mut cb) = setup_asm(); + + let addr = asm.lea(Opnd::mem(64, SP, 0)); + asm.cret(addr); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_jcc_label() +{ + let (mut asm, mut cb) = setup_asm(); + + let label = asm.new_label("foo"); + asm.cmp(EC, EC); + asm.push_insn(Insn::Je(label.clone())); + asm.write_label(label); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_jcc_ptr() +{ + let (mut asm, mut cb) = setup_asm(); + + let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32)); + asm.test( + Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32), + not_mask, + ); + asm.push_insn(Insn::Jnz(side_exit)); + + asm.compile_with_num_regs(&mut cb, 2); +} + +/// Direct jump to a stub e.g. for deferred compilation +#[test] +fn test_jmp_ptr() +{ + let (mut asm, mut cb) = setup_asm(); + + let stub = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + asm.jmp(stub); + + asm.compile_with_num_regs(&mut cb, 0); +} + +#[test] +fn test_jo() +{ + let (mut asm, mut cb) = setup_asm(); + + let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + + let arg1 = Opnd::mem(64, SP, 0); + let arg0 = Opnd::mem(64, SP, 8); + + let arg0_untag = asm.sub(arg0, Opnd::Imm(1)); + let out_val = asm.add(arg0_untag, arg1); + asm.push_insn(Insn::Jo(side_exit)); + + asm.mov(Opnd::mem(64, SP, 0), out_val); + + asm.compile_with_num_regs(&mut cb, 2); +} + +#[test] +fn test_bake_string() { + let (mut asm, mut cb) = setup_asm(); + + asm.bake_string("Hello, world!"); + asm.compile_with_num_regs(&mut cb, 0); +} + +#[test] +fn test_cmp_8_bit() { + let (mut asm, mut cb) = setup_asm(); + let reg = Assembler::get_alloc_regs()[0]; + asm.cmp(Opnd::Reg(reg).with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_no_pos_marker_callback_when_compile_fails() { + // When compilation fails (e.g. when out of memory), the code written out is malformed. + // We don't want to invoke the pos_marker callbacks with positions of malformed code. + let mut asm = Assembler::new(); + rb_zjit_prepare_options(); // for asm.compile + asm.new_block_without_id("test"); + + // Markers around code to exhaust memory limit + let fail_if_called = |_code_ptr, _cb: &_| panic!("pos_marker callback should not be called"); + asm.pos_marker(fail_if_called); + let zero = asm.load(0.into()); + let sum = asm.add(zero, 500.into()); + asm.store(Opnd::mem(64, SP, 8), sum); + asm.pos_marker(fail_if_called); + + let cb = &mut CodeBlock::new_dummy_sized(8); + assert!(asm.compile(cb).is_err(), "should fail due to tiny size limit"); +} diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs new file mode 100644 index 0000000000..d3bf847ab2 --- /dev/null +++ b/zjit/src/backend/x86_64/mod.rs @@ -0,0 +1,2461 @@ +use std::mem; + +use crate::asm::*; +use crate::asm::x86_64::*; +use crate::codegen::split_patch_point; +use crate::stats::{CompileError, trace_compile_phase}; +use crate::virtualmem::CodePtr; +use crate::cruby::*; +use crate::backend::lir::*; +use crate::cast::*; +use crate::options::asm_dump; + +// Use the x86 register type for this platform +pub type Reg = X86Reg; + +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit GP register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_type: RegType::GP, reg_no } +} + +// Callee-saved registers +pub const CFP: Opnd = Opnd::Reg(R13_REG); +pub const EC: Opnd = Opnd::Reg(R12_REG); +pub const SP: Opnd = Opnd::Reg(RBX_REG); + +// C argument registers on this platform +pub const C_ARG_OPNDS: [Opnd; 6] = [ + Opnd::Reg(RDI_REG), + Opnd::Reg(RSI_REG), + Opnd::Reg(RDX_REG), + Opnd::Reg(RCX_REG), + Opnd::Reg(R8_REG), + Opnd::Reg(R9_REG) +]; +pub const C_ARG_REGREGS: [Reg; 6] = [RDI_REG, RSI_REG, RDX_REG, RCX_REG, R8_REG, R9_REG]; + +// C return value register on this platform +pub const C_RET_REG: Reg = RAX_REG; +pub const C_RET_OPND: Opnd = Opnd::Reg(RAX_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(RSP_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(RBP_REG); + +impl CodeBlock { + // The number of bytes that are generated by jmp_ptr + pub fn jmp_ptr_bytes(&self) -> usize { 5 } +} + +/// Map Opnd to X86Opnd +impl From<Opnd> for X86Opnd { + fn from(opnd: Opnd) -> Self { + match opnd { + // NOTE: these operand types need to be lowered first + //Value(VALUE), // Immediate Ruby value, may be GC'd, movable + //VReg(usize), // Output of a preceding instruction in this block + + Opnd::VReg{..} => panic!("VReg operand made it past register allocation"), + + Opnd::UImm(val) => uimm_opnd(val), + Opnd::Imm(val) => imm_opnd(val), + Opnd::Value(VALUE(uimm)) => uimm_opnd(uimm as u64), + + // General-purpose register + Opnd::Reg(reg) => X86Opnd::Reg(reg), + + // Memory operand with displacement + Opnd::Mem(Mem{ base: MemBase::Reg(reg_no), num_bits, disp }) => { + let reg = X86Reg { + reg_no, + num_bits: 64, + reg_type: RegType::GP + }; + + mem_opnd(num_bits, X86Opnd::Reg(reg), disp) + } + + Opnd::None => panic!( + "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." + ), + + _ => panic!("unsupported x86 operand type: {opnd:?}") + } + } +} + +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for X86Opnd { + fn from(opnd: &Opnd) -> Self { + X86Opnd::from(*opnd) + } +} + +/// List of registers that can be used for register allocation. +/// This has the same number of registers for x86_64 and arm64. +/// SCRATCH0_OPND is excluded. +pub const ALLOC_REGS: &[Reg] = &[ + RDI_REG, + RSI_REG, + RDX_REG, + RCX_REG, + R8_REG, + R9_REG, + RAX_REG, +]; + +/// Special scratch register for intermediate processing. It should be used only by +/// [`Assembler::x86_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(R11_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(R10_REG); + +/// A scratch register available for use by resolve_ssa to break register copy cycles. +/// Must not overlap with ALLOC_REGS or other preserved registers. +pub const SCRATCH_REG: Reg = R11_REG; + +impl Assembler { + // This keeps frame growth below the +/-4096-byte displacement range we rely + // on for common stack-slot accesses on x86_64. + const MAX_FRAME_STACK_SLOTS: usize = 2048; + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } + + /// Get the list of registers from which we can allocate on this platform + pub fn get_alloc_regs() -> Vec<Reg> { + ALLOC_REGS.to_vec() + } + + /// Get a list of all of the caller-save registers + pub fn get_caller_save_regs() -> Vec<Reg> { + vec![RAX_REG, RCX_REG, RDX_REG, RSI_REG, RDI_REG, R8_REG, R9_REG, R10_REG, R11_REG] + } + + /// How many bytes a call and a bare bones [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + + // These are the callee-saved registers in the x86-64 SysV ABI + // RBX, RSP, RBP, and R12-R15 + + /// Split IR instructions for the x86 platform + fn x86_split(mut self) -> Assembler + { + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; + let mut iterator = self.instruction_iterator(); + + while let Some((_index, mut insn)) = iterator.next(asm) { + let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); + let is_jump = insn.is_jump(); + + if !is_jump { + insn.for_each_operand_mut(|opnd| { + // Lower Opnd::Value to Opnd::VReg or Opnd::UImm + if let Opnd::Value(value) = opnd { + // If the value is a special constant, and it fits in 32 bits, + // then we're going to output this as just an immediate + if value.special_const_p() { + if imm_num_bits(value.as_i64()) > 32 { + *opnd = asm.load(*opnd); + } else { + *opnd = Opnd::UImm(value.as_u64()); + } + // If we're already loading it, don't load it again + // If it's a jump, then we want to let parallel move + // take care of the block params (otherwise we end up + // with loads between jump instructions) + } else if !is_load { + *opnd = asm.load(*opnd); + } + } + }); + } + + // We are replacing instructions here so we know they are already + // being used. It is okay not to use their output here. + #[allow(unused_must_use)] + match &mut insn { + Insn::Add { .. } | + Insn::Sub { .. } | + Insn::Mul { .. } | + Insn::And { .. } | + Insn::Or { .. } | + Insn::Xor { .. } => { + asm.push_insn(insn); + }, + Insn::Cmp { left, right } => { + // Replace `cmp REG, 0` (4 bytes) with `test REG, REG` (3 bytes) + // when next IR is `je`, `jne`, `csel_e`, or `csel_ne` + match (&left, &right, iterator.peek().map(|(_, insn)| insn)) { + (Opnd::VReg { .. }, + Opnd::UImm(0) | Opnd::Imm(0), + Some(Insn::Je(_) | Insn::Jne(_) | Insn::CSelE { .. } | Insn::CSelNE { .. })) => { + asm.push_insn(Insn::Test { left: *left, right: *left }); + } + _ => { + // Split the instruction if `cmp` can't be encoded with given operands + match (&left, &right) { + // One of the operands should not be a memory operand + (Opnd::Mem(_), Opnd::Mem(_)) => { + *right = asm.load(*right); + } + // The left operand needs to be either a register or a memory operand + (Opnd::UImm(_) | Opnd::Imm(_), _) => { + *left = asm.load(*left); + } + _ => {}, + } + asm.push_insn(insn); + } + } + }, + Insn::Test { left, right } => { + match (&left, &right) { + (Opnd::Mem(_), Opnd::Mem(_)) => { + *right = asm.load(*right); + } + // The first operand can't be an immediate value + (Opnd::UImm(_) | Opnd::Imm(_), _) => { + *left = asm.load(*left); + } + _ => {} + } + asm.push_insn(insn); + }, + // These instructions modify their input operand in-place, so we + // need to load the input value to preserve it + Insn::LShift { opnd, .. } | + Insn::RShift { opnd, .. } | + Insn::URShift { opnd, .. } => { + *opnd = asm.load(*opnd); + asm.push_insn(insn); + }, + Insn::CSelZ { truthy, falsy, .. } | + Insn::CSelNZ { truthy, falsy, .. } | + Insn::CSelE { truthy, falsy, .. } | + Insn::CSelNE { truthy, falsy, .. } | + Insn::CSelL { truthy, falsy, .. } | + Insn::CSelLE { truthy, falsy, .. } | + Insn::CSelG { truthy, falsy, .. } | + Insn::CSelGE { truthy, falsy, .. } => { + match *truthy { + // If we have an instruction output whose live range + // spans beyond this instruction, we have to load it. + Opnd::VReg { idx: _, .. } => { + if true /* conservatively assume vreg outlives insn */ { + *truthy = asm.load(*truthy); + } + }, + Opnd::UImm(_) | Opnd::Imm(_) => { + *truthy = asm.load(*truthy); + }, + // Opnd::Value could have already been split + Opnd::Value(_) if !matches!(truthy, Opnd::VReg { .. }) => { + *truthy = asm.load(*truthy); + }, + _ => {} + } + + match falsy { + Opnd::UImm(_) | Opnd::Imm(_) => { + *falsy = asm.load(*falsy); + }, + _ => {} + } + + asm.push_insn(insn); + }, + Insn::Mov { dest, src } => { + if let Opnd::Mem(_) = dest { + asm.store(*dest, *src); + } else { + asm.mov(*dest, *src); + } + }, + Insn::Not { opnd, .. } => { + match *opnd { + // If we have an instruction output whose live range + // spans beyond this instruction, we have to load it. + Opnd::VReg { idx: _, .. } => { + if true /* conservatively assume vreg outlives insn */ { + *opnd = asm.load(*opnd); + } + }, + // We have to load memory and register operands to avoid + // corrupting them. + Opnd::Mem(_) | Opnd::Reg(_) => { + *opnd = asm.load(*opnd); + }, + // Otherwise we can just reuse the existing operand. + _ => {}, + }; + asm.push_insn(insn); + }, + Insn::CCall { opnds, .. } => { + assert!(opnds.len() <= C_ARG_OPNDS.len()); + // CCall argument setup is handled by handle_caller_saved_regs. + asm.push_insn(insn); + }, + Insn::Lea { .. } => { + asm.push_insn(insn); + }, + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Split instructions using scratch registers. To maximize the use of the register pool + /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so + /// this splits them and uses scratch registers for it. + pub fn x86_scratch_split(self) -> Assembler { + /// For some instructions, we want to be able to lower a 64-bit operand + /// without requiring more registers to be available in the register + /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold + /// the value before we immediately use it. + fn split_64bit_immediate(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Imm(value) => { + // 32-bit values will be sign-extended + if imm_num_bits(value) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + }, + Opnd::UImm(value) => { + // 32-bit values will be sign-extended + if imm_num_bits(value as i64) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd + } else { + Opnd::Imm(value as i64) + } + }, + _ => opnd + } + } + + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR). + /// In general, `out` operand is a `VReg`, so it may use MemBase::Stack, but not MemBase::StackIndirect. + fn lower_stack_membase(opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // Convert MemBase::Stack to MemBase::Reg(NATIVE_BASE_PTR) with the + // correct stack displacement. The stack slot value lives directly at + // [NATIVE_BASE_PTR + stack_disp], so we just adjust the base and + // combine displacements -- no indirection needed. + let Mem { base, disp: stack_disp, .. } = stack_state.stack_membase_to_mem(stack_membase); + Opnd::Mem(Mem { base, disp: stack_disp + opnd_disp, num_bits: opnd_num_bits }) + } + Opnd::Mem(Mem { base: MemBase::StackIndirect { .. }, .. }) => { + unreachable!("lower_stack_membase expects {opnd:?} to not have MemBase::StackIndirect") + } + _ => opnd, + } + } + + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR). + /// For MemBase::StackIndirect, load the pointer from the stack slot into a scratch register. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }) => lower_stack_membase(opnd, stack_state), + Opnd::Mem(Mem { base: MemBase::StackIndirect { stack_idx }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // The spilled value (a pointer) lives at a stack slot. Load it + // into a scratch register, then use the register as the base. + let stack_mem = stack_state.stack_membase_to_mem(MemBase::Stack { stack_idx, num_bits: 64 }); + asm.load_into(scratch_opnd, Opnd::Mem(stack_mem)); + Opnd::Mem(Mem { + base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), + disp: opnd_disp, + num_bits: opnd_num_bits, + }) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + fn assert_out_is_phys_reg_or_stack_mem(out: Opnd) { + assert!( + matches!(out, Opnd::Reg(_) | Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. })), + "x86_scratch_split expects out to be a physical register or stack memory, got {out:?}" + ); + } + + /// If both opnd and other are Opnd::Mem, split opnd with scratch_opnd. + fn split_if_both_memory(asm: &mut Assembler, opnd: Opnd, other: Opnd, scratch_opnd: Opnd) -> Opnd { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (opnd, other) { + asm.load_into(scratch_opnd.with_num_bits(opnd.rm_num_bits()), opnd); + scratch_opnd.with_num_bits(opnd.rm_num_bits()) + } else { + opnd + } + } + + /// Move src to dst, splitting it with scratch_opnd if it's a Mem-to-Mem move. Skip it if dst == src. + fn asm_mov(asm: &mut Assembler, dst: Opnd, src: Opnd, scratch_opnd: Opnd) { + if dst != src { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (dst, src) { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else if let (Opnd::Mem(_), Opnd::Value(value)) = (dst, src) { + if imm_num_bits(value.as_i64()) > 32 { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else { + asm.mov(dst, src); + } + } else { + asm.mov(dst, src); + } + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + for insn in linearized_insns.iter() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + assert_out_is_phys_reg_or_stack_mem(*out); + + // Sequential pipeline to lower binops into x86 two-operand form. + // Uses SCRATCH0 for left, SCRATCH1 for right to avoid conflicts. + + // Phase 1: Protect right if it occupies the same location as out, + // since we'll overwrite out when moving left into it. + // Compare before lowering (Stack membases change during lowering). + let right_eq_out = out == right; + *out = lower_stack_membase(*out, &stack_state); + if right_eq_out { + asm.mov(SCRATCH1_OPND, *out); + *right = SCRATCH1_OPND; + } + + // Phase 2: Lower stack memory bases + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + if !right_eq_out { + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + } + + // Phase 3: If right is a Mem whose base register equals the out + // register, materialize it before we clobber out. + if let (Opnd::Reg(out_reg), Opnd::Mem(Mem { base: MemBase::Reg(base_reg_no), .. })) = (*out, *right) { + if out_reg.reg_no == base_reg_no { + asm.mov(SCRATCH1_OPND, *right); + *right = SCRATCH1_OPND; + } + } + + // Phase 4: x86 can't encode two memory operands. + if let (Opnd::Mem(_), Opnd::Mem(_)) = (*out, *right) { + asm.mov(SCRATCH1_OPND, *right); + *right = SCRATCH1_OPND; + } + + // Phase 5: Move left into out (x86 two-operand form: OP clobbers dst). + // For Mem out, split large left immediates first (mov [mem], imm64 is invalid). + if let Opnd::Mem(_) = *out { + *left = split_64bit_immediate(asm, *left, SCRATCH0_OPND); + } + asm_mov(asm, *out, *left, SCRATCH0_OPND); + *left = *out; + + // Phase 6: Split large right immediates (>32-bit) into a register. + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + // Phase 7: Emit the instruction. + asm.push_insn(insn); + } + Insn::Mul { left, right, out } => { + assert_out_is_phys_reg_or_stack_mem(*out); + + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + *out = lower_stack_membase(*out, &stack_state); + + // imul doesn't have (Mem, Reg) encoding. Swap left and right in that case. + if let (Opnd::Mem(_), Opnd::Reg(_)) = (&left, &right) { + mem::swap(left, right); + } + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + &mut Insn::Not { opnd, out } | + &mut Insn::LShift { opnd, out, .. } | + &mut Insn::RShift { opnd, out, .. } | + &mut Insn::URShift { opnd, out, .. } => { + asm.push_insn(insn); + asm_mov(asm, out, opnd, SCRATCH0_OPND); + } + Insn::Test { left, right } | + Insn::Cmp { left, right } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + + let num_bits = match right { + Opnd::Imm(value) => Some(imm_num_bits(*value)), + Opnd::UImm(value) => Some(uimm_num_bits(*value)), + _ => None + }; + + // If the immediate is less than 64 bits (like 32, 16, 8), and the operand + // sizes match, then we can represent it as an immediate in the instruction + // without moving it to a register first. + // IOW, 64 bit immediates must always be moved to a register + // before comparisons, where other sizes may be encoded + // directly in the instruction. + let use_imm = num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64; + if !use_imm { + *right = split_64bit_immediate(asm, *right, SCRATCH0_OPND); + } + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Lea { opnd, out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(_) = target { + asm.push_insn(Insn::LeaJumpTarget { out: SCRATCH0_OPND, target: target.clone() }); + asm.mov(*out, SCRATCH0_OPND); + } + } + Insn::Load { out, opnd } | + Insn::LoadInto { dest: out, opnd } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + // Split stack membase on out before checking for memory write + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND.with_num_bits(mem_out.rm_num_bits())); + } + } + // Convert Opnd::const_ptr into Opnd::Mem. This split is done here to give + // a register for compile_exits. + &mut Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.incr_counter(Opnd::mem(64, SCRATCH0_OPND, 0), value); + } + &mut Insn::Mov { dest, src } => { + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + let src = split_stack_membase(asm, src, SCRATCH0_OPND, &stack_state); + asm_mov(asm, dest, src, SCRATCH0_OPND); + } + // Handle various operand combinations for spills on compile_exits. + &mut Insn::Store { dest, src } => { + let num_bits = dest.rm_num_bits(); + let src = split_stack_membase(asm, src, SCRATCH0_OPND, &stack_state); + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + + let src = match src { + Opnd::Reg(_) => src, + Opnd::Mem(_) => { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + Opnd::Imm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else if uimm_num_bits(imm as u64) <= num_bits { + // If the bit string is short enough for the destination, use the unsigned representation. + // Note that 64-bit and negative values are ruled out. + Opnd::UImm(imm as u64) + } else { + src + } + } + Opnd::UImm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm as i64) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else { + src.into() + } + } + Opnd::Value(_) => { + asm.load_into(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_scratch_split: {src:?}"), + }; + asm.store(dest, src); + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Emit platform-specific machine code + pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<CodePtr>, CompileError> { + fn emit_csel( + cb: &mut CodeBlock, + truthy: Opnd, + falsy: Opnd, + out: Opnd, + cmov_fn: fn(&mut CodeBlock, X86Opnd, X86Opnd), + cmov_neg: fn(&mut CodeBlock, X86Opnd, X86Opnd)){ + + // Assert that output is a register + let out_reg = out.unwrap_reg(); + + /// Check if a memory operand uses the given register as its base + fn mem_uses_reg(opnd: &Opnd, reg: Reg) -> bool { + if let Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) = opnd { + *reg_no == reg.reg_no + } else { + false + } + } + + /// Check if an operand aliases the given register (either as a + /// register operand or as a memory base). + fn aliases_reg(opnd: &Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(r) => r.reg_no == reg.reg_no, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => *reg_no == reg.reg_no, + _ => false, + } + } + + // If the truthy value is a memory operand + if let Opnd::Mem(_) = truthy { + // If out aliases truthy, we must load truthy into the scratch + // register first to avoid clobbering it with the falsy mov. + if aliases_reg(&truthy, out_reg) { + mov(cb, SCRATCH0_OPND.into(), truthy.into()); + if out != falsy { + mov(cb, out.into(), falsy.into()); + } + cmov_fn(cb, out.into(), SCRATCH0_OPND.into()); + } else { + if out != falsy { + mov(cb, out.into(), falsy.into()); + } + cmov_fn(cb, out.into(), truthy.into()); + } + } else { + // If out aliases falsy, we must load falsy into the scratch + // register first to avoid clobbering it with the truthy mov. + if aliases_reg(&falsy, out_reg) { + mov(cb, SCRATCH0_OPND.into(), falsy.into()); + if out != truthy { + mov(cb, out.into(), truthy.into()); + } + cmov_neg(cb, out.into(), SCRATCH0_OPND.into()); + } else { + if out != truthy { + mov(cb, out.into(), truthy.into()); + } + cmov_neg(cb, out.into(), falsy.into()); + } + } + } + + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest_reg: X86Opnd, value: VALUE) { + // Using movabs because mov might write value in 32 bits + movabs(cb, dest_reg, value.0 as _); + // The pointer immediate is encoded as the last part of the mov written out + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } + + // List of GC offsets + let mut gc_offsets: Vec<CodePtr> = Vec::new(); + + // Buffered list of PosMarker callbacks to fire if codegen is successful + let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // For each instruction + let mut insn_idx: usize = 0; + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { + match insn { + Insn::Comment(text) => { + cb.add_comment(text); + }, + + // Write the label at the current position + Insn::Label(target) => { + cb.write_label(target.unwrap_label()); + }, + + // Report back the current position in the generated code + Insn::PosMarker(..) => { + pos_markers.push((insn_idx, cb.get_write_ptr())); + }, + + Insn::BakeString(text) => { + for byte in text.as_bytes() { + cb.write_byte(*byte); + } + + // Add a null-terminator byte for safety (in case we pass + // this to C code) + cb.write_byte(0); + }, + + // Set up RBP as frame pointer work with unwinding + // (e.g. with Linux `perf record --call-graph fp`) + // and to allow push and pops in the function. + &Insn::FrameSetup { preserved, mut slot_count } => { + // Bump slot_count for alignment if necessary + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + let total_slots = 2 /* rbp and return address*/ + slot_count + preserved.len(); + if total_slots % 2 == 1 { + slot_count += 1; + } + push(cb, RBP); + mov(cb, RBP, RSP); + for reg in preserved { + push(cb, reg.into()); + } + if slot_count > 0 { + sub(cb, RSP, uimm_opnd((slot_count * SIZEOF_VALUE) as u64)); + } + } + &Insn::FrameTeardown { preserved } => { + let mut preserved_offset = -8; + for reg in preserved { + mov(cb, reg.into(), mem_opnd(64, RBP, preserved_offset)); + preserved_offset -= 8; + } + mov(cb, RSP, RBP); + pop(cb, RBP); + } + + Insn::Add { left, right, .. } => { + add(cb, left.into(), right.into()); + }, + + Insn::Sub { left, right, .. } => { + sub(cb, left.into(), right.into()); + }, + + Insn::Mul { left, right, .. } => { + imul(cb, left.into(), right.into()); + }, + + Insn::And { left, right, .. } => { + and(cb, left.into(), right.into()); + }, + + Insn::Or { left, right, .. } => { + or(cb, left.into(), right.into()); + }, + + Insn::Xor { left, right, .. } => { + xor(cb, left.into(), right.into()); + }, + + Insn::Not { opnd, .. } => { + not(cb, opnd.into()); + }, + + Insn::LShift { opnd, shift , ..} => { + shl(cb, opnd.into(), shift.into()) + }, + + Insn::RShift { opnd, shift , ..} => { + sar(cb, opnd.into(), shift.into()) + }, + + Insn::URShift { opnd, shift, .. } => { + shr(cb, opnd.into(), shift.into()) + }, + + // This assumes only load instructions can contain references to GC'd Value operands + Insn::Load { opnd, out } | + Insn::LoadInto { dest: out, opnd } => { + match opnd { + Opnd::Value(val) if val.heap_object_p() => { + emit_load_gc_value(cb, &mut gc_offsets, out.into(), *val); + } + _ => mov(cb, out.into(), opnd.into()) + } + }, + + Insn::LoadSExt { opnd, out } => { + movsx(cb, out.into(), opnd.into()); + }, + + Insn::Store { dest, src } | + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); + }, + + // Load effective address + Insn::Lea { opnd, out } => { + lea(cb, out.into(), opnd.into()); + }, + + // Load address of jump target + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(label) = target { + let out = *out; + // Set output to the raw address of the label + cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| { + let disp = dst_addr - src_addr; + lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap())); + Ok(()) + }); + } else { + // Set output to the jump target's raw address + let target_code = target.unwrap_code_ptr(); + let target_addr = target_code.raw_addr(cb).as_u64(); + // Constant encoded length important for patching + movabs(cb, out.into(), target_addr); + } + }, + + // Push and pop to/from the C stack + Insn::CPush(opnd) => { + push(cb, opnd.into()); + }, + Insn::CPushPair(opnd0, opnd1) => { + push(cb, opnd0.into()); + push(cb, opnd1.into()); + }, + Insn::CPop { out } => { + pop(cb, out.into()); + }, + Insn::CPopInto(opnd) => { + pop(cb, opnd.into()); + }, + Insn::CPopPairInto(opnd0, opnd1) => { + pop(cb, opnd0.into()); + pop(cb, opnd1.into()); + }, + + // C function call + Insn::CCall { fptr, .. } => { + match fptr { + Opnd::UImm(fptr) => { + call_ptr(cb, RAX, *fptr as *const u8); + } + Opnd::Reg(_) => { + call(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } + }, + + Insn::CRet(opnd) => { + // TODO: bias allocation towards return register + if *opnd != Opnd::Reg(C_RET_REG) { + mov(cb, RAX, opnd.into()); + } + + ret(cb); + }, + + // Compare + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); + } + + // Test and set flags + Insn::Test { left, right } => { + test(cb, left.into(), right.into()); + } + + Insn::JmpOpnd(opnd) => { + jmp_rm(cb, opnd.into()); + } + + // Conditional jump to a label + Insn::Jmp(target) => { + match *target { + Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), + Target::Label(label) => jmp_label(cb, label), + Target::Block(ref edge) => jmp_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Je(target) => { + match *target { + Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), + Target::Label(label) => je_label(cb, label), + Target::Block(ref edge) => je_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jne(target) => { + match *target { + Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), + Target::Label(label) => jne_label(cb, label), + Target::Block(ref edge) => jne_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jl(target) => { + match *target { + Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), + Target::Label(label) => jl_label(cb, label), + Target::Block(ref edge) => jl_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jg(target) => { + match *target { + Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr), + Target::Label(label) => jg_label(cb, label), + Target::Block(ref edge) => jg_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jge(target) => { + match *target { + Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr), + Target::Label(label) => jge_label(cb, label), + Target::Block(ref edge) => jge_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jbe(target) => { + match *target { + Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), + Target::Label(label) => jbe_label(cb, label), + Target::Block(ref edge) => jbe_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jb(target) => { + match *target { + Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr), + Target::Label(label) => jb_label(cb, label), + Target::Block(ref edge) => jb_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jz(target) => { + match *target { + Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), + Target::Label(label) => jz_label(cb, label), + Target::Block(ref edge) => jz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jnz(target) => { + match *target { + Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), + Target::Label(label) => jnz_label(cb, label), + Target::Block(ref edge) => jnz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jo(target) | + Insn::JoMul(target) => { + match *target { + Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), + Target::Label(label) => jo_label(cb, label), + Target::Block(ref edge) => jo_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Joz(..) | Insn::Jonz(..) => unreachable!("Joz/Jonz should be unused for now"), + + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in x86_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + let code_size = cb.get_write_pos().saturating_sub(last_patch_pos); + if code_size < cb.jmp_ptr_bytes() { + nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + + // Atomically increment a counter at a given memory location + Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::Mem(_))); + assert!(matches!(value, Opnd::UImm(_) | Opnd::Imm(_) ) ); + write_lock_prefix(cb); + add(cb, mem.into(), value.into()); + }, + + Insn::Breakpoint => int3(cb), + Insn::Abort => ud2(cb), + + Insn::CSelZ { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovz, cmovnz); + }, + Insn::CSelNZ { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovnz, cmovz); + }, + Insn::CSelE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmove, cmovne); + }, + Insn::CSelNE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovne, cmove); + }, + Insn::CSelL { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovl, cmovge); + }, + Insn::CSelLE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovle, cmovg); + }, + Insn::CSelG { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovg, cmovle); + }, + Insn::CSelGE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovge, cmovl); + } + }; + + insn_idx += 1; + } + + // Error if we couldn't write out everything + if cb.has_dropped_bytes() { + Err(CompileError::OutOfMemory) + } else { + // No bytes dropped, so the pos markers point to valid code + for (insn_idx, pos) in pos_markers { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { + callback(pos, cb); + } else { + panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); + } + } + + Ok(gc_offsets) + } + } + + /// Optimize and compile the stored instructions + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_regs = !self.accept_scratch_reg; + asm_dump!(self, init); + + let mut asm = trace_compile_phase("split", || self.x86_split()); + + asm_dump!(asm, split); + + trace_compile_phase("regalloc", || { + trace_compile_phase("number_instructions", || asm.number_instructions(0)); + + let live_in = trace_compile_phase("analyze_liveness", || asm.analyze_liveness()); + let intervals = trace_compile_phase("build_intervals", || asm.build_intervals(live_in)); + + // Dump live intervals if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::live_intervals) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + } + } + + let preferred_registers = trace_compile_phase("preferred_registers", || asm.preferred_register_assignments(&intervals)); + let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); + + let total_stack_slots = asm.stack_base_idx + num_stack_slots; + if total_stack_slots > Self::MAX_FRAME_STACK_SLOTS { + return Err(CompileError::NativeStackTooLarge); + } + + // Dump vreg-to-physical-register mapping if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::alloc_regs) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + println!("VReg assignments:"); + for (i, alloc) in assignments.iter().enumerate() { + if let Some(alloc) = alloc { + let range = &intervals[i].range; + let alloc_str = match alloc { + Allocation::Reg(n) => format!("{}", regs[*n]), + Allocation::Fixed(reg) => format!("{}", reg), + Allocation::Stack(n) => format!("Stack[{}]", n), + }; + println!(" v{} => {} (range: {:?}..{:?})", i, alloc_str, range.start, range.end); + } + } + } + } + + // Update FrameSetup slot_count to account for: + // 1) stack slots reserved for block params (stack_base_idx), and + // 2) register allocator spills (num_stack_slots). + trace_compile_phase("count_stack_slots", || { + for block in asm.basic_blocks.iter_mut() { + for insn in block.insns.iter_mut() { + if let Insn::FrameSetup { slot_count, .. } = insn { + *slot_count = total_stack_slots; + } + } + } + }); + + trace_compile_phase("resolve_ssa", || { + asm.handle_caller_saved_regs(&intervals, &assignments, &C_ARG_REGREGS); + asm.resolve_ssa(&intervals, &assignments); + }); + + Ok(()) + })?; + asm_dump!(asm, alloc_regs); + + // We are moved out of SSA after resolve_ssa + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + // Exit code is compiled into a separate list of instructions that we append + // to the last reachable block before scratch_split, so it gets linearized and split. + trace_compile_phase("compile_exits", || { + let exit_insns = asm.compile_exits(); + + // Append exit instructions to the last reachable block so they are + // included in linearize_instructions and processed by scratch_split. + if let Some(&last_block) = asm.block_order().last() { + for insn in exit_insns { + asm.basic_blocks[last_block.0].insns.push(insn); + asm.basic_blocks[last_block.0].insn_ids.push(None); + } + } + }); + asm_dump!(asm, compile_exits); + + if use_scratch_regs { + asm = trace_compile_phase("scratch_split", || asm.x86_scratch_split()); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = trace_compile_phase("resolve_parallel_mov", || asm.resolve_parallel_mov_pass()); + asm_dump!(asm, resolve_parallel_mov); + } + + trace_compile_phase("emit", || { + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.x86_emit(cb).inspect_err(|_| cb.clear_labels())?; + assert!(!cb.has_dropped_bytes(), "emit should not drop bytes without error"); + + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + Ok((start_ptr, gc_offsets)) + }) + } +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + use crate::options::rb_zjit_prepare_options; + use super::*; + + const BOLD_BEGIN: &str = "\x1b[1m"; + const BOLD_END: &str = "\x1b[22m"; + + fn setup_asm() -> (Assembler, CodeBlock) { + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) + } + + fn stack_mem(stack_idx: usize) -> Opnd { + Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx, num_bits: 64 }, + disp: 0, + num_bits: 64, + }) + } + + fn stack_indirect_mem(stack_idx: usize) -> Opnd { + Opnd::Mem(Mem { + base: MemBase::StackIndirect { stack_idx }, + disp: 0, + num_bits: 64, + }) + } + + #[derive(Clone, Copy, Debug)] + enum BinOpKind { + Add, + Sub, + And, + Or, + Xor, + } + + fn split_binop(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Assembler { + let (mut asm, _) = setup_asm(); + match kind { + BinOpKind::Add => asm.push_insn(Insn::Add { left, right, out }), + BinOpKind::Sub => asm.push_insn(Insn::Sub { left, right, out }), + BinOpKind::And => asm.push_insn(Insn::And { left, right, out }), + BinOpKind::Or => asm.push_insn(Insn::Or { left, right, out }), + BinOpKind::Xor => asm.push_insn(Insn::Xor { left, right, out }), + } + asm.x86_scratch_split() + } + + fn binop_mnemonic(kind: BinOpKind) -> &'static str { + match kind { + BinOpKind::Add => "add", + BinOpKind::Sub => "sub", + BinOpKind::And => "and", + BinOpKind::Or => "or", + BinOpKind::Xor => "xor", + } + } + + fn split_binop_disasm_lines(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Vec<String> { + let mut asm = split_binop(kind, left, right, out); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok(), "{kind:?}: x86_emit failed"); + + cb.disasm() + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| { + line.split_once(": ") + .map(|(_, rest)| rest.to_string()) + .unwrap_or_else(|| line.to_string()) + }) + .collect() + } + + fn assert_split_binop_case(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd, case: &str) { + fn reg_names(reg: Reg) -> (&'static str, &'static str) { + const RAX_NO: u8 = RAX_REG.reg_no; + const RDI_NO: u8 = RDI_REG.reg_no; + const R13_NO: u8 = R13_REG.reg_no; + match reg.reg_no { + RAX_NO => ("rax", "eax"), + RDI_NO => ("rdi", "edi"), + R13_NO => ("r13", "r13d"), + _ => panic!("unexpected register in test helper: {reg:?}"), + } + } + + let lines = split_binop_disasm_lines(kind, left, right, out); + let mnemonic = binop_mnemonic(kind); + let matching: Vec<(usize, &String)> = lines.iter() + .enumerate() + .filter(|(_, line)| line.starts_with(&format!("{mnemonic} "))) + .collect(); + + assert_eq!(matching.len(), 1, "{kind:?} {case}: expected exactly one `{mnemonic}` in disasm, got {lines:?}"); + assert!( + !matching[0].1.contains("], qword ptr ["), + "{kind:?} {case}: emitted mem/mem `{mnemonic}` in disasm {lines:?}" + ); + + if let (Opnd::Reg(out_reg), Opnd::Imm(_) | Opnd::UImm(_)) = (out, left) { + let (out64, out32) = reg_names(out_reg); + let prelude = &lines[..matching[0].0]; + assert!( + prelude.iter().any(|line| + line.starts_with(&format!("mov {out64}, ")) || + line.starts_with(&format!("mov {out32}, ")) || + line.starts_with(&format!("movabs {out64}, ")) + ), + "{kind:?} {case}: expected left immediate to be materialized into output register before `{mnemonic}`, got {lines:?}" + ); + } + + if matches!(right, Opnd::Imm(value) if imm_num_bits(value) > 32) + || matches!(right, Opnd::UImm(value) if imm_num_bits(value as i64) > 32) + { + assert!(lines.iter().any(|line| line.starts_with("movabs ")), "{kind:?} {case}: expected movabs materialization for 64-bit immediate, got {lines:?}"); + } + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); + asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.push_insn(Insn::Je(label)); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @" + test(): + bb0(): + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, r13, rbx, r12 + v0 = Add r13, 0x40 + Store [rbx + 0x10], v0 + Joz Exit(Interrupt), v0 + Mov rdi, eax + Mov rsi, [rbx - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[r12 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown r13, rbx, r12 + "); + } + + #[test] + #[ignore] + fn test_emit_add_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: add rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_add_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: add rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c01d8"); + } + + #[test] + #[ignore] + fn test_emit_and_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: and rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_and_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: and rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c21d8"); + } + + #[test] + fn test_emit_cmp_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"4881f8ff000000"); + } + + #[test] + fn test_emit_cmp_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); + } + + #[test] + fn test_emit_cmp_64_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, -1"); + assert_snapshot!(cb.hexdump(), @"4883f8ff"); + } + + #[test] + fn test_emit_cmp_mem_16_bits_with_imm_16() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(16, Opnd::Reg(RAX_REG), 6); + + asm.cmp(shape_opnd, Opnd::UImm(0xF000)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp word ptr [rax + 6], 0xf000"); + assert_snapshot!(cb.hexdump(), @"6681780600f0"); + } + + #[test] + fn test_emit_cmp_mem_32_bits_with_imm_32() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(RAX_REG), 4); + + asm.cmp(shape_opnd, Opnd::UImm(0xF000_0000)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp dword ptr [rax + 4], 0xf0000000"); + assert_snapshot!(cb.hexdump(), @"817804000000f0"); + } + + #[test] + #[ignore] + fn test_emit_or_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: or rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c8ff000000"); + } + + #[test] + #[ignore] + fn test_emit_or_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: or rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c09d8"); + } + + #[test] + #[ignore] + fn test_emit_sub_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: sub rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e8ff000000"); + } + + #[test] + #[ignore] + fn test_emit_sub_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: sub rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c29d8"); + } + + #[test] + fn test_emit_test_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: test rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"48f7c0ff000000"); + } + + #[test] + fn test_emit_test_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: test rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c85d8"); + } + + #[test] + #[ignore] + fn test_emit_xor_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: xor rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881f0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_xor_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: xor rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c31d8"); + } + + #[test] + fn test_merge_lea_reg() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.lea(Opnd::mem(64, SP, 8)); + asm.mov(SP, sp); // should be merged to lea + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lea rbx, [rbx + 8]"); + assert_snapshot!(cb.hexdump(), @"488d5b08"); + } + + #[test] + #[ignore] + fn test_merge_lea_mem() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.lea(Opnd::mem(64, SP, 8)); + asm.mov(Opnd::mem(64, SP, 0), sp); // should NOT be merged to lea + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); + } + + #[test] + #[ignore] + fn test_replace_cmp_0() { + let (mut asm, mut cb) = setup_asm(); + + let val = asm.load(Opnd::mem(64, SP, 8)); + asm.cmp(val, 0.into()); + let result = asm.csel_e(Qtrue.into(), Qfalse.into()); + asm.mov(Opnd::Reg(RAX_REG), result); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, qword ptr [rbx + 8] + 0x4: test rax, rax + 0x7: mov eax, 0x14 + 0xc: mov ecx, 0 + 0x11: cmovne rax, rcx + 0x15: mov rax, rax + "); + assert_snapshot!(cb.hexdump(), @"488b43084885c0b814000000b900000000480f45c14889c0"); + } + + #[test] + fn test_merge_add_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.add(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); + } + + #[test] + fn test_add_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.add_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); + } + + #[test] + fn test_merge_sub_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.sub(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); + } + + #[test] + fn test_sub_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.sub_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); + } + + #[test] + fn test_merge_and_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.and(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983e540"); + } + + #[test] + fn test_merge_or_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.or(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983cd40"); + } + + #[test] + fn test_merge_xor_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.xor(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983f540"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_no_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov rdi, rdi (optimized away) + C_ARG_OPNDS[1], // mov rsi, rsi (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov eax, 0 + 0x5: call rax + "); + assert_snapshot!(cb.hexdump(), @"b800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_single_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi and rsi form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[0], // mov rsi, rdi + C_ARG_OPNDS[2], // mov rdx, rdx (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rsi + 0x3: mov rsi, rdi + 0x6: mov rdi, r11 + 0x9: mov eax, 0 + 0xe: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89dfb800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_two_cycles() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi and rsi form a cycle, and rdx and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[0], // mov rsi, rdi + C_ARG_OPNDS[3], // mov rdx, rcx + C_ARG_OPNDS[2], // mov rcx, rdx + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rcx + 0x3: mov rcx, rdx + 0x6: mov rdx, r11 + 0x9: mov r11, rsi + 0xc: mov rsi, rdi + 0xf: mov rdi, r11 + 0x12: mov eax, 0 + 0x17: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989cb4889d14c89da4989f34889fe4c89dfb800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_large_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi, rsi, and rdx form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[2], // mov rsi, rdx + C_ARG_OPNDS[0], // mov rdx, rdi + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rdx + 0x3: mov rdx, rdi + 0x6: mov rdi, rsi + 0x9: mov rsi, r11 + 0xc: mov eax, 0 + 0x11: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989d34889fa4889f74c89deb800000000ffd0"); + } + + #[test] + #[ignore] + fn test_ccall_resolve_parallel_moves_with_insn_out() { + let (mut asm, mut cb) = setup_asm(); + + let rax = asm.load(Opnd::UImm(1)); + let rcx = asm.load(Opnd::UImm(2)); + let rdx = asm.load(Opnd::UImm(3)); + // rcx and rdx form a cycle + asm.ccall(0 as _, vec![ + rax, // mov rdi, rax + rcx, // mov rsi, rcx + rcx, // mov rdx, rcx + rdx, // mov rcx, rdx + ]); + asm.compile_with_num_regs(&mut cb, 3); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov eax, 1 + 0x5: mov ecx, 2 + 0xa: mov edx, 3 + 0xf: mov rdi, rax + 0x12: mov rsi, rcx + 0x15: mov r11, rcx + 0x18: mov rcx, rdx + 0x1b: mov rdx, r11 + 0x1e: mov eax, 0 + 0x23: call rax + "); + assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: push rdi + 0x15: push rsi + 0x16: push rdx + 0x17: push rcx + 0x18: mov eax, 0 + 0x1d: call rax + 0x1f: pop rcx + 0x20: pop rdx + 0x21: pop rsi + 0x22: pop rdi + 0x23: add rdi, rsi + 0x26: mov rdi, rdx + 0x29: add rdi, rcx + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74889d74801cf"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: mov r8d, 5 + 0x1a: push rdi + 0x1b: push rsi + 0x1c: push rdx + 0x1d: push rcx + 0x1e: push r8 + 0x20: push 0 + 0x22: mov eax, 0 + 0x27: call rax + 0x29: pop r8 + 0x2b: pop r8 + 0x2d: pop rcx + 0x2e: pop rdx + 0x2f: pop rsi + 0x30: pop rdi + 0x31: add rdi, rsi + 0x34: mov rdi, rdx + 0x37: add rdi, rcx + 0x3a: mov rdi, rdx + 0x3d: add rdi, r8 + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141506a00b800000000ffd041584158595a5e5f4801f74889d74801cf4889d74c01c7"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: push rdi + 0xb: push rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: pop rdi + 0xb: pop rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e"); + } + + #[test] + fn test_cmov_mem() { + let (mut asm, mut cb) = setup_asm(); + + let top = Opnd::mem(64, SP, 0); + let ary_opnd = SP; + let array_len_opnd = Opnd::mem(64, SP, 16); + + asm.cmp(array_len_opnd, 1.into()); + let elem_opnd = asm.csel_g(Opnd::mem(64, ary_opnd, 0), Qnil.into()); + asm.mov(top, elem_opnd); + + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cmp qword ptr [rbx + 0x10], 1 + 0x5: mov edi, 4 + 0xa: cmovg rdi, qword ptr [rbx] + 0xe: mov qword ptr [rbx], rdi + "); + assert_snapshot!(cb.hexdump(), @"48837b1001bf04000000480f4f3b48893b"); + } + + #[test] + #[ignore] + fn test_csel_split() { + let (mut asm, mut cb) = setup_asm(); + + let stack_top = Opnd::mem(64, SP, 0); + let elem_opnd = asm.csel_ne(VALUE(0x7f22c88d1930).into(), Qnil.into()); + asm.mov(stack_top, elem_opnd); + + asm.compile_with_num_regs(&mut cb, 3); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs rax, 0x7f22c88d1930 + 0xa: mov ecx, 4 + 0xf: cmove rax, rcx + 0x13: mov qword ptr [rbx], rax + "); + assert_snapshot!(cb.hexdump(), @"48b830198dc8227f0000b904000000480f44c1488903"); + } + + #[test] + fn test_mov_m32_imm32() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, C_RET_OPND, 0); + asm.mov(shape_opnd, Opnd::UImm(0x8000_0001)); + asm.mov(shape_opnd, Opnd::Imm(0x8000_0001)); + + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov dword ptr [rax], 0x80000001 + 0x6: mov dword ptr [rax], 0x80000001 + "); + assert_snapshot!(cb.hexdump(), @"c70001000080c70001000080"); + } + + #[test] + fn frame_setup_teardown_preserved_regs() { + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(JIT_PRESERVED_REGS); + asm.frame_teardown(JIT_PRESERVED_REGS); + asm.cret(C_RET_OPND); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: push r13 + 0x6: push rbx + 0x7: push r12 + 0x9: sub rsp, 8 + 0xd: mov r13, qword ptr [rbp - 8] + 0x11: mov rbx, qword ptr [rbp - 0x10] + 0x15: mov r12, qword ptr [rbp - 0x18] + 0x19: mov rsp, rbp + 0x1c: pop rbp + 0x1d: ret + "); + assert_snapshot!(cb.hexdump(), @"554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3"); + } + + #[test] + fn frame_setup_teardown_stack_base_idx() { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 5; + asm.frame_setup(&[]); + asm.frame_teardown(&[]); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: sub rsp, 0x30 + 0x8: mov rsp, rbp + 0xb: pop rbp + "); + assert_snapshot!(cb.hexdump(), @"554889e54883ec304889ec5d"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + asm = asm.x86_scratch_split(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + let gc_offsets = asm.x86_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0x1000 + 0xa: mov qword ptr [rbx], r11 + "); + assert_snapshot!(cb.hexdump(), @"49bb00100000000000004c891b"); + } + + #[test] + fn test_csel_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let left = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let right = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 1, num_bits: 64 }, disp: 2, num_bits: 64 }); + let _ = asm.csel_e(left, right); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 0xe] + 0x4: cmove r11, qword ptr [rbp - 8] + 0x9: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b5df24c0f445df84c895df8"); + } + + #[test] + fn test_lea_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let _ = asm.lea(opnd); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lea r11, [rbp - 8] + 0x4: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8d5df84c895df8"); + } + + #[test] + fn test_add_split_direct_mem() { + // RAX is safe to be clobbered because it's an output + // c_ret <- add stack[0], stack[1] + let lines = split_binop_disasm_lines(BinOpKind::Add, stack_mem(0), stack_mem(1), C_RET_OPND); + + // load scratch0, [stack[0]] + // add scratch0, [stack[1]] + // mov c_ret, scratch0 + assert_snapshot!(lines.join("\n"), @" + mov rax, qword ptr [rbp - 8] + add rax, qword ptr [rbp - 0x10] + "); + } + + #[test] + fn test_add_split_stack_indirect_left_uses_scratch0_for_base_and_result() { + // stack[1] <- add stack[mem[0]], cfp + let lines = split_binop_disasm_lines(BinOpKind::Add, stack_indirect_mem(0), CFP, stack_mem(1)); + + // load scratch0, [stack[left_base]] + // load scratch0, [scratch0] + // seed [stack[out]] from scratch0, then add right in place + assert_snapshot!(lines.join("\n"), @" + mov r11, qword ptr [rbp - 8] + mov r11, qword ptr [r11] + mov qword ptr [rbp - 0x10], r11 + add qword ptr [rbp - 0x10], r13 + "); + } + + #[test] + fn test_add_split_stack_indirect_right_uses_separate_base_scratch() { + // mem[1] <- add cfp, mem[stack[0]] + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_indirect_mem(0), stack_mem(1)); + + // load scratch1, [stack[right_base]] + // load scratch1, [scratch1] + // seed [stack[out]] from left, then add scratch1 in place + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 8] + mov r10, qword ptr [r10] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_two_stack_indirect_inputs_need_two_scratch_regs() { + // stack[2] <- add [stack[0]], [stack[1]] + let lines = split_binop_disasm_lines(BinOpKind::Add, + stack_indirect_mem(0), stack_indirect_mem(1), stack_mem(2)); + + // load scratch0, [stack[left_base]] + // load scratch0, [scratch0] + // load scratch1, [stack[right_base]] + // load scratch1, [scratch1] + // mov [stack[out]], scratch0; add [stack[out]], scratch1 + assert_snapshot!(lines.join("\n"), @" + mov r11, qword ptr [rbp - 8] + mov r10, qword ptr [rbp - 0x10] + mov r10, qword ptr [r10] + mov r11, qword ptr [r11] + mov qword ptr [rbp - 0x18], r11 + add qword ptr [rbp - 0x18], r10 + "); + } + + #[test] + fn test_add_split_memory_output_can_compute_in_place() { + // stack[1] <- add cfp, stack[0] + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(0), stack_mem(1)); + + // Seed the output slot with the left register, then perform the add in place. + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 8] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_reg_mem_mem_when_right_equals_out() { + // stack[1] <- add cfp, stack[1] + // + // Preserve the original RHS/output value first, then seed the output + // slot with the left register and finish the add in place. + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(1), stack_mem(1)); + + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 0x10] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + #[should_panic(expected = "x86_scratch_split expects out to be a physical register or stack memory")] + fn test_binop_split_rejects_non_stack_memory_output() { + let _ = split_binop(BinOpKind::Add, CFP, C_RET_OPND, Opnd::mem(64, CFP, 8)); + } + + #[test] + fn test_add_split_mem_mem_mem_when_right_equals_out() { + // stack[1] <- add stack[0], stack[1] + // + // Preserve the original RHS/output value first, then reload the LHS and + // write the result back to the output stack slot. + let mut asm = split_binop(BinOpKind::Add, stack_mem(0), stack_mem(1), stack_mem(1)); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [rbp - 0x10] + 0x4: mov r11, qword ptr [rbp - 8] + 0x8: mov qword ptr [rbp - 0x10], r11 + 0xc: add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_output_reg_reused_as_input_memory_base() { + // cfp <- add [cfp + 8], [cfp + 16] + // + // Preserve the RHS first because reusing `out` for the LHS would otherwise + // clobber the base register needed to address the RHS memory operand. + let mut asm = split_binop( + BinOpKind::Add, + Opnd::mem(64, CFP, 8), + Opnd::mem(64, CFP, 16), + CFP, + ); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [r13 + 0x10] + 0x4: mov r13, qword ptr [r13 + 8] + 0x8: add r13, r10 + "); + assert_snapshot!(cb.hexdump(), @"4d8b55104d8b6d084d01d5"); + } + + #[test] + fn test_add_split_output_reg_reused_as_stack_indirect_memory_base() { + // cfp <- add mem[stack[0]], mem[stack[1]] + // + // The LHS lowering reuses `out` as the temporary base register for the + // stack-indirect address, then the RHS uses the normal scratch register. + let mut asm = split_binop( + BinOpKind::Add, + stack_indirect_mem(0), + stack_indirect_mem(1), + CFP, + ); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 8] + 0x4: mov r10, qword ptr [rbp - 0x10] + 0x8: mov r13, qword ptr [r11] + 0xb: add r13, qword ptr [r10] + "); + } + + #[test] + fn test_add_split_output_reg_reused_as_input_memory_base_with_imm() { + // cfp <- add 7, [cfp + 16] + // + // Preserve the RHS first because reusing `out` for the immediate + // materialization would otherwise clobber the base register needed for + // the memory operand. + let lines = split_binop_disasm_lines( + BinOpKind::Add, + Opnd::Imm(7), + Opnd::mem(64, CFP, 16), + CFP, + ); + + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [r13 + 0x10] + mov r13d, 7 + add r13, r10 + "); + } + + #[test] + fn test_binop_split_matrix() { + let left_cases = [ + ("reg", CFP), + ("mem_reg", Opnd::mem(64, C_RET_OPND, 8)), + ("mem_stack", stack_mem(0)), + ("mem_stack_indirect", stack_indirect_mem(0)), + ("imm32", Opnd::Imm(7)), + ("imm64", Opnd::UImm(0x1_0000_0000)), + ]; + let right_cases = [ + ("reg", C_RET_OPND), + ("mem_reg", Opnd::mem(64, CFP, 16)), + ("mem_stack", stack_mem(1)), + ("mem_stack_indirect", stack_indirect_mem(1)), + ("imm32", Opnd::Imm(9)), + ("imm64", Opnd::UImm(0x2_0000_0000)), + ]; + let out_cases = [ + ("out_reg", C_ARG_OPNDS[0]), + ("out_mem_stack", stack_mem(2)), + ]; + let alias_out_cases = [ + ("alias_out_cfp", CFP), + ("alias_out_reg", C_RET_OPND), + ("alias_out_mem_stack", stack_mem(1)), + ]; + let alias_mem_base_cases = [ + ( + "out_reused_as_left_mem_base", + Opnd::mem(64, CFP, 8), + C_RET_OPND, + CFP, + ), + ( + "out_reused_as_right_mem_base", + C_RET_OPND, + Opnd::mem(64, CFP, 16), + CFP, + ), + ( + "out_reused_as_both_mem_bases", + Opnd::mem(64, CFP, 8), + Opnd::mem(64, CFP, 16), + CFP, + ), + ]; + + for kind in [BinOpKind::Add, BinOpKind::Sub, BinOpKind::And, BinOpKind::Or, BinOpKind::Xor] { + for (left_name, left) in left_cases { + for (right_name, right) in right_cases { + for (out_name, out) in out_cases { + let case = format!("{left_name}/{right_name}/{out_name}"); + assert_split_binop_case(kind, left, right, out, &case); + } + } + } + + for (left_name, left) in left_cases { + for (alias_name, out) in alias_out_cases { + let case = format!("{left_name}/right_eq_out/{alias_name}"); + assert_split_binop_case(kind, left, out, out, &case); + } + } + + for (case, left, right, out) in alias_mem_base_cases { + assert_split_binop_case(kind, left, right, out, case); + } + } + } +} diff --git a/zjit/src/bitset.rs b/zjit/src/bitset.rs new file mode 100644 index 0000000000..986d537d9b --- /dev/null +++ b/zjit/src/bitset.rs @@ -0,0 +1,225 @@ +//! Optimized bitset implementation. + +type Entry = u128; + +const ENTRY_NUM_BITS: usize = Entry::BITS as usize; + +// TODO(max): Make a `SmallBitSet` and `LargeBitSet` and switch between them if `num_bits` fits in +// `Entry`. +#[derive(Clone)] +pub struct BitSet<T: Into<usize> + Copy> { + entries: Vec<Entry>, + num_bits: usize, + phantom: std::marker::PhantomData<T>, +} + +impl<T: Into<usize> + Copy> BitSet<T> { + pub fn with_capacity(num_bits: usize) -> Self { + let num_entries = num_bits.div_ceil(ENTRY_NUM_BITS); + Self { entries: vec![0; num_entries], num_bits, phantom: Default::default() } + } + + /// Returns whether the value was newly inserted: true if the set did not originally contain + /// the bit, and false otherwise. + pub fn insert(&mut self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + let newly_inserted = (self.entries[entry_idx] & (1 << bit_idx)) == 0; + self.entries[entry_idx] |= 1 << bit_idx; + newly_inserted + } + + /// Set all bits to 1. + pub fn insert_all(&mut self) { + for i in 0..self.entries.len() { + self.entries[i] = !0; + } + } + + /// Clear a bit. Returns whether the bit was previously set. + pub fn remove(&mut self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + let was_set = (self.entries[entry_idx] & (1 << bit_idx)) != 0; + self.entries[entry_idx] &= !(1 << bit_idx); + was_set + } + + pub fn get(&self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + (self.entries[entry_idx] & (1 << bit_idx)) != 0 + } + + /// Modify `self` to only have bits set if they are also set in `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn intersect_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] &= other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } + + /// Modify `self` to have bits set if they are set in either `self` or `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn union_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] |= other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } + + /// Modify `self` to remove bits that are set in `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn difference_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] &= !other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } + + /// Check if two BitSets are equal. + /// `self` and `other` must have the same number of bits. + pub fn equals(&self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + self.entries == other.entries + } + + /// Returns an iterator over the indices of set bits. + /// Only iterates over bits that are set, not all possible indices. + pub fn iter_set_bits(&self) -> impl Iterator<Item = usize> + '_ { + self.entries.iter().enumerate().flat_map(move |(entry_idx, &entry)| { + let mut bits = entry; + std::iter::from_fn(move || { + if bits == 0 { + return None; + } + let bit_pos = bits.trailing_zeros() as usize; + bits &= bits - 1; // Clear the lowest set bit + Some(entry_idx * ENTRY_NUM_BITS + bit_pos) + }) + }).filter(move |&idx| idx < self.num_bits) + } +} + +#[cfg(test)] +mod tests { + use super::BitSet; + + #[test] + #[should_panic] + fn get_over_capacity_panics() { + let set = BitSet::with_capacity(0); + assert!(!set.get(0usize)); + } + + #[test] + fn with_capacity_defaults_to_zero() { + let set = BitSet::with_capacity(4); + assert!(!set.get(0usize)); + assert!(!set.get(1usize)); + assert!(!set.get(2usize)); + assert!(!set.get(3usize)); + } + + #[test] + fn insert_sets_bit() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(set.get(1usize)); + } + + #[test] + fn insert_with_set_bit_returns_false() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(!set.insert(1usize)); + } + + #[test] + fn insert_all_sets_all_bits() { + let mut set = BitSet::with_capacity(4); + set.insert_all(); + assert!(set.get(0usize)); + assert!(set.get(1usize)); + assert!(set.get(2usize)); + assert!(set.get(3usize)); + } + + #[test] + #[should_panic] + fn intersect_with_panics_with_different_num_bits() { + let mut left: BitSet<usize> = BitSet::with_capacity(3); + let right = BitSet::with_capacity(4); + left.intersect_with(&right); + } + #[test] + fn intersect_with_keeps_only_common_bits() { + let mut left = BitSet::with_capacity(3); + let mut right = BitSet::with_capacity(3); + left.insert(0usize); + left.insert(1usize); + right.insert(1usize); + right.insert(2usize); + left.intersect_with(&right); + assert!(!left.get(0usize)); + assert!(left.get(1usize)); + assert!(!left.get(2usize)); + } + + #[test] + fn test_iter_set_bits() { + let mut set: BitSet<usize> = BitSet::with_capacity(10); + set.insert(1usize); + set.insert(5usize); + set.insert(9usize); + + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![1, 5, 9]); + } + + #[test] + fn test_iter_set_bits_empty() { + let set: BitSet<usize> = BitSet::with_capacity(10); + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![]); + } + + #[test] + fn test_iter_set_bits_all() { + let mut set: BitSet<usize> = BitSet::with_capacity(5); + set.insert_all(); + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![0, 1, 2, 3, 4]); + } + + #[test] + fn test_iter_set_bits_large() { + let mut set: BitSet<usize> = BitSet::with_capacity(200); + set.insert(0usize); + set.insert(127usize); + set.insert(128usize); + set.insert(199usize); + + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![0, 127, 128, 199]); + } +} diff --git a/zjit/src/cast.rs b/zjit/src/cast.rs new file mode 100644 index 0000000000..52e2078cde --- /dev/null +++ b/zjit/src/cast.rs @@ -0,0 +1,64 @@ +//! Optimized [usize] casting trait. + +#![allow(clippy::wrong_self_convention)] + +/// Trait for casting to [usize] that allows you to say `.as_usize()`. +/// Implementation conditional on the cast preserving the numeric value on +/// all inputs and being inexpensive. +/// +/// [usize] is only guaranteed to be more than 16-bit wide, so we can't use +/// `.into()` to cast an `u32` or an `u64` to a `usize` even though in all +/// the platforms we support these casts are pretty much no-ops. +/// We could say `as usize` or `.try_convert().unwrap()` everywhere for these +/// casts but both options have undesirable consequences if and when we decide to +/// support 32-bit platforms. Unfortunately we can't implement [::core::convert::From] +/// for [usize] since both the trait and the type are external. Also, naming +/// the method `into()` also causes a name conflict. +pub(crate) trait IntoUsize { + /// Convert to usize. Implementation conditional on width of [usize]. + fn to_usize(self) -> usize; +} + +#[cfg(target_pointer_width = "64")] +impl IntoUsize for u64 { + fn to_usize(self) -> usize { + self as usize + } +} + +#[cfg(target_pointer_width = "64")] +impl IntoUsize for u32 { + fn to_usize(self) -> usize { + self as usize + } +} + +impl IntoUsize for u16 { + /// Alias for `.into()`. For convenience so you could use the trait for + /// all unsgined types. + fn to_usize(self) -> usize { + self.into() + } +} + +impl IntoUsize for u8 { + /// Alias for `.into()`. For convenience so you could use the trait for + /// all unsgined types. + fn to_usize(self) -> usize { + self.into() + } +} + +/// The `Into<u64>` Rust does not provide. +/// Convert to u64 with assurance that the value is preserved. +/// Currently, `usize::BITS == 64` holds for all platforms we support. +pub(crate) trait IntoU64 { + fn as_u64(self) -> u64; +} + +#[cfg(target_pointer_width = "64")] +impl IntoU64 for usize { + fn as_u64(self) -> u64 { + self as u64 + } +} diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs new file mode 100644 index 0000000000..d5d381acfa --- /dev/null +++ b/zjit/src/codegen.rs @@ -0,0 +1,3612 @@ +//! This module is for native code generation. + +#![allow(clippy::let_and_return)] + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::ffi::{c_int, c_long, c_void}; +use std::slice; + +use crate::backend::current::ALLOC_REGS; +use crate::invariants::{ + track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, + track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption, + track_root_box_assumption +}; +use crate::gc::append_gc_offsets; +use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; +use crate::state::ZJITState; +use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; +use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; +use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment}; +use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; +use crate::hir::{BlockHandler, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendFallbackReason}; +use crate::hir_type::{types, Type}; +use crate::options::{get_option, PerfMap}; +use crate::cast::IntoUsize; + +/// The number of stack slots used for JITFrame. gen_save_pc_for_gc() writes +/// JITFrame into this number of slots at the bottom of the native stack. +const JIT_FRAME_SIZE: usize = 1; + +/// Default maximum number of compiled versions per ISEQ. +const DEFAULT_MAX_VERSIONS: usize = 2; + +/// Maximum number of compiled versions per ISEQ. +/// Configurable via --zjit-max-versions (default: 2). +pub fn max_iseq_versions() -> usize { + unsafe { crate::options::OPTIONS.as_ref() } + .map_or(DEFAULT_MAX_VERSIONS, |opts| opts.max_versions) +} + +/// Sentinel program counter stored in C frames when runtime checks are enabled. +const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { + Some(usize::MAX as *const VALUE) +} else { + None +}; + +/// Sentinel jit_return stored on ISEQ frame push when runtime checks are enabled. +const JIT_RETURN_POISON: Option<usize> = if cfg!(feature = "runtime_checks") { + Some(ZJIT_JIT_RETURN_POISON as usize) +} else { + None +}; + +/// Ephemeral code generation state +struct JITState { + /// ISEQ version that is being compiled, which will be used by PatchPoint + version: IseqVersionRef, + + /// Low-level IR Operands indexed by High-level IR's Instruction ID + opnds: Vec<Option<Opnd>>, + + /// Labels for each basic block indexed by the BlockId + labels: Vec<Option<Target>>, + + /// JIT entry point for the `iseq` + jit_entries: Vec<Rc<RefCell<JITEntry>>>, + + /// ISEQ calls that need to be compiled later + iseq_calls: Vec<IseqCallRef>, + + /// The number of native stack slots reserved for JITFrame. + /// gen_save_pc_for_gc() writes JITFrame into the allocated space. + jit_frame_size: usize, +} + +impl JITState { + /// Create a new JITState instance + fn new(version: IseqVersionRef, num_insns: usize, num_blocks: usize, jit_frame_size: usize) -> Self { + JITState { + version, + opnds: vec![None; num_insns], + labels: vec![None; num_blocks], + jit_entries: Vec::default(), + iseq_calls: Vec::default(), + jit_frame_size, + } + } + + /// Retrieve the output of a given instruction that has been compiled + fn get_opnd(&self, insn_id: InsnId) -> lir::Opnd { + self.opnds[insn_id.0].unwrap_or_else(|| panic!("Failed to get_opnd({insn_id})")) + } + + /// Find or create a label for a given BlockId + fn get_label(&mut self, asm: &mut Assembler, lir_block_id: lir::BlockId, hir_block_id: BlockId) -> Target { + // Extend labels vector if the requested index is out of bounds + if lir_block_id.0 >= self.labels.len() { + self.labels.resize(lir_block_id.0 + 1, None); + } + + match &self.labels[lir_block_id.0] { + Some(label) => label.clone(), + None => { + let label = asm.new_label(&format!("{hir_block_id}_{lir_block_id}")); + self.labels[lir_block_id.0] = Some(label.clone()); + label + } + } + } + +} + +impl Assembler { + /// Emit a conditional jump that splits the current block, creating a new + /// fall-through block for instructions that follow. + fn split_block_jump(&mut self, jit: &mut JITState, emit: impl FnOnce(&mut Assembler, Target), target: Target) { + let hir_block_id = self.current_block().hir_block_id; + let rpo_idx = self.current_block().rpo_index; + + let fall_through_target = self.new_block(hir_block_id, false, rpo_idx); + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![], + }; + emit(self, target); + self.jmp(Target::Block(fall_through_edge)); + + self.set_current_block(fall_through_target); + + let label = jit.get_label(self, fall_through_target, hir_block_id); + self.write_label(label); + } +} + +macro_rules! define_split_jumps { + ($($name:ident => $insn:ident),+ $(,)?) => { + impl Assembler { + $( + fn $name(&mut self, jit: &mut JITState, target: Target) { + self.split_block_jump(jit, |asm, target| asm.push_insn(lir::Insn::$insn(target)), target); + } + )+ + } + }; +} + +define_split_jumps! { + jbe => Jbe, + je => Je, + jge => Jge, + jl => Jl, + jne => Jne, + jnz => Jnz, + jo => Jo, + jo_mul => JoMul, + jz => Jz, +} + +/// CRuby API to compile a given ISEQ. +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See jit_compile_exception() for details. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exception: bool) -> *const u8 { + // Don't compile when there is insufficient native stack space + if unsafe { rb_ec_stack_check(ec as _) } != 0 { + incr_counter!(skipped_native_stack_full); + return std::ptr::null(); + } + + // Take a lock to avoid writing to ISEQ in parallel with Ractors. + // with_vm_lock() does nothing if the program doesn't use Ractors. + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let mut code_ptr = with_time_stat(compile_time_ns, || gen_iseq_entry_point(cb, iseq, jit_exception)); + + if let Err(err) = &code_ptr { + // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled. + // We assert only `jit_exception: false` cases until we support exception handlers. + if ZJITState::assert_compiles_enabled() && !jit_exception { + let iseq_location = iseq_get_location(iseq, 0); + panic!("Failed to compile: {iseq_location}: {err:?}"); + } + + // For --zjit-stats, generate an entry that just increments exit_compilation_failure and exits + if get_option!(stats) { + code_ptr = gen_compile_error_counter(cb, err); + } + } + + // Always mark the code region executable if asm.compile() has been used. + // We need to do this even if code_ptr is None because gen_iseq() may have already used asm.compile(). + cb.mark_all_executable(); + + code_ptr.map_or(std::ptr::null(), |ptr| ptr.raw_ptr(cb)) + }) +} + +/// Compile an entry point for a given ISEQ +fn gen_iseq_entry_point(cb: &mut CodeBlock, iseq: IseqPtr, jit_exception: bool) -> Result<CodePtr, CompileError> { + // We don't support exception handlers yet + if jit_exception { + return Err(CompileError::ExceptionHandler); + } + + let iseq_name = iseq_get_location(iseq, 0); + trace_compile_phase(&iseq_name, || { + // Compile ISEQ into High-level IR + let function = crate::stats::with_time_stat(Counter::compile_hir_time_ns, || compile_iseq(iseq).inspect_err(|_| { + incr_counter!(failed_iseq_count); + }))?; + + // Compile the High-level IR + let IseqCodePtrs { start_ptr, .. } = gen_iseq(cb, iseq, Some(&function)).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq, 0)); + })?; + + Ok(start_ptr) + }) +} + +/// Invalidate an ISEQ version and allow it to be recompiled on the next call. +/// Both PatchPoint invalidation and exit-profiling recompilation go through this +/// function, serving as the central point for all invalidation/recompile decisions. +/// +/// TODO: evolve this into a general `handle_event(iseq, event)` state machine that +/// handles all compile lifecycle events (interpreter profiles, JIT profiles, invalidation, +/// GC) so that all compile/recompile tuning decisions live in one place. +pub fn invalidate_iseq_version(cb: &mut CodeBlock, iseq: IseqPtr, version: &mut IseqVersionRef) { + let payload = get_or_create_iseq_payload(iseq); + if !unsafe { version.as_ref() }.is_invalidated() + && payload.versions.len() < max_iseq_versions() + { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(iseq) }; + + // Recompile JIT-to-JIT calls into the invalidated ISEQ + for incoming in unsafe { version.as_ref() }.incoming.iter() { + if let Err(err) = gen_iseq_call(cb, incoming) { + debug!("{err:?}: gen_iseq_call failed during invalidation: {}", iseq_get_location(incoming.iseq.get(), 0)); + } + } + } +} + +/// Stub a branch for a JIT-to-JIT call +pub fn gen_iseq_call(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<(), CompileError> { + trace_compile_phase("compile_stub", || { + // Compile a function stub + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the JIT-to-JIT call to call the stub + let stub_addr = stub_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); + asm.ccall_into(C_RET_OPND, stub_addr, vec![]); + }); + Ok(()) + }) +} + +/// Write an entry to the perf map in /tmp +fn register_with_perf(symbol_name: String, start_ptr: usize, code_size: usize) { + use std::io::Write; + let perf_map = format!("/tmp/perf-{}.map", std::process::id()); + let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(&perf_map) else { + debug!("Failed to open perf map file: {perf_map}"); + return; + }; + let mut file = std::io::BufWriter::new(file); + let Ok(_) = writeln!(file, "{start_ptr:#x} {code_size:#x} ZJIT: {symbol_name}") else { + debug!("Failed to write {symbol_name} to perf map file: {perf_map}"); + return; + }; +} + +/// Compile a shared JIT entry trampoline +pub fn gen_entry_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + // Set up registers for CFP, EC, SP, and basic block arguments + let mut asm = Assembler::new(); + asm.new_block_without_id("gen_entry_trampoline"); + gen_entry_prologue(&mut asm); + + // Jump to the first block using a call instruction. This trampoline is used + // as rb_zjit_func_t in jit_exec(), which takes (EC, CFP, rb_jit_func_t). + // So C_ARG_OPNDS[2] is rb_jit_func_t, which is (EC, CFP) -> VALUE. + let out = asm.ccall_reg(C_ARG_OPNDS[2], VALUE_BITS); + + // Restore registers for CFP, EC, and SP after use + asm_comment!(asm, "return to the interpreter"); + asm.frame_teardown(lir::JIT_PRESERVED_REGS); + asm.cret(out); + + let (code_ptr, gc_offsets) = asm.compile(cb)?; + assert!(gc_offsets.is_empty()); + if get_option!(perf).is_some() { + let start_ptr = code_ptr.raw_addr(cb); + let end_ptr = cb.get_write_ptr().raw_addr(cb); + let code_size = end_ptr - start_ptr; + register_with_perf("entry trampoline".into(), start_ptr, code_size); + } + Ok(code_ptr) +} + +/// Compile an ISEQ into machine code if not compiled yet +fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // Return an existing pointer if it's already compiled + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + match last_status { + Some(IseqStatus::Compiled(code_ptrs)) => return Ok(code_ptrs.clone()), + Some(IseqStatus::CantCompile(err)) => return Err(err.clone()), + _ => {}, + } + // If the ISEQ already has max versions, do not compile a new version. + if payload.versions.len() >= max_iseq_versions() { + return Err(CompileError::IseqVersionLimitReached); + } + + // Compile the ISEQ. When function is None, this is a lazy compile + // from a stub hit -- wrap in a trace event covering the full compile. + let mut version = IseqVersion::new(iseq); + let code_ptrs = if function.is_none() { + trace_compile_phase(&iseq_get_location(iseq, 0), || gen_iseq_body(cb, iseq, version, function)) + } else { + gen_iseq_body(cb, iseq, version, function) + }; + match &code_ptrs { + Ok(code_ptrs) => { + unsafe { version.as_mut() }.status = IseqStatus::Compiled(code_ptrs.clone()); + incr_counter!(compiled_iseq_count); + } + Err(err) => { + unsafe { version.as_mut() }.status = IseqStatus::CantCompile(err.clone()); + incr_counter!(failed_iseq_count); + } + } + payload.versions.push(version); + code_ptrs +} + +/// Compile an ISEQ into machine code +fn gen_iseq_body(cb: &mut CodeBlock, iseq: IseqPtr, mut version: IseqVersionRef, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // If we ran out of code region, we shouldn't attempt to generate new code. + if cb.has_dropped_bytes() { + return Err(CompileError::OutOfMemory); + } + + // Convert ISEQ into optimized High-level IR if not given + let function = match function { + Some(function) => function, + None => &crate::stats::with_time_stat(Counter::compile_hir_time_ns, || compile_iseq(iseq))?, + }; + + // Compile the High-level IR + let (iseq_code_ptrs, gc_offsets, iseq_calls) = + trace_compile_phase("codegen", || { + let (iseq_code_ptrs, gc_offsets, iseq_calls) = + crate::stats::with_time_stat(Counter::compile_lir_time_ns, || gen_function(cb, iseq, version, function))?; + + // Stub callee ISEQs for JIT-to-JIT calls + trace_compile_phase("generate_jit_jit_stubs", || { + for iseq_call in iseq_calls.iter() { + gen_iseq_call(cb, iseq_call)?; + } + Ok::<(), CompileError>(()) + })?; + + Ok((iseq_code_ptrs, gc_offsets, iseq_calls)) + })?; + + // Prepare for GC + unsafe { version.as_mut() }.outgoing.extend(iseq_calls); + append_gc_offsets(iseq, version, &gc_offsets); + Ok(iseq_code_ptrs) +} + +/// Compile a function +fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, function: &Function) -> Result<(IseqCodePtrs, Vec<CodePtr>, Vec<IseqCallRef>), CompileError> { + let (mut jit, asm) = trace_compile_phase("codegen", || { + let mut jit = JITState::new(version, function.num_insns(), function.num_blocks(), JIT_FRAME_SIZE); + let mut asm = Assembler::new_with_stack_slots(JIT_FRAME_SIZE); + + // Mapping from HIR block IDs to LIR block IDs. + // This is is a one-to-one mapping from HIR to LIR blocks used for finding + // jump targets in LIR (LIR should always jump to the head of an HIR block) + let mut hir_to_lir: Vec<Option<lir::BlockId>> = vec![None; function.num_blocks()]; + + let reverse_post_order = function.reverse_post_order(); + + // Create all LIR basic blocks corresponding to HIR basic blocks + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + // Skip the entries superblock -- it's an internal CFG artifact + if block_id == function.entries_block { continue; } + let lir_block_id = asm.new_block(block_id, function.is_entry_block(block_id), rpo_idx); + hir_to_lir[block_id.0] = Some(lir_block_id); + } + + // Compile each basic block + for &block_id in reverse_post_order.iter() { + // Skip the entries superblock -- it's an internal CFG artifact + if block_id == function.entries_block { continue; } + // Set the current block to the LIR block that corresponds to this + // HIR block. + let lir_block_id = hir_to_lir[block_id.0].unwrap(); + asm.set_current_block(lir_block_id); + + // Write a label to jump to the basic block + let label = jit.get_label(&mut asm, lir_block_id, block_id); + asm.write_label(label); + + let block = function.block(block_id); + asm_comment!( + asm, "{block_id}({}): {}", + block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", "), + iseq_get_location(iseq, block.insn_idx), + ); + + // Compile all parameters + for (idx, &insn_id) in block.params().enumerate() { + match function.find(insn_id) { + Insn::Param => { + jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx)); + }, + insn => unreachable!("Non-param insn found in block.params: {insn:?}"), + } + } + + // In JIT entry blocks, compile LoadArg instructions before other instructions + // so that calling convention registers are reserved early, like Param. + if function.is_entry_block(block_id) { + for &insn_id in block.insns() { + if let Insn::LoadArg { idx, .. } = function.find(insn_id) { + jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx as usize)); + } + } + } + + // Compile all instructions + for (insn_idx, &insn_id) in block.insns().enumerate() { + let insn = function.find(insn_id); + + match insn { + Insn::CondBranch { val, if_true, if_false } => { + let val_opnd = jit.get_opnd(val); + let true_target = hir_to_lir[if_true.target.0].unwrap(); + let false_target = hir_to_lir[if_false.target.0].unwrap(); + + let true_branch = lir::BranchEdge { + target: true_target, + args: if_true.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let false_branch = lir::BranchEdge { + target: false_target, + args: if_false.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + asm.test(val_opnd, val_opnd); + asm.push_insn(lir::Insn::Jnz(Target::Block(true_branch))); + asm.jmp(Target::Block(false_branch)); + + assert!(asm.current_block().insns.last().unwrap().is_terminator()); + } + Insn::Jump(target) => { + let lir_target = hir_to_lir[target.target.0].unwrap(); + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + asm.jmp(Target::Block(branch_edge)); + assert!(asm.current_block().insns.last().unwrap().is_terminator()); + + // Jump should always be the last instruction in an HIR block + assert!(insn_idx == block.insns().len() - 1, "Jump must be the last instruction in HIR block"); + }, + _ => { + // Start a new perf range for the HIR instruction. For now, we do this only for + // non-terminator instructions because LIR blocks must end with a terminator instruction. + let perf_symbol = if get_option!(perf) == Some(PerfMap::HIR) && !insn.is_terminator() { + let insn_name = format!("{insn}").split_whitespace().next().unwrap().to_string(); + Some(perf_symbol_range_start(&mut asm, &insn_name)) + } else { + None + }; + + let result = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn); + + // Close the current perf range for the HIR instruction. + if let Some(perf_symbol) = &perf_symbol { + perf_symbol_range_end(&mut asm, perf_symbol); + } + + if let Err(last_snapshot) = result { + debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); + gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); + let reason = match insn { + Insn::Throw { .. } => SideExitReason::UnhandledHIRThrow, + Insn::InvokeBuiltin { .. } => SideExitReason::UnhandledHIRInvokeBuiltin, + _ => SideExitReason::UnhandledHIRUnknown(insn_id), + }; + gen_side_exit(&mut jit, &mut asm, &reason, None, &function.frame_state(last_snapshot)); + // Don't bother generating code after a side-exit. We won't run it. + // TODO(max): Generate ud2 or equivalent. + break; + }; + // It's fine; we generated the instruction + } + } + } + // Blocks should always end with control flow + assert!(asm.current_block().insns.last().unwrap().is_terminator()); + } + + assert!(!asm.reverse_post_order().is_empty()); + + // Validate CFG invariants after HIR to LIR lowering + asm.validate_jump_positions(); + + (jit, asm) + }); + + // Generate code if everything can be compiled + let result = asm.compile(cb); + if let Ok((start_ptr, _)) = result { + if get_option!(perf) == Some(PerfMap::ISEQ) { + let start_usize = start_ptr.raw_addr(cb); + let end_usize = cb.get_write_ptr().raw_addr(cb); + let code_size = end_usize - start_usize; + let iseq_name = iseq_get_location(iseq, 0); + register_with_perf(iseq_name, start_usize, code_size); + } + if ZJITState::should_log_compiled_iseqs() { + let iseq_name = iseq_get_location(iseq, 0); + ZJITState::log_compile(iseq_name); + } + } + result.map(|(start_ptr, gc_offsets)| { + // Make sure jit_entry_ptrs can be used as a parallel vector to jit_entry_insns() + jit.jit_entries.sort_by_key(|jit_entry| jit_entry.borrow().jit_entry_idx); + + let jit_entry_ptrs = jit.jit_entries.iter().map(|jit_entry| + jit_entry.borrow().start_addr.get().expect("start_addr should have been set by pos_marker in gen_entry_point") + ).collect(); + (IseqCodePtrs { start_ptr, jit_entry_ptrs }, gc_offsets, jit.iseq_calls) + }) +} + +/// Compile an instruction +fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Result<(), InsnId> { + // Convert InsnId to lir::Opnd + macro_rules! opnd { + ($insn_id:ident) => { + jit.get_opnd($insn_id.clone()) + }; + } + + macro_rules! opnds { + ($insn_ids:ident) => { + { + $insn_ids.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect::<Vec<_>>() + } + }; + } + + macro_rules! no_output { + ($call:expr) => { + { let () = $call; return Ok(()); } + }; + } + + if !matches!(*insn, Insn::Snapshot { .. }) { + asm_comment!(asm, "Insn: {insn_id} {insn}"); + } + + let out_opnd = match insn { + &Insn::Const { val: Const::Value(val) } => gen_const_value(val), + &Insn::Const { val: Const::CPtr(val) } => gen_const_cptr(val), + &Insn::Const { val: Const::CInt64(val) } => gen_const_long(val), + &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val), + &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val), + &Insn::Const { val: Const::CUInt64(val) } => Opnd::UImm(val), + &Insn::Const { val: Const::CAttrIndex(val) } => gen_const_attr_index_t(val), + &Insn::Const { val: Const::CShape(val) } => { + assert_eq!(SHAPE_ID_NUM_BITS, 32); + gen_const_uint32(val.0) + } + Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), + Insn::NewArray { elements, state } => gen_new_array(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), + Insn::AdjustBounds { index, length } => gen_adjust_bounds(asm, opnd!(index), opnd!(length)), + Insn::ArrayAref { array, index, .. } => gen_array_aref(asm, opnd!(array), opnd!(index)), + Insn::ArrayAset { array, index, val } => { + no_output!(gen_array_aset(asm, opnd!(array), opnd!(index), opnd!(val))) + } + Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), + Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), + Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), + // concatstrings shouldn't have 0 strings + // If it happens we abort the compilation for now + Insn::StringConcat { strings, state, .. } if strings.is_empty() => return Err(*state), + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), + Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringEqual { left, right } => gen_string_equal(asm, opnd!(left), opnd!(right)), + Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), + Insn::Param => unreachable!("block.insns should not have Insn::Param"), + Insn::LoadArg { .. } => return Ok(()), // compiled in the LoadArg pre-pass above + Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment + &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, cd, std::ptr::null(), &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendDirect { cme, iseq, recv, ref args, kw_bits, block, state, .. } => gen_send_iseq_direct( + cb, jit, asm, cme, iseq, opnd!(recv), opnds!(args), + kw_bits, &function.frame_state(state), block, + ), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), + // Ensure we have enough room fit ec, self, and arguments + // TODO remove this check when we have stack args (we can use Time.new to test it) + Insn::InvokeBuiltin { bf, state, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return Err(*state), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, *leaf, opnds!(args)), + &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), + Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FloatAdd { recv, other, state } => gen_float_add(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatSub { recv, other, state } => gen_float_sub(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatMul { recv, other, state } => gen_float_mul(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatDiv { recv, other, state } => gen_float_div(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatToInt { recv, state } => gen_float_to_int(asm, opnd!(recv), &function.frame_state(*state)), + Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right)), + Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right)), + Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right)), + Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right)), + Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right)), + Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right)), + Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), + Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), + Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + Insn::IntAnd { left, right } => asm.and(opnd!(left), opnd!(right)), + Insn::IntOr { left, right } => gen_int_or(asm, opnd!(left), opnd!(right)), + &Insn::FixnumLShift { left, right, state } => { + // We only create FixnumLShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + } + &Insn::FixnumRShift { left, right } => { + // We only create FixnumRShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_rshift(asm, opnd!(left), shift_amount) + } + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), + Insn::IsNil { val } => gen_isnil(asm, opnd!(val)), + &Insn::IsMethodCfunc { val, cd, cfunc, state } => gen_is_method_cfunc(asm, opnd!(val), cd, cfunc, &function.frame_state(state)), + &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), + &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), + &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), + Insn::Test { val } => gen_test(asm, opnd!(val)), + Insn::RefineType { val, .. } => opnd!(val), + Insn::HasType { val, expected } => gen_has_type(jit, asm, opnd!(val), *expected), + Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + &Insn::GuardBitEquals { val, expected, reason, state, recompile } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, recompile, &function.frame_state(state)), + &Insn::GuardAnyBitSet { val, mask, reason, state, .. } => gen_guard_any_bit_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), + &Insn::GuardNoBitsSet { val, mask, reason, state, .. } => gen_guard_no_bits_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), + &Insn::GuardLess { left, right, reason, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), reason, &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + Insn::CCall { cfunc, recv, args, name, owner: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), + // Give up CCallWithFrame for 7+ args since asm.ccall() supports at most 6 args (recv + args). + // There's no test case for this because no core cfuncs have this many parameters. But C extensions could have such methods. + Insn::CCallWithFrame { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => + gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::CCallWithFrameTooManyArgs), + Insn::CCallWithFrame { cfunc, recv, name, args, cme, state, block, .. } => + gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)), + Insn::CCallVariadic { cfunc, recv, name, args, cme, state, block, return_type: _, elidable: _ } => { + gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + } + Insn::GetIvar { self_val, id, ic, state } => gen_getivar(asm, opnd!(self_val), *id, *ic, &function.frame_state(*state)), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), + &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), + Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, reason, *recompile, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, *value_type, &function.frame_state(*state)), + Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), + Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), + Insn::GetSpecialSymbol { symbol_type, state } => gen_getspecial_symbol(asm, *symbol_type, &function.frame_state(*state)), + Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), + &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), + Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), + Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)), + &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), + Insn::BreakPoint => no_output!(asm.breakpoint()), + Insn::Unreachable => no_output!(asm.abort()), + &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + Insn::LoadPC => gen_load_pc(asm), + Insn::LoadEC => gen_load_ec(), + Insn::LoadSP => gen_load_sp(), + &Insn::GetEP { level } => gen_get_ep(asm, level), + Insn::LoadSelf => gen_load_self(), + &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), + &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), + &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(jit, asm, opnd!(recv), opnd!(val), function.type_of(val))), + &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + &Insn::IsA { val, class } => gen_is_a(jit, asm, opnd!(val), opnd!(class)), + &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::Throw { state, .. } => return Err(state), + &Insn::CondBranch { .. } + | &Insn::Jump { .. } | Insn::Entries { .. } => unreachable!(), + }; + + assert!(insn.has_output(), "Cannot write LIR output of HIR instruction with no output: {insn}"); + + // If the instruction has an output, remember it in jit.opnds + jit.opnds[insn_id.0] = Some(out_opnd); + + Ok(()) +} + +// Get EP at `level` from CFP +fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { + // Load environment pointer EP from CFP into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let mut ep_opnd = asm.load(ep_opnd); + + for _ in 0..level { + // Get the previous EP from the current EP + // See GET_PREV_EP(ep) macro + // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03)) + const UNTAGGING_MASK: Opnd = Opnd::Imm(!0x03); + let offset = SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL; + ep_opnd = asm.load(Opnd::mem(64, ep_opnd, offset)); + ep_opnd = asm.and(ep_opnd, UNTAGGING_MASK); + } + + ep_opnd +} + +fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + // TODO: Specialize for immediate types + // Call rb_vm_objtostring(cfp, recv, cd) + let ret = asm_ccall!(asm, rb_vm_objtostring, CFP, val, Opnd::const_ptr(cd)); + + // TODO: Call `to_s` on the receiver if rb_vm_objtostring returns Qundef + // Need to replicate what CALL_SIMPLE_METHOD does + asm_comment!(asm, "side-exit if rb_vm_objtostring returns Qundef"); + asm.cmp(ret, Qundef.into()); + asm.je(jit, side_exit(jit, state, ObjToStringFallback)); + + ret +} + +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { + match op_type as defined_type { + DEFINED_YIELD => { + // `lep_level` was precomputed at HIR construction so we can materialize the local EP + // inline without walking the parent iseq chain here. + let lep = gen_get_ep(asm, lep_level); + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let pushval = asm.load(pushval.into()); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qnil.into(), pushval) + } + _ => { + // Save the PC and SP because the callee may allocate or call #respond_to? + gen_prepare_non_leaf_call(jit, asm, state); + + // TODO: Inline the cases for each op_type + // Call vm_defined(ec, reg_cfp, op_type, obj, v) + let def_result = asm_ccall!(asm, rb_vm_defined, EC, CFP, op_type.into(), obj.into(), tested_value); + + asm.cmp(def_result.with_num_bits(8), 0.into()); + asm.csel_ne(pushval.into(), Qnil.into()) + } + } +} + +/// Similar to gen_defined for DEFINED_YIELD +fn gen_is_block_given(asm: &mut Assembler, lep: Opnd) -> Opnd { + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qfalse.into(), Qtrue.into()) +} + +fn gen_unbox_fixnum(asm: &mut Assembler, val: Opnd) -> Opnd { + asm.rshift(val, Opnd::UImm(1)) +} + +/// Set a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. +/// We generate this instruction with level=0 only when the local variable is on the heap, so we +/// can't optimize the level=0 case using the SP register. +fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { + let local_ep_offset = c_int::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_write_to_parent_iseq_local_count); + } + let ep = gen_get_ep(asm, level); + + // When we've proved that we're writing an immediate, + // we can skip the write barrier. + if val_type.is_immediate() { + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(64, ep, offset), val); + } else { + // We're potentially writing a reference to an IMEMO/env object, + // so take care of the write barrier with a function. + let local_index = -local_ep_offset; + asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), val); + } +} + +/// Returns 1 (as CBool) when VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set; returns 0 otherwise. +fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { + asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) +} + +/// Get the block parameter as a Proc, write it to the environment, +/// and mark the flag as modified. +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + // Bail out if write barrier is required. + let ep = gen_get_ep(asm, level); + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); + asm.jnz(jit, side_exit(jit, state, SideExitReason::BlockParamWbRequired)); + + // Convert block handler to Proc. + let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let proc = asm_ccall!(asm, rb_vm_bh_to_procval, EC, block_handler); + + // Write Proc to EP and mark modified. + let ep = gen_get_ep(asm, level); + let local_ep_offset = c_int::try_from(ep_offset).unwrap_or_else(|_| { + panic!("Could not convert local_ep_offset {ep_offset} to i32") + }); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(VALUE_BITS, ep, offset), proc); + + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + let flags_val = asm.load(flags); + let modified = asm.or(flags_val, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.store(flags, modified); + + // Read the Proc from EP. + let ep = gen_get_ep(asm, level); + asm.load(Opnd::mem(VALUE_BITS, ep, offset)) +} + +fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jge(jit, side_exit(jit, state, reason)); + left +} + +fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jl(jit, side_exit(jit, state, SideExitReason::GuardGreaterEq)); + left +} + +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { + unsafe extern "C" { + fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; + } + + // Anything could be called on const_missing + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) +} + +fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { + unsafe extern "C" { + fn rb_vm_get_ev_const(ec: EcPtr, klass: VALUE, id: ID, allow_nil: VALUE) -> VALUE; + } + + // Constant lookup can raise and run arbitrary Ruby code via const_missing. + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_vm_get_ev_const, EC, klass, id.0.into(), allow_nil) +} + +fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { + let bit_test: u64 = 0x01 << (index + 1); + asm.test(val, bit_test.into()); + asm.csel_z(Qtrue.into(), Qfalse.into()) +} + +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec<Opnd>) -> lir::Opnd { + assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, + "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", + unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }, + bf.argc); + if leaf { + gen_prepare_leaf_call_with_gc(asm, state); + } else { + // Anything can happen inside builtin functions + gen_prepare_non_leaf_call(jit, asm, state); + } + + let mut cargs = vec![EC]; + cargs.extend(args); + + asm.count_call_to(unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }); + asm.ccall(bf.func_ptr as *const u8, cargs) +} + +/// Record a patch point that should be invalidated on a given invariant +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { + let invariant = *invariant; + let exit = build_side_exit(jit, state); + + // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. + asm.patch_point(Target::SideExit { exit, reason: PatchPoint(invariant) }, invariant, jit.version); +} + +/// This is used by scratch_split to lower PatchPoint into PadPatchPoint and PosMarker. +/// It's called at scratch_split so that we can use the Label after side-exit deduplication in compile_exits. +pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invariant, version: IseqVersionRef) { + let Target::Label(exit_label) = *target else { + unreachable!("PatchPoint's target should have been lowered to Target::Label by compile_exits: {target:?}"); + }; + + // Fill nop instructions if the last patch point is too close. + asm.pad_patch_point(); + + // Remember the current address as a patch point + asm.pos_marker(move |code_ptr, cb| { + let side_exit_ptr = cb.resolve_label(exit_label); + match invariant { + Invariant::BOPRedefined { klass, bop } => { + track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, version); + } + Invariant::MethodRedefined { klass: _, method: _, cme } => { + track_cme_assumption(cme, code_ptr, side_exit_ptr, version); + } + Invariant::StableConstantNames { idlist } => { + track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, version); + } + Invariant::NoTracePoint => { + track_no_trace_point_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoEPEscape(iseq) => { + track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, version); + } + Invariant::SingleRactorMode => { + track_single_ractor_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoSingletonClass { klass } => { + track_no_singleton_class_assumption(klass, code_ptr, side_exit_ptr, version); + } + Invariant::RootBoxOnly => { + track_root_box_assumption(code_ptr, side_exit_ptr, version); + } + } + }); +} + +/// Generate code for a C function call that pushes a frame +fn gen_ccall_with_frame( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + block: Option<BlockHandler>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + let caller_stack_size = state.stack().len() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (and block arguments if any) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(BlockHandler::BlockIseq(block_iseq)) = block { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + write_block_code: false, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, cfunc_args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know +/// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. +fn gen_ccall(asm: &mut Assembler, cfunc: *const u8, name: ID, recv: Opnd, args: Vec<Opnd>) -> lir::Opnd { + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + asm.ccall(cfunc, cfunc_args) +} + +// Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). +// VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. +// rb_captured_block->code.iseq aliases with cfp->block_code. +fn gen_block_handler_specval(asm: &mut Assembler, blockiseq: IseqPtr) -> lir::Opnd { + asm.store(Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(blockiseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) +} + +/// Generate code for a variadic C function call +/// func(int argc, VALUE *argv, VALUE recv) +fn gen_ccall_variadic( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + block: Option<BlockHandler>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + + // Compute the caller's stack size after consuming recv and args. + // state.stack() includes recv + args, so subtract both. + let caller_stack_size = state.stack_size() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (like gen_ccall_with_frame does) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(BlockHandler::BlockIseq(blockiseq)) = block { + gen_block_handler_specval(asm, blockiseq) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + write_block_code: false, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let argv_ptr = gen_push_opnds(jit, asm, &args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, vec![args.len().into(), argv_ptr, recv]); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Emit an uncached instance variable lookup +fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, state: &FrameState) -> Opnd { + if ic.is_null() { + asm_ccall!(asm, rb_ivar_get, recv, id.0.into()) + } else { + let iseq = Opnd::Value(state.iseq.into()); + asm_ccall!(asm, rb_vm_getinstancevariable, iseq, recv, id.0.into(), Opnd::const_ptr(ic)) + } +} + +/// Emit an uncached instance variable store +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { + // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. + gen_prepare_non_leaf_call(jit, asm, state); + if ic.is_null() { + asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); + } else { + let iseq = Opnd::Value(state.iseq.into()); + asm_ccall!(asm, rb_vm_setinstancevariable, iseq, recv, id.0.into(), val, Opnd::const_ptr(ic)); + } +} + +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) +} + +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); +} + +/// Look up global variables +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { + // `Warning` module's method `warn` can be called when reading certain global variables + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_get, id.0.into()) +} + +/// Intern a string +fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_str_intern, val) +} + +/// Set global variables +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { + // When trace_var is used, setting a global variable can cause exceptions + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_set, id.0.into(), val); +} + +/// Side-exit into the interpreter +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, recompile: Option<Recompile>, state: &FrameState) { + asm.jmp(side_exit_with_recompile(jit, state, *reason, recompile)); +} + +/// Emit a special object lookup +fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, value_type: SpecialObjectType, state: &FrameState) -> Opnd { + // rb_vm_get_special_object for CBASE/CONST_BASE can call rb_singleton_class, + // which allocates (may trigger GC) and can raise TypeError on non-class + // receivers (e.g. `123.instance_eval { Const = 1 }`). Treat as non-leaf so + // the PC is saved for GC and stack/locals are spilled for rescue. + gen_prepare_non_leaf_call(jit, asm, state); + + // Get the EP of the current CFP and load it into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let ep_reg = asm.load(ep_opnd); + + asm_ccall!(asm, rb_vm_get_special_object, ep_reg, Opnd::UImm(u64::from(value_type))) +} + +fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol, state: &FrameState) -> Opnd { + // rb_backref_get reaches rb_vm_svar_lep, which calls CFP_PC/CFP_ISEQ on the + // current frame, so the PC must be saved before the call. + gen_prepare_leaf_call_with_gc(asm, state); + + // Fetch a "special" backref based on the symbol type + let backref = asm_ccall!(asm, rb_backref_get,); + + match symbol_type { + SpecialBackrefSymbol::LastMatch => { + asm_ccall!(asm, rb_reg_last_match, backref) + } + SpecialBackrefSymbol::PreMatch => { + asm_ccall!(asm, rb_reg_match_pre, backref) + } + SpecialBackrefSymbol::PostMatch => { + asm_ccall!(asm, rb_reg_match_post, backref) + } + SpecialBackrefSymbol::LastGroup => { + asm_ccall!(asm, rb_reg_match_last, backref) + } + } +} + +fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> Opnd { + // rb_backref_get reaches rb_vm_svar_lep, which calls CFP_PC/CFP_ISEQ on the + // current frame, so the PC must be saved before the call. + gen_prepare_leaf_call_with_gc(asm, state); + + // Fetch the N-th match from the last backref based on type shifted by 1 + let backref = asm_ccall!(asm, rb_backref_get,); + + asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) +} + +fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameState) { + // Check for interrupts + // see RUBY_VM_CHECK_INTS(ec) macro + asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); + // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, + // signal_exec, or rb_postponed_job_flush. + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); + asm.test(interrupt_flag, interrupt_flag); + asm.jnz(jit, side_exit(jit, state, SideExitReason::Interrupt)); +} + +fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_resurrect, val) +} + +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aref, hash, key) +} + +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aset, hash, key, val); +} + +fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_push, array, val); +} + +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) +} + +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) +} + +fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) -> lir::Opnd { + asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) +} + +fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { + // rb_vm_check_match is not leaf unless flag is VM_CHECKMATCH_TYPE_WHEN. + // See also: leafness_of_checkmatch() and check_match() + if flag != VM_CHECKMATCH_TYPE_WHEN { + gen_prepare_non_leaf_call(jit, asm, state); + } + + unsafe extern "C" { + fn rb_vm_check_match(ec: EcPtr, target: VALUE, pattern: VALUE, flag: u32) -> VALUE; + } + + asm_ccall!(asm, rb_vm_check_match, EC, target, pattern, flag.into()) +} + +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_ary_concat, left, right); +} + +fn gen_load_pc(asm: &mut Assembler) -> Opnd { + asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC)) +} + +fn gen_load_ec() -> Opnd { + EC +} + +fn gen_load_sp() -> Opnd { + SP +} + +fn gen_load_self() -> Opnd { + Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF) +} + +fn gen_load_field(asm: &mut Assembler, recv: Opnd, id: FieldName, offset: i32, return_type: Type) -> Opnd { + gen_incr_counter(asm, Counter::load_field_count); + asm_comment!(asm, "Load field id={id} offset={offset}"); + let recv = asm.load_mem(recv); + asm.load(Opnd::mem(return_type.num_bits(), recv, offset)) +} + +fn gen_store_field(asm: &mut Assembler, recv: Opnd, id: FieldName, offset: i32, val: Opnd, val_type: Type) { + gen_incr_counter(asm, Counter::store_field_count); + asm_comment!(asm, "Store field id={id} offset={offset}"); + let recv = asm.load_mem(recv); + asm.store(Opnd::mem(val_type.num_bits(), recv, offset), val); +} + +fn gen_write_barrier(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, val: Opnd, val_type: Type) { + // See RB_OBJ_WRITE/rb_obj_write: it's just assignment and rb_obj_written(). + // rb_obj_written() does: if (!RB_SPECIAL_CONST_P(val)) { rb_gc_writebarrier(recv, val); } + if !val_type.is_immediate() { + asm_comment!(asm, "Write barrier"); + let recv = asm.load_mem(recv); + + // Create a result block that all paths converge to + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = Target::Block(lir::BranchEdge { target: result_block, args: vec![] }); + + // If non-false immediate, don't fire write barrier + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, result_edge.clone()); + + // If false, don't fire write barrier + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge.clone()); + + // Heap object; fire the write barrier + asm_ccall!(asm, rb_gc_writebarrier, recv, val); + asm.jmp(result_edge); + + // Join block + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); + } +} + +/// Compile an interpreter entry block to be inserted into an ISEQ +fn gen_entry_prologue(asm: &mut Assembler) { + asm_comment!(asm, "ZJIT entry trampoline"); + // Save the registers we'll use for CFP, EP, SP + asm.frame_setup(lir::JIT_PRESERVED_REGS); + + // EC and CFP are passed as arguments + asm.mov(EC, C_ARG_OPNDS[0]); + asm.mov(CFP, C_ARG_OPNDS[1]); + + // Load the current SP from the CFP into REG_SP + asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +} + +/// Compile a constant +fn gen_const_value(val: VALUE) -> lir::Opnd { + // Just propagate the constant value and generate nothing + Opnd::Value(val) +} + +/// Compile Const::CPtr +fn gen_const_cptr(val: *const u8) -> lir::Opnd { + Opnd::const_ptr(val) +} + +fn gen_const_long(val: i64) -> lir::Opnd { + Opnd::Imm(val) +} + +fn gen_const_uint16(val: u16) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +fn gen_const_uint32(val: u32) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +fn gen_const_attr_index_t(val: attr_index_t) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +/// Compile a basic block argument +fn gen_param(asm: &mut Assembler, _idx: usize) -> lir::Opnd { + let vreg = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(vreg); + vreg +} + +/// Compile a dynamic dispatch with block +fn gen_send( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_send, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch with `...` +fn gen_send_forward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_sendforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch without block +fn gen_send_without_block( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_send_without_block, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +/// Compile a direct call to an ISEQ method. +/// If `block_handler` is provided, it's used as the specval for the new frame (for forwarding blocks). +/// Otherwise, `VM_BLOCK_HANDLER_NONE` is used. +fn gen_send_iseq_direct( + cb: &mut CodeBlock, + jit: &mut JITState, + asm: &mut Assembler, + cme: *const rb_callable_method_entry_t, + iseq: IseqPtr, + recv: Opnd, + args: Vec<Opnd>, + kw_bits: u32, + state: &FrameState, + block: Option<BlockHandler>, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::iseq_optimized_send_count); + + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); + let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); + gen_stack_overflow_check(jit, asm, state, stack_growth); + + // Save cfp->pc and cfp->sp for the caller frame + // Can't use gen_prepare_non_leaf_call because we need special SP math. + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver + + gen_spill_locals(jit, asm, state); + gen_spill_stack(jit, asm, state); + + // This mirrors vm_caller_setup_arg_block() in for the `blockiseq != NULL` case. + // The HIR specialization guards ensure we will only reach here for literal blocks, + // not &block forwarding, &:foo, etc. Thise are rejected in `type_specialize` by + // `unspecializable_call_type`. + let block_handler = block.map(|bh| match bh { BlockHandler::BlockIseq(b) => gen_block_handler_specval(asm, b), BlockHandler::BlockArg => unreachable!("BlockArg in gen_send_iseq_direct") }); + + let callee_is_bmethod = VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(cme) }; + + let (frame_type, specval) = if callee_is_bmethod { + // Extract EP from the Proc instance + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let bmethod_frame_type = VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA; + // Tag the captured EP like VM_GUARDED_PREV_EP() in vm_call_iseq_bmethod() + let bmethod_specval = (capture.ep.addr() | 1).into(); + (bmethod_frame_type, bmethod_specval) + } else { + let specval = block_handler.unwrap_or_else(|| VM_BLOCK_HANDLER_NONE.into()); + (VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, specval) + }; + + // Set up the new frame + // TODO: Lazily materialize caller frames on side exits or when needed + gen_push_frame(asm, args.len(), state, ControlFrame { + recv, + iseq: Some(iseq), + cme, + frame_type, + specval, + write_block_code: iseq_may_write_block_code(iseq), + }); + + // Write "keyword_bits" to the callee's frame if the callee accepts keywords. + // This is a synthetic local/parameter that the callee reads via checkkeyword to determine + // which optional keyword arguments need their defaults evaluated. + // We write this to the local table slot at bits_start so that: + // 1. The interpreter can read it via checkkeyword if we side-exit + // 2. The JIT entry can read it from the callee frame slot + if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + let bits_start = unsafe { (*keyword).bits_start } as usize; + let unspecified_bits = VALUE::fixnum_from_usize(kw_bits as usize); + let bits_offset = (state.stack().len() - args.len() + bits_start) * SIZEOF_VALUE; + asm_comment!(asm, "write keyword bits to callee frame"); + asm.store(Opnd::mem(64, SP, bits_offset as i32), unspecified_bits.into()); + } + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (state.stack().len() + local_size - args.len() + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let params = unsafe { iseq.params() }; + + // For &block, the JIT entrypoint expects the block_handler as an argument + // This HIR param is not actually used, things read from specval from the VM frame today. + // TODO: Remove unused param from HIR, or pass specval through c_args. + // See https://github.com/ruby/ruby/pull/15911#discussion_r2710544982 + let needs_block = params.flags.has_block() != 0; + + // Set up arguments + let mut c_args = Vec::with_capacity({ + // This is a heuristic to avoid re-allocation, not necessary for correctness + 1 /* recv */ + args.len() + if needs_block { 1 } else { 0 } + }); + c_args.push(recv); + c_args.extend(&args); + if needs_block { + if callee_is_bmethod { + // For bmethods, specval is the captured EP, not the block handler. + // The block param needs nil (no block) or a Proc value. + assert!(block_handler.is_none(), "at the moment, HIR builder never emits a direct send for a to-bmethod send-with-literal-block"); + c_args.push(Qnil.into()); + } else { + c_args.push(specval); + } + } + + let num_optionals_passed = if params.flags.has_opt() != 0 { + // See vm_call_iseq_setup_normal_opt_start in vm_inshelper.c + let lead_num = params.lead_num as u32; + let opt_num = params.opt_num as u32; + let post_num = params.post_num as u32; + let keyword = params.keyword; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } } as u32; + assert!(args.len() as u32 <= lead_num + opt_num + post_num + kw_total_num); + // For computing optional positional entry point, only count positional args + // and exclude the always-present lead and post slots. + let positional_argc = args.len() as u32 - kw_total_num; + let num_optionals_passed = positional_argc.saturating_sub(lead_num + post_num); + num_optionals_passed + } else { + 0 + }; + + // Make a method call. The target address will be rewritten once compiled. + let iseq_call = IseqCall::new(iseq, num_optionals_passed.try_into().expect("checked in HIR"), args.len().try_into().expect("checked in HIR")); + let dummy_ptr = cb.get_write_ptr().raw_ptr(cb); + jit.iseq_calls.push(iseq_call.clone()); + let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call); + + // If a callee side-exits, i.e. returns Qundef, propagate the return value to the caller. + // The caller will side-exit the callee into the interpreter. + // TODO: Let side exit code pop all JIT frames to optimize away this cmp + je. + asm_comment!(asm, "side-exit if callee side-exits"); + asm.cmp(ret, Qundef.into()); + // Restore the C stack pointer on exit + asm.je(jit, ZJITState::get_exit_trampoline().into()); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + ret +} + +/// Compile for invokeblock +fn gen_invokeblock( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeblock"); + unsafe extern "C" { + fn rb_vm_invokeblock(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokeblock, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +/// Compile invokeblock for IFUNC block handlers. +/// Calls rb_vm_yield_with_cfunc directly. +fn gen_invokeblock_ifunc( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + block_handler: Opnd, + args: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + let _ = cd; // cd is not needed for the direct call + + gen_prepare_non_leaf_call(jit, asm, state); + + // Push args to memory so we can pass argv pointer + let argv_ptr = gen_push_opnds(jit, asm, &args); + + // Untag the block handler to get the captured block pointer + // captured = block_handler & ~0x3 + asm_comment!(asm, "untag block handler to get captured block"); + let captured = asm.and(block_handler, Opnd::Imm(!0x3i64)); + + asm_comment!(asm, "call rb_vm_yield_with_cfunc"); + unsafe extern "C" { + fn rb_vm_yield_with_cfunc( + ec: EcPtr, + captured: VALUE, + argc: i32, + argv: *const VALUE, + ) -> VALUE; + } + asm_ccall!(asm, rb_vm_yield_with_cfunc, EC, captured, (args.len() as i64).into(), argv_ptr) +} + +fn gen_invokeproc( + jit: &mut JITState, + asm: &mut Assembler, + recv: Opnd, + args: Vec<Opnd>, + kw_splat: bool, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeproc"); + + let argv_ptr = gen_push_opnds(jit, asm, &args); + let kw_splat_opnd = Opnd::Imm(i64::from(kw_splat)); + asm_ccall!( + asm, + rb_optimized_call, + recv, + EC, + args.len().into(), + argv_ptr, + kw_splat_opnd, + VM_BLOCK_HANDLER_NONE.into() + ) +} + +/// Compile a dynamic dispatch for `super` +fn gen_invokesuper( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch"); + unsafe extern "C" { + fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuper, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch for `super` with `...` +fn gen_invokesuperforward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch (forwarding)"); + unsafe extern "C" { + fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuperforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a string resurrection +fn gen_string_copy(asm: &mut Assembler, recv: Opnd, chilled: bool, state: &FrameState) -> Opnd { + // TODO: split rb_ec_str_resurrect into separate functions + gen_prepare_leaf_call_with_gc(asm, state); + let chilled = if chilled { Opnd::Imm(1) } else { Opnd::Imm(0) }; + asm_ccall!(asm, rb_ec_str_resurrect, EC, recv, chilled) +} + +fn gen_string_equal(asm: &mut Assembler, left: Opnd, right: Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_yarv_str_eql_internal, left, right) +} + +/// Compile an array duplication instruction +fn gen_array_dup( + asm: &mut Assembler, + val: lir::Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_ary_resurrect, val) +} + +/// Compile a new array instruction +fn gen_new_array( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + let num: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + if elements.is_empty() { + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, 0i64.into(), Opnd::UImm(0)) + } else { + let argv = gen_push_opnds(jit, asm, &elements); + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, num.into(), argv) + } +} + +/// Adjust potentially-negative index by the given length, returning the adjusted index. If still negative, +/// return a negative number, which indicates the index is still out-of-bounds. +fn gen_adjust_bounds(asm: &mut Assembler, index: Opnd, length: Opnd) -> lir::Opnd { + let adjusted = asm.add(index, length); + asm.test(index, index); + asm.csel_l(adjusted, index) +} + +/// Compile array access (`array[index]`) +fn gen_array_aref( + asm: &mut Assembler, + array: Opnd, + index: Opnd, +) -> lir::Opnd { + let unboxed_idx = asm.load_mem(index); + let array = asm.load_mem(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.load(Opnd::mem(VALUE_BITS, elem_ptr, 0)) +} + +fn gen_array_aset( + asm: &mut Assembler, + array: Opnd, + index: Opnd, + val: Opnd, +) { + let unboxed_idx = asm.load_mem(index); + let array = asm.load_mem(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.store(Opnd::mem(VALUE_BITS, elem_ptr, 0), val); +} + +fn gen_array_pop(asm: &mut Assembler, array: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_pop, array) +} + +fn gen_array_length(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let array = asm.load_mem(array); + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + let embedded_len = asm.and(flags, (RARRAY_EMBED_LEN_MASK as u64).into()); + let embedded_len = asm.rshift(embedded_len, (RARRAY_EMBED_LEN_SHIFT as u64).into()); + // cmov between the embedded length and heap length depending on the embed flag + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_len = Opnd::mem(c_long::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_LEN); + asm.csel_nz(embedded_len, heap_len) +} + +fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_ptr = Opnd::mem(usize::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_PTR); + let embedded_ptr = asm.lea(Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RARRAY_AS_ARY)); + asm.csel_nz(embedded_ptr, heap_ptr) +} + +/// Compile opt_newarray_hash - create a hash from array elements +fn gen_opt_newarray_hash( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + // `Array#hash` will hash the elements of the array. + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_hash(ec: EcPtr, array_len: u32, elts: *const VALUE) -> VALUE; + } + + asm.ccall( + rb_vm_opt_newarray_hash as *const u8, + vec![EC, (array_len as u32).into(), elements_ptr], + ) +} + +/// Compile ArrayMax - find the maximum element among array elements +fn gen_array_max( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(VALUE_BITS, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_max(ec: EcPtr, num: u32, elts: *const VALUE) -> VALUE; + } + + asm.ccall( + rb_vm_opt_newarray_max as *const u8, + vec![EC, array_len.into(), elements_ptr], + ) +} + +/// Find the minimum element among array elements +fn gen_array_min( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(VALUE_BITS, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_min(ec: EcPtr, num: u32, elts: *const VALUE) -> VALUE; + } + + asm.ccall( + rb_vm_opt_newarray_min as *const u8, + vec![EC, array_len.into(), elements_ptr], + ) +} + +fn gen_array_include( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the target. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 1; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_include_p(ec: EcPtr, num: c_long, elts: *const VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_include_p, + EC, array_len.into(), elements_ptr, target + ) +} + +fn gen_array_pack_buffer( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + fmt: Opnd, + buffer: Option<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the fmt, and optionally the buffer. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = if buffer.is_some() { + state.stack().len() - elements.len() - 2 + } else { + state.stack().len() - elements.len() - 1 + }; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_pack_buffer(ec: EcPtr, num: c_long, elts: *const VALUE, fmt: VALUE, buffer: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_pack_buffer, + EC, array_len.into(), elements_ptr, fmt, buffer.unwrap_or_else(|| Qundef.into()) + ) +} + +fn gen_dup_array_include( + jit: &JITState, + asm: &mut Assembler, + ary: VALUE, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + unsafe extern "C" { + fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_duparray_include_p, + EC, ary.into(), target + ) +} + +fn gen_is_a(jit: &mut JITState, asm: &mut Assembler, obj: Opnd, class: Opnd) -> lir::Opnd { + let builtin_type = match class { + Opnd::Value(value) if value == unsafe { rb_cString } => Some(RUBY_T_STRING), + Opnd::Value(value) if value == unsafe { rb_cArray } => Some(RUBY_T_ARRAY), + Opnd::Value(value) if value == unsafe { rb_cHash } => Some(RUBY_T_HASH), + _ => None + }; + + if let Some(builtin_type) = builtin_type { + asm_comment!(asm, "IsA by matching builtin type"); + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + + // Create a result block that all paths converge to + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = |v| Target::Block(lir::BranchEdge { + target: result_block, + args: vec![v], + }); + + let val = asm.load_mem(obj); + + // Immediate -> definitely not String/Array/Hash + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, result_edge(Qfalse.into())); + + // Qfalse -> definitely not String/Array/Hash + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge(Qfalse.into())); + + // Heap object -> check builtin type + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let obj_builtin_type = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(obj_builtin_type, Opnd::UImm(builtin_type as u64)); + let result = asm.csel_e(Qtrue.into(), Qfalse.into()); + asm.jmp(result_edge(result)); + + // Result block -- receives the value via block parameter (phi node) + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); + let param = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(param); + param + } else { + asm_ccall!(asm, rb_obj_is_kind_of, obj, class) + } +} + +/// Compile a new hash instruction +fn gen_new_hash( + jit: &mut JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + if elements.is_empty() { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_new,) + } else { + gen_prepare_non_leaf_call(jit, asm, state); + + let argv = gen_push_opnds(jit, asm, &elements); + asm_ccall!(asm, rb_hash_new_with_bulk_insert, elements.len().into(), argv) + } +} + +/// Compile a new range instruction +fn gen_new_range( + jit: &JITState, + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + // Sometimes calls `low.<=>(high)` + gen_prepare_non_leaf_call(jit, asm, state); + + // Call rb_range_new(low, high, flag) + asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) +} + +fn gen_new_range_fixnum( + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) +} + +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_obj_alloc, val) +} + +fn gen_object_alloc_class(asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { + // Allocating an object for a known class with default allocator is leaf; see doc for + // `ObjectAllocClass`. + gen_prepare_leaf_call_with_gc(asm, state); + if unsafe { rb_zjit_class_has_default_allocator(class) } { + // TODO(max): inline code to allocate an instance + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + } else { + assert!(class_has_leaf_allocator(class), "class passed to ObjectAllocClass must have a leaf allocator"); + let alloc_func = unsafe { rb_zjit_class_get_alloc_func(class) }; + assert!(alloc_func.is_some(), "class {} passed to ObjectAllocClass must have an allocator", get_class_name(class)); + asm_comment!(asm, "call allocator for class {}", get_class_name(class)); + asm.count_call_to(&format!("{}::allocator", get_class_name(class))); + asm.ccall(alloc_func.unwrap() as *const u8, vec![class.into()]) + } +} + +/// Compile a frame setup. If jit_entry_idx is Some, remember the address of it as a JIT entry. +fn gen_entry_point(jit: &mut JITState, asm: &mut Assembler, jit_entry_idx: Option<usize>) { + if let Some(jit_entry_idx) = jit_entry_idx { + let jit_entry = JITEntry::new(jit_entry_idx); + jit.jit_entries.push(jit_entry.clone()); + asm.pos_marker(move |code_ptr, _| { + jit_entry.borrow_mut().start_addr.set(Some(code_ptr)); + }); + } + asm.frame_setup(&[]); + + // Publish the JITFrame slot's location via cfp->jit_return. The slot at + // [NATIVE_BASE_PTR - 8] is left uninitialized here; the JIT design relies on + // gen_save_pc_for_gc() to populate it before any C call, and on cross-ractor + // barriers ensuring that no other ractor scans this CFP before such a call. + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), NATIVE_BASE_PTR); + + // Poison the JITFrame slot. It should be read only after gen_save_pc_for_gc(). + if let Some(jit_return_poison) = JIT_RETURN_POISON { + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), jit_return_poison.into()); + } +} + +/// Compile code that exits from JIT code with a return value +fn gen_return(asm: &mut Assembler, val: lir::Opnd) { + // Pop the current frame (ec->cfp++) + // Note: the return PC is already in the previous CFP + asm_comment!(asm, "pop stack frame"); + let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, incr_cfp); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + // Order here is important. Because we're about to tear down the frame, + // we need to load the return value, which might be part of the frame. + asm.load_into(C_RET_OPND, val); + + // Return from the function + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(C_RET_OPND); +} + +/// Compile Fixnum + Fixnum +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Add left + right and test for overflow + let left_untag = asm.sub(left, Opnd::Imm(1)); + let out_val = asm.add(left_untag, right); + asm.jo(jit, side_exit(jit, state, FixnumAddOverflow)); + + out_val +} + +/// Compile Fixnum - Fixnum +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Subtract left - right and test for overflow + let val_untag = asm.sub(left, right); + asm.jo(jit, side_exit(jit, state, FixnumSubOverflow)); + asm.add(val_untag, Opnd::Imm(1)) +} + +/// Compile Fixnum * Fixnum +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Do some bitwise gymnastics to handle tag bits + // x * y is translated to (x >> 1) * (y - 1) + 1 + let left_untag = asm.rshift(left, Opnd::UImm(1)); + let right_untag = asm.sub(right, Opnd::UImm(1)); + let out_val = asm.mul(left_untag, right_untag); + + // Test for overflow + asm.jo_mul(jit, side_exit(jit, state, FixnumMultOverflow)); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum / Fixnum +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + // Side exit if rhs is 0 + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(jit, side_exit(jit, state, FixnumDivByZero)); + asm_ccall!(asm, rb_jit_fix_div_fix, left, right) +} + +/// Compile Float + Float +fn gen_float_add(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_plus, recv, other) +} + +/// Compile Float - Float +fn gen_float_sub(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_minus, recv, other) +} + +/// Compile Float * Float +fn gen_float_mul(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_mul, recv, other) +} + +/// Compile Float / Float +fn gen_float_div(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_div, recv, other) +} + +/// Compile Float#to_i (truncate to integer) +fn gen_float_to_int(asm: &mut Assembler, recv: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_flo_to_i, recv) +} + +/// Compile Fixnum == Fixnum +fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_e(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum != Fixnum +fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum < Fixnum +fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_l(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum <= Fixnum +fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_le(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum > Fixnum +fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_g(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum >= Fixnum +fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ge(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum & Fixnum +fn gen_fixnum_and(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.and(left, right) +} + +/// Compile Fixnum | Fixnum +fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) +} + +/// Compile C integer | C integer. +fn gen_int_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) +} + +/// Compile Fixnum ^ Fixnum +fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + // XOR and then re-tag the resulting fixnum + let out_val = asm.xor(left, right); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum << Fixnum +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit + let out_val = asm.lshift(in_val, shift_amount.into()); + let unshifted = asm.rshift(out_val, shift_amount.into()); + asm.cmp(in_val, unshifted); + asm.jne(jit, side_exit(jit, state, FixnumLShiftOverflow)); + // Re-tag the output value + let out_val = asm.add(out_val, 1.into()); + out_val +} + +/// Compile Fixnum >> Fixnum +fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let result = asm.rshift(left, shift_amount.into()); + // Re-tag the output value + asm.or(result, 1.into()) +} + +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Check for left % 0, which raises ZeroDivisionError + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(jit, side_exit(jit, state, FixnumModByZero)); + asm_ccall!(asm, rb_fix_mod_fix, left, right) +} + +fn gen_fixnum_aref(asm: &mut Assembler, recv: lir::Opnd, index: lir::Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_fix_aref, recv, index) +} + +// Compile val == nil +fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.cmp(val, Qnil.into()); + // TODO: Implement and use setcc + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_method_cfunc(asm: &mut Assembler, val: lir::Opnd, cd: *const rb_call_data, cfunc: *const u8, state: &FrameState) -> lir::Opnd { + unsafe extern "C" { + fn rb_vm_method_cfunc_is(iseq: IseqPtr, cd: *const rb_call_data, recv: VALUE, cfunc: *const u8) -> VALUE; + } + asm_ccall!(asm, rb_vm_method_cfunc_is, VALUE::from(state.iseq).into(), Opnd::const_ptr(cd), val, Opnd::const_ptr(cfunc)) +} + +fn gen_is_bit_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_bit_not_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.test(val, val); + asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) +} + +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Load the value, then test for overflow and tag it + let val = asm.load_mem(val); + let shifted = asm.lshift(val, Opnd::UImm(1)); + asm.jo(jit, side_exit(jit, state, BoxFixnumOverflow)); + asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) +} + +fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_obj_as_string_result, str, val) +} + +/// Evaluate if a value is truthy +/// Produces a CBool type (0 or 1) +/// In Ruby, only nil and false are falsy +/// Everything else evaluates to true +fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + // Test if any bit (outside of the Qnil bit) is on + // See RB_TEST(), include/ruby/internal/special_consts.h + asm.test(val, Opnd::Imm(!Qnil.as_i64())); + asm.csel_e(0.into(), 1.into()) +} + +fn gen_has_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, ty: Type) -> lir::Opnd { + if ty.is_subtype(types::Fixnum) { + asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::Flonum) { + // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG + let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); + asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::StaticSymbol) { + // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG + // Use 8-bit comparison like YJIT does. + // If `val` is a constant (rare but possible), put it in a register to allow masking. + let val = asm.load_imm(val); + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::NilClass) { + asm.cmp(val, Qnil.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::TrueClass) { + asm.cmp(val, Qtrue.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::FalseClass) { + asm.cmp(val, Qfalse.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {ty}"); + } else if let Some(expected_class) = ty.runtime_exact_ruby_class() { + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + + // Create a result block that all paths converge to + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = |v| Target::Block(lir::BranchEdge { + target: result_block, + args: vec![v], + }); + + // If val isn't in a register, load it to use it as the base of Opnd::mem later. + // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) + let val = asm.load_mem(val); + + // Immediate -> definitely not the class + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, result_edge(Opnd::Imm(0))); + + // Qfalse -> definitely not the class + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge(Opnd::Imm(0))); + + // Heap object -> check klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); + asm.cmp(klass, Opnd::Value(expected_class)); + let result = asm.csel_e(Opnd::UImm(1), Opnd::Imm(0)); + asm.jmp(result_edge(result)); + + // Result block -- receives the value via block parameter (phi node) + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); + let param = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(param); + param + } else { + unimplemented!("unsupported type: {ty}"); + } +} + +/// Compile a type check with a side exit +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { + gen_incr_counter(asm, Counter::guard_type_count); + if guard_type.is_subtype(types::Fixnum) { + asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + asm.jz(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::Flonum) { + // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG + let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); + asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); + asm.jne(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::StaticSymbol) { + // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG + // Use 8-bit comparison like YJIT does. + // If `val` is a constant (rare but possible), put it in a register to allow masking. + let val = asm.load_imm(val); + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.jne(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::NilClass) { + asm.cmp(val, Qnil.into()); + asm.jne(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::TrueClass) { + asm.cmp(val, Qtrue.into()); + asm.jne(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::FalseClass) { + asm.cmp(val, Qfalse.into()); + asm.jne(jit, side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {guard_type}"); + } else if let Some(expected_class) = guard_type.runtime_exact_ruby_class() { + asm_comment!(asm, "guard exact class for non-immediate types"); + + // If val isn't in a register, load it to use it as the base of Opnd::mem later. + // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) + let val = asm.load_mem(val); + + // Check if it's a special constant + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, side_exit.clone()); + + // Check if it's false + asm.cmp(val, Qfalse.into()); + asm.je(jit, side_exit.clone()); + + // Load the class from the object's klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); + + asm.cmp(klass, Opnd::Value(expected_class)); + asm.jne(jit, side_exit); + } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() { + let side = side_exit(jit, state, GuardType(guard_type)); + + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, side.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(jit, side.clone()); + + // Mask and check the builtin type + let val = asm.load_mem(val); + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(builtin_type as u64)); + asm.jne(jit, side); + } else if guard_type.bit_equal(types::HeapBasicObject) { + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.cmp(val, Opnd::Value(Qfalse)); + asm.je(jit, side_exit.clone()); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, side_exit); + } else { + unimplemented!("unsupported type: {guard_type}"); + } + val +} + +/// Compile an identity check with a side exit +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option<Recompile>, state: &FrameState) -> lir::Opnd { + if matches!(reason, SideExitReason::GuardShape(_) ) { + gen_incr_counter(asm, Counter::guard_shape_count); + } + let expected_opnd: Opnd = match expected { + crate::hir::Const::Value(v) => { Opnd::Value(v) } + crate::hir::Const::CInt64(v) => { v.into() } + crate::hir::Const::CShape(v) => { Opnd::UImm(v.0 as u64) } + _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), + }; + asm.cmp(val, expected_opnd); + asm.jnz(jit, side_exit_with_recompile(jit, state, reason, recompile)); + val +} + +fn mask_to_opnd(mask: crate::hir::Const) -> Option<Opnd> { + match mask { + crate::hir::Const::CUInt8(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt16(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt32(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt64(v) => Some(Opnd::UImm(v)), + _ => None + } +} + +/// Compile a bitmask check with a side exit if none of the masked bits are not set +fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_any_bit_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jz(jit, side_exit(jit, state, reason)); + val +} + +/// Compile a bitmask check with a side exit if any of the masked bits are set +fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_no_bits_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jnz(jit, side_exit(jit, state, reason)); + val +} + +/// Generate code that records unoptimized C functions if --zjit-stats is enabled +fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) { + if get_option!(stats) { + asm.incr_counter(Opnd::const_ptr(counter_ptr as *const u8), Opnd::UImm(1)); + } +} + +/// Generate code that increments a counter if --zjit-stats +fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { + if get_option!(stats) { + let ptr = counter_ptr(counter); + gen_incr_counter_ptr(asm, ptr); + } +} + +/// Increment a counter for each DynamicSendReason. If the variant has +/// a counter prefix to break down the details, increment that as well. +fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReason) { + gen_incr_counter(asm, send_fallback_counter(reason)); + + use SendFallbackReason::*; + match reason { + Uncategorized(opcode) => { + gen_incr_counter_ptr(asm, send_fallback_counter_ptr_for_opcode(opcode)); + } + SendWithoutBlockNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_method_type(method_type)); + } + SendWithoutBlockNotOptimizedMethodTypeOptimized(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_optimized_method_type(method_type)); + } + SendNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_method_type(method_type)); + } + SuperNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_super_method_type(method_type)); + } + _ => {} + } +} + +/// Check if an ISEQ contains instructions that may write to block_code +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock). +/// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. +#[allow(non_upper_case_globals)] +fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { + let encoded_size = unsafe { rb_iseq_encoded_size(iseq) }; + let mut insn_idx: u32 = 0; + + while insn_idx < encoded_size { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let opcode = unsafe { rb_iseq_bare_opcode_at_pc(iseq, pc) } as u32; + + match opcode { + YARVINSN_send | YARVINSN_sendforward | + YARVINSN_invokesuper | YARVINSN_invokesuperforward | + YARVINSN_invokeblock => { + return true; + } + _ => {} + } + + insn_idx = insn_idx.saturating_add(unsafe { rb_insn_len(VALUE(opcode as usize)) }.try_into().unwrap()); + } + + false +} + +/// Save only the PC to CFP. Use this when you need to call gen_save_sp() +/// immediately after with a custom stack size (e.g., gen_ccall_with_frame +/// adjusts SP to exclude receiver and arguments). +fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { + let opcode: usize = state.get_opcode().try_into().unwrap(); + let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; + + gen_incr_counter(asm, Counter::vm_write_jit_frame_count); + asm_comment!(asm, "save JITFrame to CFP"); + let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), Opnd::const_ptr(jit_frame)); + + // CFP_PC for a live JIT frame routes through the JITFrame on the native + // stack (cfp->jit_return points to NATIVE_BASE_PTR), so we don't need to + // touch cfp->pc here. Poisoning cfp->pc with PC_POISON would actively + // break the case where rb_zjit_materialize_frames() previously copied + // jit_frame->pc into cfp->pc and cleared cfp->jit_return: the JIT keeps + // running, lands on this routine again, and the poison would replace + // the valid materialized pc behind the GC's back. +} + +/// Save the current PC on the CFP as a preparation for calling a C function +/// that may allocate objects and trigger GC. Use gen_prepare_non_leaf_call() +/// if it may raise exceptions or call arbitrary methods. +/// +/// Unlike YJIT, we don't need to save the stack slots to protect them from GC +/// because the backend spills all live registers onto the C stack on CCall. +/// However, to avoid marking uninitialized stack slots, this also updates SP, +/// which may have cfp->sp for a past frame or a past non-leaf call. +fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState, leaf: bool) { + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack_size()); + if leaf { + asm.expect_leaf_ccall(state.stack_size()); + } +} + +fn gen_prepare_leaf_call_with_gc(asm: &mut Assembler, state: &FrameState) { + // In gen_prepare_call_with_gc(), we update cfp->sp for leaf calls too. + // + // Here, cfp->sp may be pointing to either of the following: + // 1. cfp->sp for a past frame, which gen_push_frame() skips to initialize + // 2. cfp->sp set by gen_prepare_non_leaf_call() for the current frame + // + // When (1), to avoid marking dead objects, we need to set cfp->sp for the current frame. + // When (2), setting cfp->sp at gen_push_frame() and not updating cfp->sp here could lead to + // keeping objects longer than it should, so we set cfp->sp at every call of this function. + // + // We use state.without_stack() to pass stack_size=0 to gen_save_sp() because we don't write + // VM stack slots on leaf calls, which leaves those stack slots uninitialized. ZJIT keeps + // live objects on the C stack, so they are protected from GC properly. + gen_prepare_call_with_gc(asm, &state.without_stack(), true); +} + +/// Save the current SP on the CFP +fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { + // Update cfp->sp which will be read by the interpreter. We also have the SP register in JIT + // code, and ZJIT's codegen currently assumes the SP register doesn't move, e.g. gen_param(). + // So we don't update the SP register here. We could update the SP register to avoid using + // an extra register for asm.lea(), but you'll need to manage the SP offset like YJIT does. + gen_incr_counter(asm, Counter::vm_write_sp_count); + asm_comment!(asm, "save SP to CFP: {}", stack_size); + let sp_addr = asm.lea(Opnd::mem(64, SP, stack_size as i32 * SIZEOF_VALUE_I32)); + let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); + asm.mov(cfp_sp, sp_addr); +} + +/// Spill locals onto the stack. +fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Avoid spilling locals that have been spilled before and not changed. + gen_incr_counter(asm, Counter::vm_write_locals_count); + asm_comment!(asm, "spill locals"); + for (idx, &insn_id) in state.locals().enumerate() { + asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(state.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Spill the virtual stack onto the stack. +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // This function does not call gen_save_sp() at the moment because + // gen_send_without_block_direct() spills stack slots above SP for arguments. + gen_incr_counter(asm, Counter::vm_write_stack_count); + asm_comment!(asm, "spill stack"); + for (idx, &insn_id) in state.stack().enumerate() { + asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Prepare for calling a C function that may call an arbitrary method. +/// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Lazily materialize caller frames when needed + // Save PC for backtraces and allocation tracing + // and SP to avoid marking uninitialized stack slots + gen_prepare_call_with_gc(asm, state, false); + + // Spill the virtual stack in case it raises an exception + // and the interpreter uses the stack for handling the exception + gen_spill_stack(jit, asm, state); + + // Spill locals in case the method looks at caller Bindings + gen_spill_locals(jit, asm, state); +} + +/// Frame metadata written by gen_push_frame() +struct ControlFrame { + recv: Opnd, + iseq: Option<IseqPtr>, + cme: *const rb_callable_method_entry_t, + frame_type: u32, + /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. + /// For the type of frames we push, block handler or the parent EP. + specval: lir::Opnd, + /// Whether to write block_code = 0 at frame push time. + /// True when the callee ISEQ may write to block_code (has send/invokesuper/invokeblock). + write_block_code: bool, +} + +/// Compile an interpreter frame +fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: ControlFrame) { + // Locals are written by the callee frame on side-exits or non-leaf calls + + // See vm_push_frame() for details + asm_comment!(asm, "push cme, specval, frame type"); + // ep[-2]: cref of cme + let local_size = if let Some(iseq) = frame.iseq { + (unsafe { get_iseq_body_local_table_size(iseq) }) as i32 + } else { + 0 + }; + let ep_offset = state.stack().len() as i32 + local_size - argc as i32 + VM_ENV_DATA_SIZE as i32 - 1; + // ep[-2]: CME + asm.store(Opnd::mem(64, SP, (ep_offset - 2) * SIZEOF_VALUE_I32), VALUE::from(frame.cme).into()); + // ep[-1]: specval + asm.store(Opnd::mem(64, SP, (ep_offset - 1) * SIZEOF_VALUE_I32), frame.specval); + // ep[0]: ENV_FLAGS + asm.store(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32), frame.frame_type.into()); + + // Write to the callee CFP + fn cfp_opnd(offset: i32) -> Opnd { + Opnd::mem(64, CFP, offset - (RUBY_SIZEOF_CONTROL_FRAME as i32)) + } + + asm_comment!(asm, "push callee control frame"); + + if frame.iseq.is_some() { + // PC, SP, and ISEQ are written lazily by the callee on side-exits, non-leaf calls, or GC. + // cfp->jit_return will be written by gen_entry_point() on the callee after this frame push. + if frame.write_block_code { + asm_comment!(asm, "write block_code for iseq that may use it"); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + } + } else { + // C frames don't have a PC and ISEQ in normal operation. ISEQ frames set PC on gen_save_pc_for_gc(). + // When runtime checks are enabled we poison the PC for C frames so accidental reads stand out. + if let (None, Some(pc)) = (frame.iseq, PC_POISON) { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); + // block_code must be written explicitly because the interpreter reads + // captured->code.ifunc directly from cfp->block_code (not through JITFrame). + // Without this, stale data from a previous frame occupying this CFP slot + // can be used as an ifunc pointer, causing a segfault. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + // C frames share a single static JITFrame (rb_zjit_c_frame). Setting + // cfp->jit_return to the ZJIT_JIT_RETURN_C_FRAME sentinel tells + // CFP_ZJIT_FRAME() to use that shared frame, so we don't need to + // allocate a per-call JITFrame for C method pushes. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), (ZJIT_JIT_RETURN_C_FRAME as usize).into()); + } + + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); + let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); +} + +/// Stack overflow check: fails if CFP<=SP at any point in the callee. +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { + asm_comment!(asm, "stack overflow check"); + // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 + // adds to the margin another control frame with `&bounds[1]`. + const { assert!(RUBY_SIZEOF_CONTROL_FRAME % SIZEOF_VALUE == 0, "sizeof(rb_control_frame_t) is a multiple of sizeof(VALUE)"); } + let cfp_growth = 2 * (RUBY_SIZEOF_CONTROL_FRAME / SIZEOF_VALUE); + let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; + let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); + asm.cmp(CFP, stack_limit); + asm.jbe(jit, side_exit(jit, state, StackOverflow)); +} + + +/// Inverse of ep_offset_to_local_idx(). See ep_offset_to_local_idx() for details. +pub fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }; + local_size_and_idx_to_ep_offset(local_size.to_usize(), local_idx) +} + +/// Convert the number of locals and a local index to an offset from the EP +pub fn local_size_and_idx_to_ep_offset(local_size: usize, local_idx: usize) -> i32 { + local_size as i32 - local_idx as i32 - 1 + VM_ENV_DATA_SIZE as i32 +} + +/// Convert the number of locals and a local index to an offset from the BP. +/// We don't move the SP register after entry, so we often use SP as BP. +pub fn local_size_and_idx_to_bp_offset(local_size: usize, local_idx: usize) -> i32 { + local_size_and_idx_to_ep_offset(local_size, local_idx) + 1 +} + +/// Convert ISEQ into High-level IR +fn compile_iseq(iseq: IseqPtr) -> Result<Function, CompileError> { + // Convert ZJIT instructions back to bare instructions + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + + // Reject ISEQs with very large temp stacks. + // We cannot encode too large offsets to access locals in arm64. + let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; + if stack_max >= i8::MAX as u32 { + debug!("ISEQ stack too large: {stack_max}"); + return Err(CompileError::IseqStackTooLarge); + } + + let hir = trace_compile_phase("build_hir", || + crate::stats::with_time_stat(Counter::compile_hir_build_time_ns, || iseq_to_hir(iseq)) + ); + let mut function = match hir { + Ok(function) => function, + Err(err) => { + debug!("ZJIT: iseq_to_hir: {err:?}: {}", iseq_get_location(iseq, 0)); + return Err(CompileError::ParseError(err)); + } + }; + if !get_option!(disable_hir_opt) { + trace_compile_phase("optimize", || function.optimize()); + } + function.dump_hir(); + Ok(function) +} + +/// Build a Target::SideExit +fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, state); + Target::SideExit { exit, reason } +} + +/// Build a Target::SideExit that optionally triggers exit_recompile on the exit path. +fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExitReason, recompile: Option<Recompile>) -> Target { + let mut exit = build_side_exit(jit, state); + exit.recompile = recompile.map(|strategy| SideExitRecompile { + iseq: Opnd::Value(VALUE::from(state.iseq)), + insn_idx: state.insn_idx() as u32, + strategy, + }); + Target::SideExit { exit, reason } +} + +/// Build a side-exit context +fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { + let mut stack = Vec::new(); + for &insn_id in state.stack() { + stack.push(jit.get_opnd(insn_id)); + } + + let mut locals = Vec::new(); + for &insn_id in state.locals() { + locals.push(jit.get_opnd(insn_id)); + } + + SideExit{ + pc: Opnd::const_ptr(state.pc), + stack, + locals, + iseq: state.iseq, + recompile: None, + } +} + +#[cfg(target_arch = "x86_64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + $vis extern "sysv64" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(target_arch = "aarch64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + $vis extern "C" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(test)] +pub(crate) use c_callable; + +c_callable! { + /// Called from JIT side-exit code to profile operands and trigger recompilation. + /// For send instructions (argc >= 0): profiles receiver + args from the stack. + /// For shape guard exits (argc == -1): profiles self from the CFP. + /// Once enough profiles are gathered, invalidates the ISEQ for recompilation. + pub(crate) fn exit_recompile(ec: EcPtr, iseq_raw: VALUE, insn_idx: u32, argc: i32) { + // Fast check before taking the VM lock: skip if already invalidated or + // at the version limit. This avoids expensive lock acquisition on every + // shape guard exit after the recompile has already been triggered. + { + let iseq: IseqPtr = iseq_raw.as_iseq(); + let payload = get_or_create_iseq_payload(iseq); + let already_done = payload.versions.last() + .map_or(false, |v| unsafe { v.as_ref() }.is_invalidated()) + || payload.versions.len() >= max_iseq_versions(); + if already_done { + return; + } + } + + with_vm_lock(src_loc!(), || { + let iseq: IseqPtr = iseq_raw.as_iseq(); + let payload = get_or_create_iseq_payload(iseq); + + // For no-profile sends, skip if already profiled at this insn_idx. + // For shape guard exits (argc == -1), always re-profile because the + // original YARV profiles were monomorphic but runtime showed new shapes. + if argc >= 0 && payload.profile.done_profiling_at(insn_idx as usize) { + return; + } + + with_time_stat(Counter::profile_time_ns, || { + let cfp = unsafe { get_ec_cfp(ec) }; + + let should_recompile = if argc >= 0 { + let sp = unsafe { get_cfp_sp(cfp) }; + // Profile the receiver and arguments for this send instruction + payload.profile.profile_send_at(iseq, insn_idx as usize, sp, argc as usize) + } else { + // Profile self for shape guard exits (argc == -1) + let self_val = unsafe { get_cfp_self(cfp) }; + payload.profile.profile_self_at(iseq, insn_idx as usize, self_val) + }; + + // Once we have enough profiles, invalidate and recompile the ISEQ + if should_recompile { + if let Some(version) = payload.versions.last_mut() { + let cb = ZJITState::get_code_block(); + invalidate_iseq_version(cb, iseq, version); + cb.mark_all_executable(); + } + } + }); + }); + } +} + +c_callable! { + /// Generated code calls this function with the SysV calling convention. See [gen_function_stub]. + /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. + /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported + /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE, ec: EcPtr) -> *const u8 { + // Make sure cfp is ready to be scanned by other Ractors and GC before taking the barrier + { + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let params = unsafe { iseq.params() }; + let entry_idx = iseq_call.jit_entry_idx.to_usize(); + let entry_insn_idx = params.opt_table_slice().get(entry_idx) + .unwrap_or_else(|| panic!("function_stub: opt_table out of bounds. {params:#?}, entry_idx={entry_idx}")) + .as_u32(); + // gen_push_frame() doesn't set PC or ISEQ, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC and ISEQ. + // Clear jit_return so the interpreter reads cfp->pc and cfp->iseq directly. + let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idx) }; + unsafe { rb_set_cfp_pc(cfp, pc) }; + unsafe { (*cfp)._iseq = iseq }; + unsafe { (*cfp).jit_return = std::ptr::null_mut() }; + } + + with_vm_lock(src_loc!(), || { + // Re-create the Rc inside the VM lock because IseqCall's interior + // mutability (Cell<IseqPtr>) requires exclusive access. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let argc = iseq_call.argc; + let num_opts_filled = iseq_call.jit_entry_idx; + + // JIT-to-JIT calls don't eagerly fill nils to non-parameter locals. + // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. + fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, argc: u16, num_opts_filled: u16, compile_error: &CompileError) { + unsafe { + // Caller frames are materialized by the materialize_exit trampoline before unwinding native frames. + // The current frame's pc and iseq are already set by function_stub_hit before this point. + + // Set SP which gen_push_frame() doesn't set + rb_set_cfp_sp(cfp, sp); + + let local_size = get_iseq_body_local_table_size(iseq).to_usize(); + let params = iseq.params(); + let params_size = params.size.to_usize(); + let frame_base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, 0) as isize); + let locals = slice::from_raw_parts_mut(frame_base, local_size); + // Fill nils to uninitialized (non-parameter) locals + locals.get_mut(params_size..).unwrap_or_default().fill(Qnil); + + // SendDirect packs args without gaps for unfilled optionals. + // When we exit to the interpreter, we need to shift args right + // to create the gap and nil-fill the unfilled optional slots. + // + // Example: def target(req, a = a, b = b, kw:); target(1, kw: 2) + // lead_num=1, opt_num=2, opts_filled=0, argc=2 + // + // locals[] as placed by SendDirect (argc=2, no gaps): + // [req, kw_val, ?, ?, ?, ...] + // 0 1 + // ^----caller's args----^ + // + // locals[] expected by interpreter (params_size=4): + // [req, a, b, kw_val, ?, ...] + // 0 1 2 3 + // ^nil ^nil^--moved--^ + // + // gap_start = lead_num + opts_filled = 1 + // gap_end = lead_num + opt_num = 3 + // We move locals[gap_start..argc] to locals[gap_end..], then + // nil-fill locals[gap_start..gap_end]. + let opt_num: usize = params.opt_num.try_into().expect("ISEQ opt_num should be non-negative"); + let opts_filled = num_opts_filled.to_usize(); + let opts_unfilled = opt_num.saturating_sub(opts_filled); + if opts_unfilled > 0 { + let argc = argc.to_usize(); + let lead_num: usize = params.lead_num.try_into().expect("ISEQ lead_num should be non-negative"); + let param_locals = &mut locals[..params_size]; + // Gap of unspecified optional parameters + let gap_start = lead_num + opts_filled; + let gap_end = lead_num + opt_num; + // When there are arguments in the gap, shift them past the gap + let args_overlapping_gap = gap_start..argc; + if !args_overlapping_gap.is_empty() { + assert!( + gap_end.checked_add(args_overlapping_gap.len()) + .is_some_and(|new_end| new_end <= param_locals.len()) , + "shift past gap out-of-bounds. params={params:#?} args_overlapping_gap={args_overlapping_gap:?}" + ); + param_locals.copy_within(args_overlapping_gap, gap_end); + } + // Nil-fill the now-vacant optional parameter slots + param_locals[gap_start..gap_end].fill(Qnil); + } + } + + // Increment a compile error counter for --zjit-stats + if get_option!(stats) { + incr_counter_by(exit_counter_for_compile_error(compile_error), 1); + } + } + + // If we already know we can't compile the ISEQ, or there is insufficient native + // stack space, fail early without cb.mark_all_executable(). + // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole + // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. + let cb = ZJITState::get_code_block(); + let native_stack_full = unsafe { rb_ec_stack_check(ec as _) } != 0; + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + let compile_error = match last_status { + Some(IseqStatus::CantCompile(err)) => Some(err), + _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ if native_stack_full => { + incr_counter!(skipped_native_stack_full); + Some(&CompileError::NativeStackTooLarge) + }, + _ => None, + }; + if let Some(compile_error) = compile_error { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, compile_error); + return ZJITState::get_materialize_exit_trampoline_with_counter().raw_ptr(cb); + } + + // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point. + let code_ptr = with_time_stat(compile_time_ns, || function_stub_hit_body(cb, &iseq_call)); + if code_ptr.is_ok() { + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.incoming.push(iseq_call); + } + } + let code_ptr = code_ptr.unwrap_or_else(|compile_error| { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, &compile_error); + ZJITState::get_materialize_exit_trampoline_with_counter() + }); + cb.mark_all_executable(); + code_ptr.raw_ptr(cb) + }) + } +} + +/// Compile an ISEQ for a function stub +fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<CodePtr, CompileError> { + // Compile the stubbed ISEQ + let IseqCodePtrs { jit_entry_ptrs, .. } = gen_iseq(cb, iseq_call.iseq.get(), None).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the stub to call the code pointer + let jit_entry_ptr = jit_entry_ptrs[iseq_call.jit_entry_idx.to_usize()]; + let code_addr = jit_entry_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + trace_compile_phase("compile_stub", || { + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); + asm.ccall_into(C_RET_OPND, code_addr, vec![]); + }); + }); + + Ok(jit_entry_ptr) +} + +/// Compile a stub for an ISEQ called by SendDirect +fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("gen_function_stub"); + asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + + // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. + // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. + asm.load_into(scratch_reg, Opnd::const_ptr(Rc::into_raw(iseq_call))); + asm.cpush(scratch_reg); + asm.jmp(ZJITState::get_function_stub_hit_trampoline().into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function stub is called. +/// See [gen_function_stub] for how it's used. +pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("function_stub_hit_trampoline"); + asm_comment!(asm, "function_stub_hit trampoline"); + + asm.cpop_into(scratch_reg); + + // Maintain alignment for x86_64, and set up a frame for arm64 properly + asm.frame_setup(&[]); + + asm_comment!(asm, "preserve argument registers"); + + for pair in ALLOC_REGS.chunks(2) { + match *pair { + [reg0, reg1] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + } + [reg] => { + asm.cpush(Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } + } + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 + } + + // We can't directly pass the scratch register in to the ccall because + // we're going to have parallel move automatically handle coping registers + // in to the C calling convention and the parallel move algorithm needs + // a scratch register to break any cycles. If we use the scratch register + // as a C call parameter, then parallel move wouldn't be able to break + // cycles without clobbering something + asm.mov(C_ARG_OPNDS[0], scratch_reg); + // Compile the stubbed ISEQ + let jump_addr = asm_ccall!(asm, function_stub_hit, C_ARG_OPNDS[0], CFP, SP, EC); + asm.mov(scratch_reg, jump_addr); + + asm_comment!(asm, "restore argument registers"); + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpop_into(Opnd::Reg(ALLOC_REGS[0])); + } + + for pair in ALLOC_REGS.chunks(2).rev() { + match *pair { + [reg] => { + asm.cpop_into(Opnd::Reg(reg)); + } + [reg0, reg1] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + } + _ => unreachable!("chunks(2)") + } + } + + // Discard the current frame since the JIT function will set it up again + asm.frame_teardown(&[]); + + // Jump to scratch_reg so that cpop_into() doesn't clobber it + asm.jmp_opnd(scratch_reg); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function exits without restoring PC and the stack +pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id("exit_trampoline"); + + asm_comment!(asm, "side-exit trampoline"); + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that materializes ZJIT frames before unwinding native frames. +pub fn gen_materialize_exit_trampoline(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + unsafe extern "C" { + fn rb_zjit_materialize_frames(ec: EcPtr, cfp: CfpPtr); + } + + let mut asm = Assembler::new(); + asm.new_block_without_id("materialize_exit_trampoline"); + + asm_comment!(asm, "materialize ZJIT frames"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + asm_ccall!(asm, rb_zjit_materialize_frames, EC, CFP); + asm.jmp(Target::CodePtr(exit_trampoline)); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that increments exit_compilation_failure and jumps to materialize_exit_trampoline. +pub fn gen_materialize_exit_trampoline_with_counter(cb: &mut CodeBlock, materialize_exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id("materialize_exit_trampoline_with_counter"); + + asm_comment!(asm, "function stub exit trampoline"); + gen_incr_counter(&mut asm, exit_compile_error); + asm.jmp(Target::CodePtr(materialize_exit_trampoline)); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Reserve native stack space and write operands into it. +fn gen_push_opnds(jit: &JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { + let argv = if opnds.len() > 0 { + // Make sure the Assembler will reserve a sufficient stack size for given opnds + asm_comment!(asm, "allocate space on C stack for {} values", opnds.len()); + asm.alloc_stack(jit, opnds.len()) + } else { + asm_comment!(asm, "no opnds to allocate"); + Opnd::UImm(0) + }; + + // Write operands into stack slots allocated by asm.alloc_stack() + for (idx, &opnd) in opnds.iter().enumerate() { + asm.mov(Opnd::mem(VALUE_BITS, argv, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + + argv +} + +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_opnd_ptr = gen_push_opnds(jit, asm, &values); + asm_ccall!(asm, rb_reg_new_from_values, values.len().into(), first_opnd_ptr, opt.into()) +} + +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_string_ptr = gen_push_opnds(jit, asm, &strings); + asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr) +} + +// Generate RSTRING_PTR +fn get_string_ptr(asm: &mut Assembler, string: Opnd) -> Opnd { + asm_comment!(asm, "get string pointer for embedded or heap"); + let string = asm.load_mem(string); + let flags = Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RSTRING_NOEMBED as u64).into()); + let heap_ptr = asm.load(Opnd::mem( + usize::BITS as u8, + string, + RUBY_OFFSET_RSTRING_AS_HEAP_PTR, + )); + // Load the address of the embedded array + // (struct RString *)(obj)->as.ary + let ary = asm.lea(Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RSTRING_AS_ARY)); + asm.csel_nz(heap_ptr, ary) +} + +fn gen_string_getbyte(asm: &mut Assembler, string: Opnd, index: Opnd) -> Opnd { + let string_ptr = get_string_ptr(asm, string); + // TODO(max): Use SIB indexing here once the backend supports it + let string_ptr = asm.add(string_ptr, index); + let byte = asm.load(Opnd::mem(8, string_ptr, 0)); + // Zero-extend the byte to 64 bits + let byte = byte.with_num_bits(64); + let byte = asm.and(byte, 0xFF.into()); + // Tag the byte + let byte = asm.lshift(byte, Opnd::UImm(1)); + asm.or(byte, Opnd::UImm(1)) +} + +fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, value: Opnd) -> Opnd { + // rb_str_setbyte is not leaf, but we guard types and index ranges in HIR + asm_ccall!(asm, rb_str_setbyte, string, index, value) +} + +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_str_buf_append, string, val) +} + +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) +} + +/// Generate a JIT entry that just increments exit_compilation_failure and exits +fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id("compile_error_counter"); + gen_incr_counter(&mut asm, exit_compile_error); + gen_incr_counter(&mut asm, exit_counter_for_compile_error(compile_error)); + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(0, gc_offsets.len()); + code_ptr + }) +} + +/// Given the number of spill slots needed for a function, return the number of bytes +/// the function needs to allocate on the stack for the stack frame. +fn aligned_stack_bytes(num_slots: usize) -> usize { + // Both x86_64 and arm64 require the stack to be aligned to 16 bytes. + // Since SIZEOF_VALUE is 8 bytes, we need to round up the size to the nearest even number. + let num_slots = num_slots + (num_slots % 2); + num_slots * SIZEOF_VALUE +} + +impl Assembler { + /// Allocate stack space on top of the stack slots reserved for JITFrame, + /// and return a pointer to the allocated space. + fn alloc_stack(&mut self, jit: &JITState, stack_size: usize) -> Opnd { + let total_stack_size = jit.jit_frame_size + stack_size; + self.stack_base_idx = self.stack_base_idx.max(total_stack_size); + // high addr + // +------------------------+ + // | return address | + // +------------------------+ + // | previous frame pointer | <- NATIVE_BASE_PTR == cfp->jit_return + // +------------------------+ + // | JITFrame pointer | <- jit.jit_frame_size, read by CFP_ZJIT_FRAME(cfp) + // +------------------------+ + // | opnds.last() | + // +------------------------+ + // | ... | + // +------------------------+ + // | opnds.first() | <- pointer returned by alloc_stack() + // +------------------------+ + // | register spill slots | if any + // +------------------------+ + // | FrameSetup align slot | if needed + // +------------------------+ + // low addr + self.sub(NATIVE_BASE_PTR, (SIZEOF_VALUE * total_stack_size).into()) + } + + /// Emits a load for memory based operands and returns a vreg, + /// otherwise returns recv. + fn load_mem(&mut self, recv: Opnd) -> Opnd { + match recv { + Opnd::VReg { .. } | Opnd::Reg(_) => recv, + _ => self.load(recv), + } + } + + /// Emits a load for constant based operands and returns a vreg, + /// otherwise returns recv. + fn load_imm(&mut self, recv: Opnd) -> Opnd { + match recv { + Opnd::Value { .. } | Opnd::UImm(_) | Opnd::Imm(_) => self.load(recv), + _ => recv, + } + } + + /// Make a C call while marking the start and end positions for IseqCall + fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &IseqCallRef) -> Opnd { + // We need to create our own branch rc objects so that we can move the closure below + let start_iseq_call = iseq_call.clone(); + let end_iseq_call = iseq_call.clone(); + + self.ccall_with_pos_markers( + fptr, + opnds, + move |code_ptr, _| { + start_iseq_call.start_addr.set(Some(code_ptr)); + }, + move |code_ptr, _| { + end_iseq_call.end_addr.set(Some(code_ptr)); + }, + ) + } +} + +/// Store info about a JIT entry point +pub struct JITEntry { + /// Index that corresponds to an entry in [crate::cruby::IseqParameters::opt_table_slice] + jit_entry_idx: usize, + /// Position where the entry point starts + start_addr: Cell<Option<CodePtr>>, +} + +impl JITEntry { + /// Allocate a new JITEntry + fn new(jit_entry_idx: usize) -> Rc<RefCell<Self>> { + let jit_entry = JITEntry { + jit_entry_idx, + start_addr: Cell::new(None), + }; + Rc::new(RefCell::new(jit_entry)) + } +} + +/// Store info about a JIT-to-JIT call +#[derive(Debug)] +pub struct IseqCall { + /// Callee ISEQ that start_addr jumps to + pub iseq: Cell<IseqPtr>, + + /// Index that corresponds to an entry in [crate::cruby::IseqParameters::opt_table_slice] + jit_entry_idx: u16, + + /// Argument count passing to the HIR function + argc: u16, + + /// Position where the call instruction starts + start_addr: Cell<Option<CodePtr>>, + + /// Position where the call instruction ends (exclusive) + end_addr: Cell<Option<CodePtr>>, +} + +pub type IseqCallRef = Rc<IseqCall>; + +impl IseqCall { + /// Allocate a new IseqCall + fn new(iseq: IseqPtr, jit_entry_idx: u16, argc: u16) -> IseqCallRef { + let iseq_call = IseqCall { + iseq: Cell::new(iseq), + start_addr: Cell::new(None), + end_addr: Cell::new(None), + jit_entry_idx, + argc, + }; + Rc::new(iseq_call) + } + + /// Regenerate a IseqCall with a given callback + fn regenerate(&self, cb: &mut CodeBlock, callback: impl Fn(&mut Assembler)) { + cb.with_write_ptr(self.start_addr.get().expect("expected a start address"), |cb| { + let mut asm = Assembler::new(); + asm.new_block_without_id("regenerate"); + callback(&mut asm); + asm.compile(cb).unwrap(); + assert_eq!(self.end_addr.get().unwrap(), cb.get_write_ptr()); + }); + } +} + +type PerfSymbol = Rc<RefCell<Option<(CodePtr, String)>>>; + +/// Mark the start of a perf symbol range via pos_marker. +/// Returns a handle to pass to perf_symbol_range_end. +pub fn perf_symbol_range_start(asm: &mut Assembler, symbol_name: &str) -> PerfSymbol { + let symbol_name = symbol_name.to_string(); + let perf_symbol: PerfSymbol = Rc::new(RefCell::new(None)); + let current = perf_symbol.clone(); + asm.pos_marker(move |start, _| { + let mut current = current.borrow_mut(); + assert!(current.is_none(), "perf symbol range already open"); + *current = Some((start, symbol_name.clone())); + }); + perf_symbol +} + +/// Mark the end of a perf symbol range via pos_marker. +pub fn perf_symbol_range_end(asm: &mut Assembler, perf_symbol: &PerfSymbol) { + let current = perf_symbol.clone(); + asm.pos_marker(move |end, cb| { + if let Some((start, name)) = current.borrow_mut().take() { + let start_addr = start.raw_addr(cb); + let code_size = end.raw_addr(cb) - start_addr; + register_with_perf(name, start_addr, code_size); + } + }); +} + +#[cfg(test)] +#[path = "codegen_tests.rs"] +mod tests; diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs new file mode 100644 index 0000000000..b099371718 --- /dev/null +++ b/zjit/src/codegen_tests.rs @@ -0,0 +1,5714 @@ +#![cfg(test)] + +use super::{gen_insn, JITState}; +use crate::asm::CodeBlock; +use crate::backend::lir::Assembler; +use crate::codegen::max_iseq_versions; +use crate::cruby::*; +use crate::hir::{Insn, iseq_to_hir}; +use crate::options::{rb_zjit_prepare_options, set_call_threshold}; +use crate::payload::IseqVersion; +use crate::hir::tests::hir_build_tests::assert_contains_opcode; +use crate::payload::*; +use insta::assert_snapshot; + +#[test] +fn test_breakpoint_hir_codegen() { + rb_zjit_prepare_options(); + + eval("def test_breakpoint_hir_codegen = nil"); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", "test_breakpoint_hir_codegen")); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + let breakpoint = function.push_insn(function.entries_block, Insn::BreakPoint); + + let mut jit = JITState::new( + IseqVersion::new(iseq), + function.num_insns(), + function.num_blocks(), + 0, + ); + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + let mut cb = CodeBlock::new_dummy(); + + gen_insn(&mut cb, &mut jit, &mut asm, &function, breakpoint, &function.find(breakpoint)).unwrap(); + asm.compile_with_num_regs(&mut cb, 0); + + #[cfg(target_arch = "x86_64")] + assert_eq!(cb.hexdump(), "cc"); + + #[cfg(target_arch = "aarch64")] + assert_eq!(cb.hexdump(), "00003ed4"); +} + +#[test] +fn test_call_itself() { + assert_snapshot!(inspect(" + def test = 42.itself + test + test + "), @"42"); +} + +#[test] +fn test_nil() { + assert_snapshot!(inspect(" + def test = nil + test + test + "), @"nil"); +} + +#[test] +fn test_putobject() { + assert_snapshot!(inspect(" + def test = 1 + test + test + "), @"1"); +} + +#[test] +fn test_dupstring() { + eval(r##" + def test = "#{""}" + test + "##); + assert_contains_opcode("test", YARVINSN_dupstring); + assert_snapshot!(assert_compiles(r##"test"##), @r#""""#); +} + +#[test] +fn test_dupchilledstring() { + eval(r#" + def test = "" + test + "#); + assert_contains_opcode("test", YARVINSN_dupchilledstring); + assert_snapshot!(assert_compiles(r#"test"#), @r#""""#); +} + +#[test] +fn test_leave_param() { + assert_snapshot!(inspect(" + def test(n) = n + test(5) + test(5) + "), @"5"); +} + +#[test] +fn test_getglobal_with_warning() { + eval(r#" + Warning[:deprecated] = true + + module Warning + def warn(message) + raise + end + end + + def test + $= + rescue + "rescued" + end + $VERBOSE = true + test + "#); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(assert_compiles(r#"test"#), @r#""rescued""#); +} + +#[test] +fn test_setglobal() { + eval(" + def test + $a = 1 + $a + end + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(assert_compiles("test"), @"1"); +} + +#[test] +fn test_string_intern() { + eval(r#" + def test + :"foo#{123}" + end + test + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(assert_compiles(r#"test"#), @":foo123"); +} + +#[test] +fn test_duphash() { + eval(" + def test + {a: 1} + end + test + "); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(assert_compiles("test"), @"{a: 1}"); +} + +#[test] +fn test_pushtoarray() { + eval(" + def test + [*[], 1, 2, 3] + end + test + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(assert_compiles("test"), @"[1, 2, 3]"); +} + +#[test] +fn test_splatarray_new_array() { + eval(" + def test a + [*a, 3] + end + test [1, 2] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(assert_compiles("test [1, 2]"), @"[1, 2, 3]"); +} + +#[test] +fn test_splatarray_existing_array() { + eval(" + def foo v + [1, 2, v] + end + def test a + foo(*a) + end + test [3] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(assert_compiles("test [3]"), @"[1, 2, 3]"); +} + +#[test] +fn test_concattoarray() { + eval(" + def test(*a) + [1, 2, *a] + end + test 3 + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(assert_compiles("test 3"), @"[1, 2, 3]"); +} + +#[test] +fn test_definedivar() { + eval(" + def test + v0 = defined?(@a) + @a = nil + v1 = defined?(@a) + remove_instance_variable :@a + v2 = defined?(@a) + [v0, v1, v2] + end + test + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(assert_compiles("test"), @r#"[nil, "instance-variable", nil]"#); +} + +#[test] +fn test_setglobal_with_trace_var_exception() { + eval(r#" + def test + $a = 1 + rescue + "rescued" + end + trace_var(:$a) { raise } + test + "#); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(assert_compiles(r#"test"#), @r#""rescued""#); +} + +#[test] +fn test_getlocal_after_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_instance_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + instance_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_module_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + Kernel.module_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_class_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + Kernel.class_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_setlocal() { + assert_snapshot!(inspect(" + def test(n) + m = n + m + end + test(3) + test(3) + "), @"3"); +} + +#[test] +fn test_return_nonparam_local() { + assert_snapshot!(inspect(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test + test + "), @"nil"); +} + +#[test] +fn test_nonparam_local_nil_in_jit_call() { + assert_snapshot!(inspect(r#" + def f(a) + a ||= 1 + if false; b = 1; end + eval("-> { p 'x#{b}' }") + end + + 4.times.map { f(1).call } + "#), @r#"["x", "x", "x", "x"]"#); +} + +#[test] +fn test_kwargs_with_exit_and_local_invalidation() { + assert_snapshot!(inspect(r#" + def a(b:, c:) + if c == :b + return -> {} + end + Class # invalidate locals + + raise "c is :b!" if c == :b + end + + def test + # note opposite order of kwargs + a(c: :c, b: :b) + end + + 4.times { test } + :ok + "#), @":ok"); +} + +#[test] +fn test_kwargs_with_max_direct_send_arg_count() { + assert_snapshot!(inspect(" + def kwargs(five, six, a:, b:, c:, d:, e:, f:) + [a, b, c, d, five, six, e, f] + end + + 5.times.flat_map do + [ + kwargs(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8), + kwargs(5, 6, d: 4, c: 3, b: 2, a: 1, e: 7, f: 8) + ] + end.uniq + "), @"[[1, 2, 3, 4, 5, 6, 7, 8]]"); +} + +#[test] +fn test_setlocal_on_eval() { + assert_snapshot!(inspect(" + @b = binding + eval('a = 1', @b) + eval('a', @b) + "), @"1"); +} + +#[test] +fn test_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2, c = 3) + [a, b, c] + end + [test(1), test(10, 20), test(100, 200, 300)] + "), @"[[1, 2, 3], [10, 20, 3], [100, 200, 300]]"); +} + +#[test] +fn test_optional_arguments_setlocal() { + assert_snapshot!(inspect(" + def test(a = (b = 2)) + [a, b] + end + [test, test(1)] + "), @"[[2, 2], [1, nil]]"); +} + +#[test] +fn test_optional_arguments_cyclic() { + assert_snapshot!(inspect(" + test = proc { |a=a| a } + [test.call, test.call(1)] + "), @"[nil, 1]"); +} + +#[test] +fn test_getblockparamproxy() { + eval(" + def test(&block) + 0.then(&block) + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(assert_compiles("test { 1 }"), @"1"); +} + +#[test] +fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + 0.then(&block) + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(inspect("test { 1 }"), @"1"); +} + +#[test] +fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + 0.then(&block) + end + end + test { 1 }.call + "); + assert_snapshot!(inspect("test { 1 }.call"), @"1"); +} + +#[test] +fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(assert_compiles("test { 2 }"), @"2"); +} + +#[test] +fn test_getblockparam() { + eval(" + def test(&blk) + blk + end + test { 2 }.call + "); + assert_contains_opcode("test", YARVINSN_getblockparam); + assert_snapshot!(assert_compiles("test { 2 }.call"), @"2"); +} + +#[test] +fn test_setblockparam() { + eval(" + def test(&block) + block = proc { 3 } + blk = block + blk.call + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = proc { 3 } + blk = block + blk.call + end.call + end + test { 1 } + "); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_getblockparamproxy_after_setblockparam() { + eval(" + def test(&block) + block = proc { 3 } + block.call + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_getblockparam_used_twice_in_args() { + eval(" + def f(*args) = args + def test(&blk) + b = blk + f(*[1], blk) + blk + end + test {1}.call + "); + assert_contains_opcode("test", YARVINSN_getblockparam); + assert_snapshot!(assert_compiles("test {1}.call"), @"1"); +} + +#[test] +fn test_optimized_method_call_proc_call() { + eval(" + def test(p) + p.call(1) + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"2"); +} + +#[test] +fn test_optimized_method_call_proc_aref() { + eval(" + def test(p) + p[2] + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"4"); +} + +#[test] +fn test_optimized_method_call_proc_yield() { + eval(" + def test(p) + p.yield(3) + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"6"); +} + +#[test] +fn test_optimized_method_call_proc_kw_splat() { + eval(" + def test(p, h) + p.call(**h) + end + test(proc { |**kw| kw[:a] + kw[:b] }, { a: 1, b: 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |**kw| kw[:a] + kw[:b] }, { a: 1, b: 2 })"), @"3"); +} + +#[test] +fn test_optimized_method_call_proc_call_splat() { + assert_snapshot!(inspect(" + p = proc { |x| x + 1 } + def test(p) + ary = [42] + p.call(*ary) + end + test(p) + test(p) + "), @"43"); +} + +#[test] +fn test_optimized_method_call_proc_call_kwarg() { + assert_snapshot!(inspect(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test(p) + test(p) + "), @"1"); +} + +#[test] +fn test_setlocal_on_eval_with_spill() { + assert_snapshot!(inspect(" + @b = binding + eval('a = 1; itself', @b) + eval('a', @b) + "), @"1"); +} + +#[test] +fn test_nested_local_access() { + assert_snapshot!(inspect(" + 1.times do |l2| + 1.times do |l1| + define_method(:test) do + l1 = 1 + l2 = 2 + l3 = 3 + [l1, l2, l3] + end + end + end + + test + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_local_written_by_blockiseq() { + assert_snapshot!(inspect(" + def test + l1 = nil + l2 = nil + tap do |_| + l1 = 1 + tap do |_| + l2 = 2 + end + end + + [l1, l2] + end + + test + test + "), @"[1, 2]"); +} + +#[test] +fn test_no_ep_escape_patch_point_after_send_does_not_repeat_send() { + eval(r#" + $send_count = 0 + + def test + captured = nil + tap do |_| + $send_count += 1 + -> { captured } if $send_count == 2 + end + $send_count + end + "#); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(assert_compiles_allowing_exits("[test, test, test]"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_without_block() { + assert_snapshot!(inspect(" + def foo = 1 + def bar(a) = a - 1 + def baz(a, b) = a - b + + def test1 = foo + def test2 = bar(3) + def test3 = baz(4, 1) + + [test1, test2, test3] + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_six_args() { + assert_snapshot!(inspect(" + def foo(a1, a2, a3, a4, a5, a6) + [a1, a2, a3, a4, a5, a6] + end + + def test + foo(1, 2, 3, 4, 5, 6) + end + + test # profile send + test + "), @"[1, 2, 3, 4, 5, 6]"); +} + +#[test] +fn test_send_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2) = [a, b] + def entry = [test(1), test(3, 4)] + entry + entry + "), @"[[1, 2], [3, 4]]"); +} + +#[test] +fn test_send_nil_block_arg() { + assert_snapshot!(inspect(" + def test = block_given? + def entry = test(&nil) + test + test + "), @"false"); +} + +#[test] +fn test_send_symbol_block_arg() { + assert_snapshot!(inspect(" + def test = [1, 2].map(&:to_s) + test + test + "), @r#"["1", "2"]"#); +} + +#[test] +fn test_send_variadic_with_block() { + assert_snapshot!(inspect(" + A = [1, 2, 3] + B = [\"a\", \"b\", \"c\"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "), @r#"[[1, "a"], [2, "b"], [3, "c"]]"#); +} + +#[test] +fn test_send_kwarg_optional() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2) = [a, b] + def entry = test + entry + entry + "), @"[1, 2]"); +} + +#[test] +fn test_send_kwarg_optional_too_many() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10) = [a, b, c, d, e, f, g, h, i, j] + def entry = test + entry + entry + "), @"[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"); +} + +#[test] +fn test_send_kwarg_required_and_optional() { + assert_snapshot!(inspect(" + def test(a:, b: 2) = [a, b] + def entry = test(a: 3) + entry + entry + "), @"[3, 2]"); +} + +#[test] +fn test_send_kwarg_to_hash() { + assert_snapshot!(inspect(" + def test(hash) = hash + def entry = test(a: 3) + entry + entry + "), @"{a: 3}"); +} + +#[test] +fn test_send_kwarg_to_ccall() { + assert_snapshot!(inspect(r#" + def test(s) = s.each_line(chomp: true).to_a + def entry = test(%(a\nb\nc)) + entry + entry + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_kwarg_and_block_to_ccall() { + assert_snapshot!(inspect(r#" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + def entry = test(%(a\nb\nc)) + entry + entry + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_kwarg_with_too_many_args_to_c_call() { + assert_snapshot!(inspect(r#" + def test(a:, b:, c:, d:, e:) = sprintf("%s %s %s %s %s", a, b, c, d, kwargs: e) + def entry = test(e: :e, d: :d, c: :c, a: :a, b: :b) + entry + entry + "#), @r#""a b c d {kwargs: :e}""#); +} + +#[test] +fn test_send_kwsplat() { + assert_snapshot!(inspect(" + def test(a:) = a + def entry = test(**{a: 3}) + entry + entry + "), @"3"); +} + +#[test] +fn test_send_kwrest() { + assert_snapshot!(inspect(" + def test(**kwargs) = kwargs + def entry = test(a: 3) + entry + entry + "), @"{a: 3}"); +} + +#[test] +fn test_send_req_kwreq() { + assert_snapshot!(inspect(" + def test(a, c:) = [a, c] + def entry = test(1, c: 3) + entry + entry + "), @"[1, 3]"); +} + +#[test] +fn test_send_req_opt_kwreq() { + assert_snapshot!(inspect(" + def test(a, b = 2, c:) = [a, b, c] + def entry = [test(1, c: 3), test(-1, -2, c: -3)] + entry + entry + "), @"[[1, 2, 3], [-1, -2, -3]]"); +} + +#[test] +fn test_send_req_opt_kwreq_kwopt() { + assert_snapshot!(inspect(" + def test(a, b = 2, c:, d: 4) = [a, b, c, d] + def entry = [test(1, c: 3), test(-1, -2, d: -4, c: -3)] + entry + entry + "), @"[[1, 2, 3, 4], [-1, -2, -3, -4]]"); +} + +#[test] +fn test_send_unexpected_keyword() { + assert_snapshot!(inspect(" + def test(a: 1) = a*2 + def entry + test(z: 2) + rescue ArgumentError + :error + end + + entry + entry + "), @":error"); +} + +#[test] +fn test_pos_optional_with_maybe_too_many_args() { + assert_snapshot!(inspect(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "), @"[[1, 2, 3, 4, 5, 6], [10, 20, 30, 4, 5, 6], [10, 20, 30, 40, 50, 60]]"); +} + +#[test] +fn test_send_kwarg_partial_optional() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3) = [a, b, c] + def entry = [test, test(b: 20), test(c: 30, a: 10)] + entry + entry + "), @"[[1, 2, 3], [1, 20, 3], [10, 2, 30]]"); +} + +#[test] +fn test_send_kwarg_optional_a_lot() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3, d: 4, e: 5, f: 6) = [a, b, c, d, e, f] + def entry = [test, test(d: 7, f: 9, e: 8), test(f: 12, e: 10, d: 8, c: 6, b: 4, a: 2)] + entry + entry + "), @"[[1, 2, 3, 4, 5, 6], [1, 2, 3, 7, 8, 9], [2, 4, 6, 8, 10, 12]]"); +} + +#[test] +fn test_send_kwarg_non_constant_default() { + assert_snapshot!(inspect(" + def make_default = 2 + def test(a: 1, b: make_default) = [a, b] + def entry = [test, test(a: 10)] + entry + entry + "), @"[[1, 2], [10, 2]]"); +} + +#[test] +fn test_send_kwarg_optional_static_with_side_exit() { + assert_snapshot!(inspect(" + def callee(a: 1, b: 2) + x = binding.local_variable_get(:a) + [a, b, x] + end + + def entry + callee(a: 10) + end + + entry + entry + "), @"[10, 2, 10]"); +} + +#[test] +fn test_send_hash_to_kwarg_only_method() { + assert_snapshot!(inspect(r#" + def callee(a:) = a + + def entry + callee({a: 1}) + rescue ArgumentError + "ArgumentError" + end + + entry + entry + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_send_hash_to_optional_kwarg_only_method() { + assert_snapshot!(inspect(r#" + def callee(a: nil) = a + + def entry + callee({a: 1}) + rescue ArgumentError + "ArgumentError" + end + + entry + entry + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_send_all_arg_types() { + assert_snapshot!(inspect(" + def test(a, b = :opt, c, d:, e: :kwo) = [a, b, c, d, e, block_given?] + def entry = test(:req, :post, d: :kwr) {} + entry + entry + "), @"[:req, :opt, :post, :kwr, :kwo, true]"); +} + +#[test] +fn test_send_ccall_variadic_with_different_receiver_classes() { + assert_snapshot!(inspect(r#" + def test(obj) = obj.start_with?("a") + [test("abc"), test(:abc)] + "#), @"[true, true]"); +} + +#[test] +fn test_forwardable_iseq() { + assert_snapshot!(inspect(" + def test(...) = 1 + test + test + "), @"1"); +} + +#[test] +fn test_sendforward() { + eval(" + def callee(a, b) = [a, b] + def test(...) = callee(...) + test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_sendforward); + assert_snapshot!(assert_compiles("test(1, 2)"), @"[1, 2]"); +} + +#[test] +fn test_iseq_with_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2) = [a, b] + [test(1), test(3, 4)] + "), @"[[1, 2], [3, 4]]"); +} + +#[test] +fn test_invokesuper() { + assert_snapshot!(inspect(" + class Foo + def foo(a) = a + 1 + def bar(a) = a + 10 + end + + class Bar < Foo + def foo(a) = super(a) + 2 + def bar(a) = super + 20 + end + + bar = Bar.new + [bar.foo(3), bar.bar(30)] + "), @"[6, 60]"); +} + +#[test] +fn test_invokesuper_to_iseq() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "A"]"#); +} + +#[test] +fn test_invokesuper_with_args() { + assert_snapshot!(inspect(r#" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + ["B", super(x) + 1] + end + end + + def test + B.new.foo(5) + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", 11]"#); +} + +#[test] +fn test_invokesuper_with_args_to_rest_param() { + assert_snapshot!(inspect(r#" + class A + def foo(x, *rest) + [x, rest] + end + end + + class B < A + def foo(x, y, z) + ["B", *super(x, y, z)] + end + end + + def test + B.new.foo("a", "b", "c") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "a", ["b", "c"]]"#); +} + +#[test] +fn test_invokesuper_with_block() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super { "from_block" }] + end + end + + def test + B.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "from_block"]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_no_args() { + assert_snapshot!(inspect(r#" + class MyString < String + def length + ["MyString", super] + end + end + + def test + MyString.new("abc").length + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", 3]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_simple_args() { + assert_snapshot!(inspect(r#" + class MyString < String + def include?(other) + ["MyString", super(other)] + end + end + + def test + MyString.new("abc").include?("bc") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", true]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_with_optional_arg() { + assert_snapshot!(inspect(r#" + class MyString < String + def byteindex(needle, offset = 0) + ["MyString", super(needle, offset)] + end + end + + def test + MyString.new("hello world").byteindex("world") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", 6]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_varargs() { + assert_snapshot!(inspect(r#" + class MyString < String + def end_with?(str) + ["MyString", super(str)] + end + end + + def test + MyString.new("abc").end_with?("bc") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", true]"#); +} + +#[test] +fn test_string_new_preserves_string_arg() { + assert_snapshot!(inspect(r#" + def test + str = "hello" + String.new(str) + :ok + end + + test + test + "#), @":ok"); +} + +#[test] +fn test_invokesuper_multilevel() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + class C < B + def foo + ["C", super] + end + end + + def test + C.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["C", ["B", "A"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # should forward the block from caller + end + end + + def test + B.new.foo { "forwarded_block" } + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "forwarded_block"]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_with_args() { + assert_snapshot!(inspect(r#" + class A + def foo(x) + [x, (block_given? ? yield : "no_block")] + end + end + + class B < A + def foo(x) + ["B", super(x)] # explicit args, but block should still be forwarded + end + end + + def test + B.new.foo("arg_value") { "forwarded" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", ["arg_value", "forwarded"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_no_block_given() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # no block given by caller + end + end + + def test + B.new.foo # called without a block + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "no_block"]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_multilevel() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # forwards block to A + end + end + + class C < B + def foo + ["C", super] # forwards block to B, which forwards to A + end + end + + def test + C.new.foo { "deep_block" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["C", ["B", "deep_block"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_param() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo(&block) + ["B", super] # should forward &block implicitly + end + end + + def test + B.new.foo { "block_param_forwarded" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "block_param_forwarded"]"#); +} + +#[test] +fn test_invokesuper_with_blockarg() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no block" + end + end + + class B < A + def foo(&blk) + other_block = proc { "different block" } + ["B", super(&other_block)] + end + end + + def test + B.new.foo { "passed block" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "different block"]"#); +} + +#[test] +fn test_invokesuper_with_symbol_to_proc() { + assert_snapshot!(inspect(r#" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + ["B", super(items, &:succ)] + end + end + + def test + B.new.foo([2, 4, 6]) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", [3, 5, 7]]"#); +} + +#[test] +fn test_invokesuper_with_splat() { + assert_snapshot!(inspect(r#" + class A + def foo(a, b, c) + a + b + c + end + end + + class B < A + def foo(*args) + ["B", super(*args)] + end + end + + def test + B.new.foo(1, 2, 3) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", 6]"#); +} + +#[test] +fn test_invokesuper_with_kwargs() { + assert_snapshot!(inspect(r#" + class A + def foo(x:, y:) + "x=#{x}, y=#{y}" + end + end + + class B < A + def foo(x:, y:) + ["B", super(x: x, y: y)] + end + end + + def test + B.new.foo(x: 1, y: 2) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "x=1, y=2"]"#); +} + +#[test] +fn test_invokesuper_with_kw_splat() { + assert_snapshot!(inspect(r#" + class A + def foo(x:, y:) + "x=#{x}, y=#{y}" + end + end + + class B < A + def foo(**kwargs) + ["B", super(**kwargs)] + end + end + + def test + B.new.foo(x: 1, y: 2) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "x=1, y=2"]"#); +} + +#[test] +fn test_invokesuper_with_include() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper (super -> A#foo) + test # compile with super -> A#foo + + # Now include a module in B that defines foo - super should go to M#foo instead + module M + def foo + "M" + end + end + B.include(M) + + test # should call M#foo, not A#foo + "#), @r#"["B", "M"]"#); +} + +#[test] +fn test_invokesuper_with_prepend() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper (super -> A#foo) + test # compile with super -> A#foo + + # Now prepend a module that defines foo - super should go to M#foo instead + module M + def foo + "M" + end + end + A.prepend(M) + + test # should call M#foo, not A#foo + "#), @r#"["B", "M"]"#); +} + +#[test] +fn test_invokesuper_with_keyword_args() { + assert_snapshot!(inspect(r#" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + def test + B.new.foo("image data") + end + + test + test + "#), @r#"{content: "image data"}"#); +} + +#[test] +fn test_invokesuper_with_optional_keyword_args() { + assert_snapshot!(inspect(" + class Parent + def foo(a, b: 2, c: 3) = [a, b, c] + end + + class Child < Parent + def foo(a) = super(a) + end + + def test = Child.new.foo(1) + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokesuperforward() { + assert_snapshot!(inspect(" + class A + def foo(a,b,c) = [a,b,c] + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, 3) + end + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokesuperforward_with_args_kwargs_and_block() { + assert_snapshot!(inspect(" + class A + def foo(*args, **kwargs, &block) + [args, kwargs, block&.call] + end + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, x: 3) { 4 } + end + + test + test + "), @"[[1, 2], {x: 3}, 4]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default() { + assert_snapshot!(inspect(" + def dbl(x = 1) = x * 2 + + def foo(a: dbl, b: dbl(2), c: dbl(2 ** 3)) + [a, b, c] + end + + def test + [ + foo, + foo(a: 10), + foo(b: 20), + foo(c: 30), + foo(a: 10, b: 20, c: 30) + ] + end + + test + test + "), @"[[2, 4, 16], [10, 4, 16], [2, 20, 16], [2, 4, 30], [10, 20, 30]]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_not_evaluated_when_provided() { + assert_snapshot!(inspect(" + def foo(a: raise, b: raise, c: raise) + [a, b, c] + end + + def test + foo(a: 1, b: 2, c: 3) + end + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_evaluated_when_not_provided() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_with_non_constant_keyword_default_jit_to_jit() { + assert_snapshot!(inspect(" + def make_default(x) = x * 2 + + def callee(a: make_default(1), b: make_default(2), c: make_default(3)) + [a, b, c] + end + + def caller_method + callee + end + + # Warm up callee first so it gets JITted + callee + callee + + # Now warm up caller - this creates JIT-to-JIT call + caller_method + caller_method + "), @"[2, 4, 6]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_side_exit() { + assert_snapshot!(inspect(" + def make_b = 2 + + def callee(a: 1, b: make_b, c: 3) + x = binding.local_variable_get(:a) + y = binding.local_variable_get(:b) + z = binding.local_variable_get(:c) + [x, y, z] + end + + def test + callee(a: 10, c: 30) + end + + test + test + "), @"[10, 2, 30]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_evaluation_order() { + assert_snapshot!(inspect(r#" + def log(x) + $order << x + x + end + + def foo(a: log("a"), b: log("b"), c: log("c")) + [a, b, c] + end + + def test + results = [] + + $order = [] + foo + results << $order.dup + + $order = [] + foo(a: "A") + results << $order.dup + + $order = [] + foo(b: "B") + results << $order.dup + + $order = [] + foo(c: "C") + results << $order.dup + + results + end + + test + test + "#), @r#"[["a", "b", "c"], ["b", "c"], ["a", "c"], ["a", "b"]]"#); +} + +#[test] +fn test_send_with_too_many_non_constant_keyword_defaults() { + assert_snapshot!(inspect(" + def many_kwargs( k1: 1, k2: 2, k3: 3, k4: 4, k5: 5, k6: 6, k7: 7, k8: 8, k9: 9, k10: 10, k11: 11, k12: 12, k13: 13, k14: 14, k15: 15, k16: 16, k17: 17, k18: 18, k19: 19, k20: 20, k21: 21, k22: 22, k23: 23, k24: 24, k25: 25, k26: 26, k27: 27, k28: 28, k29: 29, k30: 30, k31: 31, k32: 32, k33: 33, k34: k33 + 1) = k1 + k34 + def t = many_kwargs + t + t + "), @"35"); +} + +#[test] +fn test_invokebuiltin_delegate() { + assert_snapshot!(inspect(" + def test = [].clone(freeze: true) + r = test + r2 = test + [r2, r2.frozen?] + "), @"[[], true]"); +} + +#[test] +fn test_opt_plus_const() { + assert_snapshot!(inspect(" + def test = 1 + 2 + test # profile opt_plus + test + "), @"3"); +} + +#[test] +fn test_opt_plus_fixnum() { + assert_snapshot!(inspect(" + def test(a, b) = a + b + test(0, 1) # profile opt_plus + test(1, 2) + "), @"3"); +} + +#[test] +fn test_opt_plus_chain() { + assert_snapshot!(inspect(" + def test(a, b, c) = a + b + c + test(0, 1, 2) # profile opt_plus + test(1, 2, 3) + "), @"6"); +} + +#[test] +fn test_opt_plus_left_imm() { + assert_snapshot!(inspect(" + def test(a) = 1 + a + test(1) # profile opt_plus + test(2) + "), @"3"); +} + +#[test] +fn test_opt_plus_type_guard_exit() { + assert_snapshot!(inspect(" + def test(a) = 1 + a + test(1) # profile opt_plus + [test(2), test(2.0)] + "), @"[3, 3.0]"); +} + +#[test] +fn test_opt_plus_type_guard_exit_with_locals() { + assert_snapshot!(inspect(" + def test(a) + local = 3 + 1 + a + local + end + test(1) # profile opt_plus + [test(2), test(2.0)] + "), @"[6, 6.0]"); +} + +#[test] +fn test_opt_plus_type_guard_nested_exit() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) # profile send + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); +} + +#[test] +fn test_opt_plus_type_guard_nested_exit_with_locals() { + assert_snapshot!(inspect(" + def side_exit(n) + local = 2 + 1 + n + local + end + def jit_frame(n) + local = 3 + 1 + side_exit(n) + local + end + def entry(n) = jit_frame(n) + entry(2) # profile send + [entry(2), entry(2.0)] + "), @"[9, 9.0]"); +} + +#[test] +fn test_opt_minus() { + assert_snapshot!(inspect(" + def test(a, b) = a - b + test(2, 1) # profile opt_minus + test(6, 4) + "), @"2"); +} + +#[test] +fn test_opt_mult() { + assert_snapshot!(inspect(" + def test(a, b) = a * b + test(1, 2) # profile opt_mult + test(2, 3) + "), @"6"); +} + +#[test] +fn test_opt_mult_overflow() { + assert_snapshot!(inspect(" + def test(a, b) + a * b + end + test(1, 1) # profile opt_mult + + r1 = test(2, 3) + r2 = test(2, -3) + r3 = test(2 << 40, 2 << 41) + r4 = test(2 << 40, -2 << 41) + r5 = test(1 << 62, 1 << 62) + + [r1, r2, r3, r4, r5] + "), @"[6, -6, 9671406556917033397649408, -9671406556917033397649408, 21267647932558653966460912964485513216]"); +} + +#[test] +fn test_opt_eq() { + eval(" + def test(a, b) = a == b + test(0, 2) # profile opt_eq + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(assert_compiles("[test(1, 1), test(0, 1)]"), @"[true, false]"); +} + +#[test] +fn test_opt_eq_with_minus_one() { + eval(" + def test(a) = a == -1 + test(1) # profile opt_eq + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(assert_compiles("[test(0), test(-1)]"), @"[false, true]"); +} + +#[test] +fn test_opt_neq_dynamic() { + eval(" + def test(a, b) = a != b + test(0, 2) # profile opt_neq + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(assert_compiles("[test(1, 1), test(0, 1)]"), @"[false, true]"); +} + +#[test] +fn test_opt_neq_fixnum() { + assert_snapshot!(inspect(" + def test(a, b) = a != b + test(0, 2) # profile opt_neq + [test(1, 1), test(0, 1)] + "), @"[false, true]"); +} + +#[test] +fn test_opt_neq_string_nil() { + assert_snapshot!(inspect(r#" + def test(str) = str != nil + test("x") # profile opt_neq + [test("x"), test(nil)] + "#), @"[true, false]"); +} + +#[test] +fn test_opt_neq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s != s + test("x") # profile opt_neq + [test("x"), test("y")] + "#), @"[false, false]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_distinct_literals() { + assert_snapshot!(inspect(r#" + def test = "a" != "b" + test # profile opt_neq + [test, test] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_one_side_known_literal() { + assert_snapshot!(inspect(r#" + def test(s) = "a" != s + test("a") # profile opt_neq + [test("a"), test("b")] + "#), @"[false, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_distinct_objects() { + assert_snapshot!(inspect(r#" + def test(s, t) = s != t + test("x", "x") # profile opt_neq + [test("x", "x"), test("x", "y")] + "#), @"[false, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_eq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s == s + test("x") # profile opt_eq + [test("x"), test("y")] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_distinct_literals() { + assert_snapshot!(inspect(r#" + def test = "a" == "b" + test # profile opt_eq + [test, test] + "#), @"[false, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_one_side_known_literal() { + assert_snapshot!(inspect(r#" + def test(s) = "a" == s + test("a") # profile opt_eq + [test("a"), test("b")] + "#), @"[true, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_distinct_objects() { + assert_snapshot!(inspect(r#" + def test(s, t) = s == t + test("x", "x") # profile opt_eq + [test("x", "x"), test("x", "y")] + "#), @"[true, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eqq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s === s + test("x") # profile opt_send_without_block + [test("x"), test("y")] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); +} + +#[test] +fn test_opt_lt() { + eval(" + def test(a, b) = a < b + test(2, 3) # profile opt_lt + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[true, false, false]"); +} + +#[test] +fn test_opt_lt_with_literal_lhs() { + eval(" + def test(n) = 2 < n + test(2) # profile opt_lt + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(assert_compiles("[test(1), test(2), test(3)]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_le() { + eval(" + def test(a, b) = a <= b + test(2, 3) # profile opt_le + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[true, true, false]"); +} + +#[test] +fn test_opt_gt() { + eval(" + def test(a, b) = a > b + test(2, 3) # profile opt_gt + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(assert_compiles_allowing_exits("[test([1]), test(\"1\"), test({})]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_succ() { + eval(" + def test(obj) = obj.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(assert_compiles_allowing_exits(r#"[test(-1), test("A")]"#), @r#"[0, "B"]"#); +} + +#[test] +fn test_opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles_allowing_exits("[test(0b1101, 3), test([3, 2, 1, 4], [8, 1, 2, 3])]"), @"[1, [3, 2, 1]]"); +} + +#[test] +fn test_opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles_allowing_exits("[test(0b1000, 3), test([3, 2, 1], [1, 2, 3])]"), @"[11, [3, 2, 1]]"); +} + +#[test] +fn test_fixnum_and() { + eval(" + def test(a, b) = a & b + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles(" + [ + test(5, 3), + test(0b011, 0b110), + test(-0b011, 0b110) + ] + "), @"[1, 2, 4]"); +} + +#[test] +fn test_fixnum_and_side_exit() { + eval(" + def test(a, b) = a & b + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles_allowing_exits(" + [ + test(2, 2), + test(0b011, 0b110), + test(true, false) + ] + "), @"[2, 2, false]"); +} + +#[test] +fn test_fixnum_or() { + eval(" + def test(a, b) = a | b + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles(" + [ + test(5, 3), + test(1, 2), + test(1, -4) + ] + "), @"[7, 3, -3]"); +} + +#[test] +fn test_fixnum_or_side_exit() { + eval(" + def test(a, b) = a | b + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles_allowing_exits(" + [ + test(1, 2), + test(2, 2), + test(true, false) + ] + "), @"[3, 2, true]"); +} + +#[test] +fn test_fixnum_xor() { + assert_snapshot!(inspect(" + def test(a, b) = a ^ b + [ + test(5, 3), + test(-5, 3), + test(1, 2) + ] + "), @"[6, -8, 3]"); +} + +#[test] +fn test_fixnum_xor_side_exit() { + assert_snapshot!(inspect(" + def test(a, b) = a ^ b + [ + test(5, 3), + test(5, 3), + test(true, false) + ] + "), @"[6, 6, true]"); +} + +#[test] +fn test_fixnum_mul() { + eval(" + C = 3 + def test(n) = C * n + test(4) + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(assert_compiles("test(4)"), @"12"); +} + +#[test] +fn test_fixnum_div() { + eval(" + C = 48 + def test(n) = C / n + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("test(4)"), @"12"); +} + +#[test] +fn test_fixnum_floor() { + eval(" + C = 3 + def test(n) = C / n + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("test(4)"), @"0"); +} + +#[test] +fn test_opt_not() { + eval(" + def test(obj) = !obj + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(assert_compiles_allowing_exits("[test(nil), test(false), test(0)]"), @"[true, true, false]"); +} + +#[test] +fn test_opt_regexpmatch2() { + eval(" + def test(haystack) = /needle/ =~ haystack + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(assert_compiles(r#"[test("kneedle"), test("")]"#), @"[1, nil]"); +} + +#[test] +fn test_opt_ge() { + eval(" + def test(a, b) = a >= b + test(2, 3) # profile opt_ge + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[false, true, true]"); +} + +#[test] +fn test_opt_new_does_not_push_frame() { + eval(" + class Foo + attr_reader :backtrace + def initialize + @backtrace = caller + end + end + def test = Foo.new + test + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(" + foo = test + foo.backtrace.find { |frame| frame.include?('Class#new') } + "), @"nil"); +} + +#[test] +fn test_opt_new_with_redefined() { + eval(r#" + class Foo + def self.new = "foo" + def initialize = raise("unreachable") + end + def test = Foo.new + test + "#); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(r#"test"#), @r#""foo""#); +} + +#[test] +fn test_opt_new_invalidate_new() { + eval(r#" + class Foo; end + def test = Foo.new + test + "#); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(r#" + result = [test.class.name] + def Foo.new = "foo" + result << test + "#), @r#"["Foo", "foo"]"#); +} + +#[test] +fn test_opt_newarray_send_include_p() { + eval(" + def test(x) + [:y, 1, Object.new].include?(x) + end + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("[test(1), test(\"n\")]"), @"[true, false]"); +} + +#[test] +fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) ? :true : :false + end + end + def test(x) + [:y, 1, Object.new].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(x) + [:y, 1, Object.new].include?(x) + end + test(1) + [test(1), test(\"n\")] + "), @"[:true, :false]"); +} + +#[test] +fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:y, 1].include?(x) + end + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(assert_compiles("[test(1), test(\"n\")]"), @"[true, false]"); +} + +#[test] +fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) ? :true : :false + end + end + def test(x) + [:y, 1].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(x) + [:y, 1].include?(x) + end + test(1) + [test(1), test(\"n\")] + "), @"[:true, :false]"); +} + +#[test] +fn test_opt_newarray_send_pack() { + eval(r#" + def test(num) + [num].pack('C') + end + test(65) + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles(r#" + [test(65), test(66), test(67)] + "#), @r#"["A", "B", "C"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_redefined() { + eval(r#" + class Array + alias_method :old_pack, :pack + def pack(fmt, buffer: nil) + "override:#{old_pack(fmt, buffer: buffer)}" + end + end + def test(num) + [num].pack('C') + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(r#" + [test(65), test(66), test(67)] + "#), @r#"["override:A", "override:B", "override:C"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(num, buffer) + [num].pack('C', buffer:) + end + test(65, "") + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles(r#" + buf = "" + [test(65, buf), test(66, buf), test(67, buf), buf] + "#), @r#"["ABC", "ABC", "ABC", "ABC"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + alias_method :old_pack, :pack + def pack(fmt, buffer: nil) + old_pack(fmt, buffer: buffer) + "b" + end + end + def test(num, buffer) + [num].pack('C', buffer:) + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(r#" + def test(num, buffer) + [num].pack('C', buffer:) + end + buf = "" + test(65, buf) + buf = "" + [test(65, buf), buf] + "#), @r#"["b", "A"]"#); +} + +#[test] +fn test_opt_newarray_send_hash() { + eval(" + def test(x) + [1, 2, x].hash + end + test(20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("test(20).class"), @"Integer"); +} + +#[test] +fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + def test(x) + [1, 2, x].hash + end + test(20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits("test(20)"), @"42"); +} + +#[test] +fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + test(10, 20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("[test(10, 20), test(40, 30)]"), @"[20, 40]"); +} + +#[test] +fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(a,b) = [a,b].max + test(15, 30) + [test(15, 30), test(45, 35)] + "), @"[60, 90]"); +} + +#[test] +fn test_new_hash_empty() { + eval(" + def test = {} + test + "); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles("test"), @"{}"); +} + +#[test] +fn test_new_hash_nonempty() { + eval(r#" + def test + key = "key" + value = "value" + num = 42 + result = 100 + {key => value, num => result} + end + test + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test"#), @r#"{"key" => "value", 42 => 100}"#); +} + +#[test] +fn test_new_hash_single_key_value() { + eval(r#" + def test = {"key" => "value"} + test + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test"#), @r#"{"key" => "value"}"#); +} + +#[test] +fn test_new_hash_with_computation() { + eval(r#" + def test(a, b) + {"sum" => a + b, "product" => a * b} + end + test(2, 3) + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test(2, 3)"#), @r#"{"sum" => 5, "product" => 6}"#); +} + +#[test] +fn test_new_hash_with_user_defined_hash_method() { + assert_snapshot!(inspect(r#" + class CustomKey + attr_reader :val + def initialize(val) + @val = val + end + def hash + @val.hash + end + def eql?(other) + other.is_a?(CustomKey) && @val == other.val + end + end + def test + key = CustomKey.new("key") + hash = {key => "value"} + hash[key] == "value" + end + test + test + "#), @"true"); +} + +#[test] +fn test_new_hash_with_user_hash_method_exception() { + assert_snapshot!(inspect(r#" + class BadKey + def hash + raise "Hash method failed!" + end + end + def test + key = BadKey.new + {key => "value"} + end + begin + test + rescue => e + e.class + end + begin + test + rescue => e + e.class + end + "#), @"RuntimeError"); +} + +#[test] +fn test_new_hash_with_user_eql_method_exception() { + assert_snapshot!(inspect(r#" + class BadKey + def hash + 42 + end + def eql?(other) + raise "Eql method failed!" + end + end + def test + key1 = BadKey.new + key2 = BadKey.new + {key1 => "value1", key2 => "value2"} + end + begin + test + rescue => e + e.class + end + begin + test + rescue => e + e.class + end + "#), @"RuntimeError"); +} + +#[test] +fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(assert_compiles(" + result = [test] + class Hash + def freeze = 5 + end + result << test + "), @"[{}, 5]"); +} + +#[test] +fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze = 5 + end + def test = {}.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_aset_hash() { + eval(" + def test(h, k, v) + h[k] = v + end + test({}, :key, 42) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("h = {}; test(h, :key, 42); h[:key]"), @"42"); +} + +#[test] +fn test_opt_aset_hash_returns_value() { + assert_snapshot!(inspect(" + def test(h, k, v) + h[k] = v + end + test({}, :key, 100) + test({}, :key, 100) + "), @"100"); +} + +#[test] +fn test_opt_aset_hash_string_key() { + assert_snapshot!(inspect(r#" + def test(h, k, v) + h[k] = v + end + h = {} + test(h, "foo", "bar") + test(h, "foo", "bar") + h["foo"] + "#), @r#""bar""#); +} + +#[test] +fn test_opt_aset_hash_subclass() { + assert_snapshot!(inspect(" + class MyHash < Hash; end + def test(h, k, v) + h[k] = v + end + h = MyHash.new + test(h, :key, 42) + test(h, :key, 42) + h[:key] + "), @"42"); +} + +#[test] +fn test_opt_aset_hash_too_few_args() { + assert_snapshot!(inspect(r#" + def test(h) + h.[]= 123 + rescue ArgumentError + "ArgumentError" + end + test({}) + test({}) + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_opt_aset_hash_too_many_args() { + assert_snapshot!(inspect(r#" + def test(h) + h[:a, :b] = :c + rescue ArgumentError + "ArgumentError" + end + test({}) + test({}) + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(assert_compiles(" + result = [test] + class Array + def freeze = 5 + end + result << test + "), @"[[], 5]"); +} + +#[test] +fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze = 5 + end + def test = [].freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(assert_compiles(r#" + result = [test] + class String + def freeze = 5 + end + result << test + "#), @r#"["", 5]"#); +} + +#[test] +fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze = 5 + end + def test = ''.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_str_uminus() { + eval(" + def test = -'' + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(assert_compiles(r#" + result = [test] + class String + def -@ = 5 + end + result << test + "#), @r#"["", 5]"#); +} + +#[test] +fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@ = 5 + end + def test = -'' + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_new_array_empty() { + eval(" + def test = [] + test + "); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(assert_compiles("test"), @"[]"); +} + +#[test] +fn test_new_array_nonempty() { + assert_snapshot!(inspect(" + def a = 5 + def test = [a] + test + test + "), @"[5]"); +} + +#[test] +fn test_new_array_order() { + assert_snapshot!(inspect(" + def a = 3 + def b = 2 + def c = 1 + def test = [a, b, c] + test + test + "), @"[3, 2, 1]"); +} + +#[test] +fn test_array_dup() { + assert_snapshot!(inspect(" + def test = [1,2,3] + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_array_fixnum_aref() { + eval(" + def test(x) = [1,2,3][x] + test(2) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(2)"), @"3"); +} + +#[test] +fn test_array_fixnum_aref_negative_index() { + eval(" + def test(x) = [1,2,3][x] + test(-1) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(-1)"), @"3"); +} + +#[test] +fn test_array_fixnum_aref_out_of_bounds_positive() { + eval(" + def test(x) = [1,2,3][x] + test(10) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles_allowing_exits("test(10)"), @"nil"); +} + +#[test] +fn test_array_fixnum_aref_out_of_bounds_negative() { + eval(" + def test(x) = [1,2,3][x] + test(-10) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles_allowing_exits("test(-10)"), @"nil"); +} + +#[test] +fn test_array_fixnum_aref_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, idx) = arr[idx] + test(MyArray[1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(MyArray[1,2,3], 2)"), @"3"); +} + +#[test] +fn test_array_aref_non_fixnum_index() { + assert_snapshot!(inspect(r#" + def test(arr, idx) = arr[idx] + test([1,2,3], 1) + test([1,2,3], 1) + begin + test([1,2,3], "1") + rescue => e + e.class + end + "#), @"TypeError"); +} + +#[test] +fn test_array_fixnum_aset() { + eval(" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("arr = [1,2,3]; test(arr, 2); arr"), @"[1, 2, 7]"); +} + +#[test] +fn test_array_fixnum_aset_returns_value() { + eval(" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("test([1,2,3], 2)"), @"7"); +} + +#[test] +fn test_array_fixnum_aset_out_of_bounds() { + assert_snapshot!(inspect(" + def test(arr) + arr[5] = 7 + end + arr = [1,2,3] + test(arr) + arr = [1,2,3] + test(arr) + arr + "), @"[1, 2, 3, nil, nil, 7]"); +} + +#[test] +fn test_array_fixnum_aset_negative_index() { + assert_snapshot!(inspect(" + def test(arr) + arr[-1] = 7 + end + arr = [1,2,3] + test(arr) + arr = [1,2,3] + test(arr) + arr + "), @"[1, 2, 7]"); +} + +#[test] +fn test_array_fixnum_aset_shared() { + assert_snapshot!(inspect(" + def test(arr, idx, val) + arr[idx] = val + end + arr = (0..50).to_a + test(arr, 0, -1) + test(arr, 1, -2) + shared = arr[10, 20] + test(shared, 0, 999) + [arr[10], shared[0], arr[0], arr[1]] + "), @"[10, 999, -1, -2]"); +} + +#[test] +fn test_array_fixnum_aset_frozen() { + assert_snapshot!(inspect(" + def test(arr, idx, val) + arr[idx] = val + end + arr = [1,2,3] + test(arr, 1, 9) + test(arr, 1, 9) + arr.freeze + begin + test(arr, 1, 9) + rescue => e + e.class + end + "), @"FrozenError"); +} + +#[test] +fn test_array_fixnum_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, idx) + arr[idx] = 7 + end + test(MyArray.new, 0) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("arr = MyArray.new; test(arr, 0); arr[0]"), @"7"); +} + +#[test] +fn test_array_aset_non_fixnum_index() { + assert_snapshot!(inspect(r#" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 0) + test([1,2,3], 0) + begin + test([1,2,3], "0") + rescue => e + e.class + end + "#), @"TypeError"); +} + +#[test] +fn test_empty_array_pop() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop + test([]) + test([]) + "), @"nil"); +} + +#[test] +fn test_array_pop_no_arg() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop + test([32, 33, 42]) + test([32, 33, 42]) + "), @"42"); +} + +#[test] +fn test_array_pop_arg() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop(2) + test([32, 33, 42]) + test([32, 33, 42]) + "), @"[33, 42]"); +} + +#[test] +fn test_new_range_inclusive() { + assert_snapshot!(inspect(" + def test(a, b) = a..b + test(1, 5) + test(1, 5) + "), @"1..5"); +} + +#[test] +fn test_new_range_exclusive() { + assert_snapshot!(inspect(" + def test(a, b) = a...b + test(1, 5) + test(1, 5) + "), @"1...5"); +} + +#[test] +fn test_new_range_with_literal() { + assert_snapshot!(inspect(" + def test(n) = n..10 + test(3) + test(3) + "), @"3..10"); +} + +#[test] +fn test_new_range_fixnum_both_literals_inclusive() { + eval(" + def test() + a = 2 + (1..a) + end + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test; test"), @"1..2"); +} + +#[test] +fn test_new_range_fixnum_both_literals_exclusive() { + eval(" + def test() + a = 2 + (1...a) + end + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test; test"), @"1...2"); +} + +#[test] +fn test_new_range_fixnum_low_literal_inclusive() { + eval(" + def test(a) = (1..a) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"1..3"); +} + +#[test] +fn test_new_range_fixnum_low_literal_exclusive() { + eval(" + def test(a) = (1...a) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"1...3"); +} + +#[test] +fn test_new_range_fixnum_high_literal_inclusive() { + eval(" + def test(a) = (a..10) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"3..10"); +} + +#[test] +fn test_new_range_fixnum_high_literal_exclusive() { + eval(" + def test(a) = (a...10) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"3...10"); +} + +#[test] +fn test_if() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + end + end + test(3) + [test(3), test(7)] + "), @"[0, nil]"); +} + +#[test] +fn test_if_else() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + else + 1 + end + end + test(3) + [test(3), test(7)] + "), @"[0, 1]"); +} + +#[test] +fn test_if_else_params() { + assert_snapshot!(inspect(" + def test(n, a, b) + if n < 5 + a + else + b + end + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[1, 20]"); +} + +#[test] +fn test_if_else_nested() { + assert_snapshot!(inspect(" + def test(a, b, c, d, e) + if 2 < a + if a < 4 + b + else + c + end + else + if a < 0 + d + else + e + end + end + end + test(-1, 1, 2, 3, 4) + [ + test(-1, 1, 2, 3, 4), + test( 0, 5, 6, 7, 8), + test( 3, 9, 10, 11, 12), + test( 5, 13, 14, 15, 16), + ] + "), @"[3, 8, 9, 14]"); +} + +#[test] +fn test_if_else_chained() { + assert_snapshot!(inspect(" + def test(a) + (if 2 < a then 1 else 2 end) + (if a < 4 then 10 else 20 end) + end + test(0) + [test(0), test(3), test(5)] + "), @"[12, 11, 21]"); +} + +#[test] +fn test_if_elsif_else() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + elsif 8 < n + 1 + else + 2 + end + end + test(3) + [test(3), test(7), test(9)] + "), @"[0, 2, 1]"); +} + +#[test] +fn test_ternary_operator() { + assert_snapshot!(inspect(" + def test(n, a, b) + n < 5 ? a : b + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[1, 20]"); +} + +#[test] +fn test_ternary_operator_nested() { + assert_snapshot!(inspect(" + def test(n, a, b) + (n < 5 ? a : b) + 1 + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[2, 21]"); +} + +#[test] +fn test_while_loop() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + i = i + 1 + end + i + end + test(10) + test(10) + "), @"10"); +} + +#[test] +fn test_while_loop_chain() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + i = i + 1 + end + while i < n * 10 + i = i * 3 + end + i + end + test(5) + [test(5), test(10)] + "), @"[135, 270]"); +} + +#[test] +fn test_while_loop_nested() { + assert_snapshot!(inspect(" + def test(n, m) + i = 0 + while i < n + j = 0 + while j < m + j += 2 + end + i += j + end + i + end + test(0, 0) + [test(0, 0), test(1, 3), test(10, 5)] + "), @"[0, 4, 12]"); +} + +#[test] +fn test_while_loop_if_else() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + if n >= 10 + return -1 + else + i = i + 1 + end + end + i + end + test(9) + [test(9), test(10)] + "), @"[9, -1]"); +} + +#[test] +fn test_if_while_loop() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + if n < 10 + while i < n + i += 1 + end + else + while i < n + i += 3 + end + end + i + end + test(9) + [test(9), test(10)] + "), @"[9, 12]"); +} + +#[test] +fn test_live_reg_past_ccall() { + assert_snapshot!(inspect(" + def callee = 1 + def test = callee + callee + test + test + "), @"2"); +} + +#[test] +fn test_method_call() { + assert_snapshot!(inspect(" + def callee(a, b) + a - b + end + def test + callee(4, 2) + 10 + end + test + test + "), @"12"); +} + +#[test] +fn test_recursive_fact() { + assert_snapshot!(inspect(" + def fact(n) + if n == 0 + return 1 + end + return n * fact(n-1) + end + fact(0) + [fact(0), fact(3), fact(6)] + "), @"[1, 6, 720]"); +} + +#[test] +fn test_recursive_fib() { + assert_snapshot!(inspect(" + def fib(n) + if n < 2 + return n + end + return fib(n-1) + fib(n-2) + end + fib(0) + [fib(0), fib(3), fib(4)] + "), @"[0, 2, 3]"); +} + +#[test] +fn test_spilled_basic_block_args() { + assert_snapshot!(inspect(" + def test(n1, n2) + n3 = 3 + n4 = 4 + n5 = 5 + n6 = 6 + n7 = 7 + n8 = 8 + n9 = 9 + n10 = 10 + if n1 < n2 + n1 + n2 + n3 + n4 + n5 + n6 + n7 + n8 + n9 + n10 + end + end + test(1, 2) + test(1, 2) + "), @"55"); +} + +#[test] +fn test_putself() { + assert_snapshot!(inspect(" + class Integer + def minus(a) + self - a + end + end + 5.minus(2) + 5.minus(2) + "), @"3"); +} + +#[test] +fn test_getinstancevariable_nil() { + assert_snapshot!(inspect(" + def test() = @foo + test() + test() + "), @"nil"); +} + +#[test] +fn test_getinstancevariable() { + assert_snapshot!(inspect(" + @foo = 3 + def test() = @foo + test() + test() + "), @"3"); +} + +#[test] +fn test_getinstancevariable_miss() { + assert_snapshot!(inspect(" + class C + def foo + @foo + end + def foo_then_bar + @foo = 1 + @bar = 2 + end + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + o1 = C.new + o1.foo_then_bar + result = [] + result << o1.foo + result << o1.foo + o2 = C.new + o2.bar_then_foo + result << o2.foo + result + "), @"[1, 1, 4]"); +} + +#[test] +fn test_setinstancevariable() { + assert_snapshot!(inspect(" + def test() = @foo = 1 + test() + test() + @foo + "), @"1"); +} + +#[test] +fn test_getclassvariable() { + assert_snapshot!(inspect(" + class Foo + def self.test = @@x + end + Foo.class_variable_set(:@@x, 42) + Foo.test() + Foo.test() + "), @"42"); +} + +#[test] +fn test_getclassvariable_raises() { + assert_snapshot!(inspect(r#" + class Foo + def self.test = @@x + end + begin + Foo.test + Foo.test + rescue NameError => e + e.message + end + "#), @r#""uninitialized class variable @@x in Foo""#); +} + +#[test] +fn test_setclassvariable() { + assert_snapshot!(inspect(" + class Foo + def self.test = @@x = 42 + end + Foo.test() + Foo.test() + Foo.class_variable_get(:@@x) + "), @"42"); +} + +#[test] +fn test_setclassvariable_raises() { + assert_snapshot!(inspect(r#" + class Foo + def self.test = @@x = 42 + freeze + end + begin + Foo.test + Foo.test + rescue FrozenError => e + e.message + end + "#), @r#""can't modify frozen Class: Foo""#); +} + +#[test] +fn test_attr_reader() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 4 + end + end + def test(c) = c.foo + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[4, 4]"); +} + +#[test] +fn test_attr_accessor_getivar() { + eval(" + class C + attr_accessor :foo + def initialize + @foo = 4 + end + end + def test(c) = c.foo + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[4, 4]"); +} + +#[test] +fn test_getivar_t_data_then_string() { + // This is a regression test for a type confusion miscomp where + // we end up reading the fields object using an offset off of a + // string, assuming that it has a the same layout as a T_DATA object. + // At the time of writing the fields object of strings are stored + // in a global table, out-of-line of each string. + // The string and the thread end up sharing one shape ID. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class Thread + include GetThousand + end + class String + include GetThousand + end + OBJ = Thread.new { } + OBJ.join + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + OBJ.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + OBJ.test; OBJ.test # profile and compile for Thread (T_DATA) + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + +#[test] +fn test_getivar_t_object_then_string() { + // This test construct an object and a string that have the same set of ivars. + // They wouldn't share the same shape ID, though, and we rely on this fact in + // our guards. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class MyObject + include GetThousand + end + class String + include GetThousand + end + OBJ = MyObject.new + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + OBJ.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + OBJ.test; OBJ.test # profile and compile for MyObject + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + +#[test] +fn test_getivar_t_class_then_string() { + // This is a regression test for a type confusion miscomp where + // we end up reading the fields object using an offset off of a + // string, assuming that it has a the same layout as a T_CLASS object. + // At the time of writing the fields object of strings are stored + // in a global table, out-of-line of each string. + // The string and the class end up sharing one shape ID. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class MyClass + extend GetThousand + end + class String + include GetThousand + end + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + MyClass.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + p MyClass.test; p MyClass.test # profile and compile for MyClass + p STR.test + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + + +#[test] +fn test_attr_accessor_setivar() { + eval(" + class C + attr_accessor :foo + def initialize + @foo = 4 + end + end + def test(c) + c.foo = 5 + c.foo + end + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[5, 5]"); +} + +#[test] +fn test_attr_writer() { + eval(" + class C + attr_writer :foo + def initialize + @foo = 4 + end + def get_foo = @foo + end + def test(c) + c.foo = 5 + c.get_foo + end + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[5, 5]"); +} + +#[test] +fn test_getconstant() { + eval(" + class Foo + CONST = 1 + end + def test(klass) + klass::CONST + end + test(Foo) + "); + assert_contains_opcode("test", YARVINSN_getconstant); + assert_snapshot!(assert_compiles("test(Foo)"), @"1"); +} + +#[test] +fn test_expandarray_no_splat() { + eval(" + def test(o) + a, b = o + [a, b] + end + test [3, 4] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles("test [3, 4]"), @"[3, 4]"); +} + +#[test] +fn test_expandarray_splat() { + eval(" + def test(o) + a, *b = o + [a, b] + end + test [3, 4] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles_allowing_exits("test [3, 4]"), @"[3, [4]]"); +} + +#[test] +fn test_expandarray_splat_post() { + eval(" + def test(o) + a, *b, c = o + [a, b, c] + end + test [3, 4, 5] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles_allowing_exits("test [3, 4, 5]"), @"[3, [4], 5]"); +} + +#[test] +fn test_constant_invalidation() { + eval(" + class C; end + def test = C + test + test + C = 123 + "); + assert_contains_opcode("test", YARVINSN_opt_getconstant_path); + assert_snapshot!(assert_compiles("test"), @"123"); +} + +#[test] +fn test_constant_path_invalidation() { + eval(" + module A + module B; end + end + module Foo + C = 'Foo::C' + end + A::B = Foo + def test = A::B::C + "); + assert_contains_opcode("test", YARVINSN_opt_getconstant_path); + assert_snapshot!(assert_compiles(r#" + module A + module B; end + end + module Foo + C = "Foo::C" + end + module Bar + C = "Bar::C" + end + A::B = Foo + def test = A::B::C + result = [] + result << test + result << test + A::B = Bar + result << test + result + "#), @r#"["Foo::C", "Foo::C", "Bar::C"]"#); +} + +#[test] +fn test_dupn() { + eval(" + def test(array) = (array[1, 2] ||= :rhs) + test([1, 1]) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(assert_compiles_allowing_exits(" + one = [1, 1] + start_empty = [] + [test(one), one, test(start_empty), start_empty] + "), @"[[1], [1, 1], :rhs, [nil, :rhs]]"); +} + +#[test] +fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); +} + +#[test] +fn test_defined_with_defined_values() { + eval(" + class Foo; end + def bar; end + $ruby = 1 + def test = [defined?(Foo), defined?(bar), defined?($ruby)] + test + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("test"), @r#"["constant", "method", "global-variable"]"#); +} + +#[test] +fn test_defined_with_undefined_values() { + eval(" + def test = [defined?(FooUndef), defined?(bar_undef), defined?($ruby_undef)] + test + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("test"), @"[nil, nil, nil]"); +} + +#[test] +fn test_defined_with_method_call() { + eval(r#" + def test = [defined?("x".reverse(1)), defined?("x".reverse(1).reverse)] + test + "#); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles(r#"test"#), @r#"["method", nil]"#); +} + +#[test] +fn test_defined_method_raise() { + assert_snapshot!(inspect(r#" + class C + def assert_equal expected, actual + if expected != actual + raise "NO" + end + end + def test_defined_method + assert_equal(nil, defined?("x".reverse(1).reverse)) + end + end + c = C.new + result = [] + result << c.test_defined_method + result << c.test_defined_method + result << c.test_defined_method + result + "#), @"[nil, nil, nil]"); +} + +#[test] +fn test_defined_yield() { + eval(" + def test = defined?(yield) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("[test, test, test{}]"), @r#"[nil, nil, "yield"]"#); +} + +#[test] +fn test_defined_yield_from_block() { + assert_snapshot!(inspect(" + def test + yield_self { yield_self { defined?(yield) } } + end + [test, test, test{}] + "), @r#"[nil, nil, "yield"]"#); +} + +#[test] +fn test_block_given_p() { + assert_snapshot!(inspect(" + def test = block_given? + [test, test, test{}] + "), @"[false, false, true]"); +} + +#[test] +fn test_block_given_p_from_block() { + assert_snapshot!(inspect(" + def test + yield_self { yield_self { block_given? } } + end + [test, test, test{}] + "), @"[false, false, true]"); +} + +#[test] +fn test_invokeblock_without_block_after_jit_call() { + assert_snapshot!(inspect(r#" + def test(*arr, &b) + arr.class + yield + end + test { } + begin + test + rescue => e + e.message + end + "#), @r#""no block given (yield)""#); +} + +#[test] +fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias bar test + 10 + end + test + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(assert_compiles("bar"), @"10"); +} + +#[test] +fn test_putspecialobject_const_base() { + assert_snapshot!(inspect(" + Foo = 1 + def test = Foo + test + test + "), @"1"); +} + +#[test] +fn test_branchnil() { + eval(" + def test(x) + x&.succ + end + test(0) + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(assert_compiles("[test(1), test(nil)]"), @"[2, nil]"); +} + +#[test] +fn test_nil_nil() { + eval(" + def test = nil.nil? + test + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test"), @"true"); +} + +#[test] +fn test_non_nil_nil() { + eval(" + def test = 1.nil? + test + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test"), @"false"); +} + +#[test] +fn test_getspecial_last_match() { + eval(r#" + def test(str) + str =~ /hello/ + $& + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello""#); +} + +#[test] +fn test_getspecial_match_pre() { + eval(r#" + def test(str) + str =~ /world/ + $` + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello ""#); +} + +#[test] +fn test_getspecial_match_post() { + eval(r#" + def test(str) + str =~ /hello/ + $' + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#"" world""#); +} + +#[test] +fn test_getspecial_match_last_group() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $+ + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""world""#); +} + +#[test] +fn test_getspecial_numbered_match_1() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $1 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello""#); +} + +#[test] +fn test_getspecial_numbered_match_2() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $2 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""world""#); +} + +#[test] +fn test_getspecial_numbered_match_nonexistent() { + eval(r#" + def test(str) + str =~ /(hello)/ + $2 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @"nil"); +} + +#[test] +fn test_getspecial_no_match() { + eval(r#" + def test(str) + str =~ /xyz/ + $& + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @"nil"); +} + +#[test] +fn test_getspecial_complex_pattern() { + eval(r#" + def test(str) + str =~ /(\d+)/ + $1 + end + test("abc123def") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("abc123def")"#), @r#""123""#); +} + +#[test] +fn test_getspecial_multiple_groups() { + eval(r#" + def test(str) + str =~ /(\d+)-(\d+)/ + $2 + end + test("123-456") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("123-456")"#), @r#""456""#); +} + +// In a JIT-to-JIT call, gen_push_frame writes JIT_RETURN_POISON to the +// callee's cfp->jit_return (runtime_checks builds). On the *first* such +// call the function stub trampoline clears jit_return to NULL, so the +// crash only manifests on the second JIT-to-JIT hit when the stub has +// been patched to jump directly to the callee's JIT entry. Putting $& as +// the first C call in the callee keeps the poison live until +// gen_getspecial_symbol calls rb_backref_get → rb_vm_svar_lep → CFP_PC → +// CFP_ZJIT_FRAME, which dereferences the poison without the prep fix. +#[test] +fn test_getspecial_symbol_in_jit_to_jit_callee() { + eval(r#" + def callee = $& + def caller_method = callee + + # Warm up callee so it JITs + callee + callee + + # First call to caller_method profiles; second JITs caller_method + # and runs through the function-stub-hit path which clears + # jit_return. The third call goes through the patched stub with + # POISON intact, hitting the bug. + caller_method + caller_method + "#); + assert_contains_opcode("callee", YARVINSN_getspecial); + assert_snapshot!(assert_compiles("caller_method"), @"nil"); +} + +// Same JIT-to-JIT setup, exercising gen_getspecial_number ($N). +#[test] +fn test_getspecial_number_in_jit_to_jit_callee() { + eval(r#" + def callee = $1 + def caller_method = callee + + callee + callee + + caller_method + caller_method + "#); + assert_contains_opcode("callee", YARVINSN_getspecial); + assert_snapshot!(assert_compiles("caller_method"), @"nil"); +} + +#[test] +fn test_profile_under_nested_jit_call() { + assert_snapshot!(inspect(" + def profile + 1 + 2 + end + def jit_call(flag) + if flag + profile + end + end + def entry(flag) + jit_call(flag) + end + [entry(false), entry(false), entry(true)] + "), @"[nil, nil, 3]"); +} + +#[test] +fn test_bop_redefined() { + assert_snapshot!(inspect(" + def test + 1 + 2 + end + test + [test, Integer.class_eval { def +(_) = 100 }, test] + "), @"[3, :+, 100]"); +} + +#[test] +fn test_bop_redefined_with_adjacent_patch_points() { + assert_snapshot!(inspect(" + def test + 1 + 2 + 3 + 4 + 5 + end + test + [test, Integer.class_eval { def +(_) = 100 }, test] + "), @"[15, :+, 100]"); +} + +#[test] +fn test_method_redefined_with_top_self() { + assert_snapshot!(inspect(r#" + def foo + "original" + end + def test = foo + test; test + result1 = test + def foo + "redefined" + end + result2 = test + [result1, result2] + "#), @r#"["original", "redefined"]"#); +} + +#[test] +fn test_method_redefined_with_module() { + assert_snapshot!(inspect(r#" + module Foo + def self.foo = "original" + end + def test = Foo.foo + test + result1 = test + def Foo.foo = "redefined" + result2 = test + [result1, result2] + "#), @r#"["original", "redefined"]"#); +} + +#[test] +fn test_module_name_with_guard_passes() { + assert_snapshot!(inspect(r#" + def test(mod) + mod.name + end + test(String) + test(Integer) + "#), @r#""Integer""#); +} + +#[test] +fn test_module_name_with_guard_side_exit() { + assert_snapshot!(inspect(r#" + class MyClass + def name = "Bar" + end + def test(mod) + mod.name + end + results = [] + results << test(String) + results << test(Integer) + results << test(MyClass.new) + results + "#), @r#"["String", "Integer", "Bar"]"#); +} + +#[test] +fn test_objtostring_calls_to_s_on_non_strings() { + assert_snapshot!(inspect(r##" + results = [] + class Foo + def to_s + "foo" + end + end + def test(str) + "#{str}" + end + results << test(Foo.new) + results << test(Foo.new) + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_rewrite_does_not_call_to_s_on_strings() { + assert_snapshot!(inspect(r##" + results = [] + class String + def to_s + "bad" + end + end + def test(foo) + "#{foo}" + end + results << test("foo") + results << test("foo") + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_rewrite_does_not_call_to_s_on_string_subclasses() { + assert_snapshot!(inspect(r##" + results = [] + class StringSubclass < String + def to_s + "bad" + end + end + foo = StringSubclass.new("foo") + def test(str) + "#{str}" + end + results << test(foo) + results << test(foo) + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_profiled_string_fastpath() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test('foo'); test('foo') + "##), @r#""foo""#); +} + +#[test] +fn test_objtostring_profiled_string_subclass_fastpath() { + assert_snapshot!(inspect(r##" + class MyString < String; end + def test(str) + "#{str}" + end + foo = MyString.new("foo") + test(foo); test(foo) + "##), @r#""foo""#); +} + +#[test] +fn test_objtostring_profiled_string_fastpath_exits_on_nonstring() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test('foo') + test(1) + "##), @r#""1""#); +} + +#[test] +fn test_objtostring_profiled_nonstring_calls_to_s() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test([1,2,3]); + test([1,2,3]); + "##), @r#""[1, 2, 3]""#); +} + +#[test] +fn test_objtostring_profiled_nonstring_guard_exits_when_string() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test([1,2,3]); + test('foo'); + "##), @r#""foo""#); +} + +#[test] +fn test_string_bytesize_with_guard() { + assert_snapshot!(inspect(" + def test(str) + str.bytesize + end + test('hello') + test('world') + "), @"5"); +} + +#[test] +fn test_string_bytesize_multibyte() { + assert_snapshot!(inspect(r#" + def test(s) + s.bytesize + end + test("💎") + test("💎") + "#), @"4"); +} + +#[test] +fn test_nil_value_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(nil) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(nil)"), @"true"); +} + +#[test] +fn test_nil_value_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(nil) + test(nil) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(1)"), @"false"); +} + +#[test] +fn test_true_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(true) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(true)"), @"false"); +} + +#[test] +fn test_true_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(true) + test(true) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_false_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(false) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(false)"), @"false"); +} + +#[test] +fn test_false_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(false) + test(false) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_integer_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(2)"), @"false"); +} + +#[test] +fn test_integer_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(1) + test(2) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_float_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(1.0) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(2.0)"), @"false"); +} + +#[test] +fn test_float_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(1.0) + test(2.0) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_symbol_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(:foo) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(:bar)"), @"false"); +} + +#[test] +fn test_symbol_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(:foo) + test(:bar) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_class_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(String) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(Integer)"), @"false"); +} + +#[test] +fn test_class_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(String) + test(Integer) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_module_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(Enumerable) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(Kernel)"), @"false"); +} + +#[test] +fn test_module_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(Enumerable) + test(Kernel) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_basic_object_guard_works_with_immediate() { + assert_snapshot!(inspect(" + class Foo; end + def test(val) = val.class + test(Foo.new) + test(Foo.new) + test(nil) + "), @"NilClass"); +} + +#[test] +fn test_basic_object_guard_works_with_false() { + assert_snapshot!(inspect(" + class Foo; end + def test(val) = val.class + test(Foo.new) + test(Foo.new) + test(false) + "), @"FalseClass"); +} + +#[test] +fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + test + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(assert_compiles(r##"test"##), @r#""123""#); +} + +#[test] +fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + test + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(assert_compiles(r##"test"##), @r#""""#); +} + +#[test] +fn test_regexp_interpolation() { + eval(r##" + def test = /#{1}#{2}#{3}/ + test + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(assert_compiles(r##"test"##), @"/123/"); +} + +#[test] +fn test_new_range_non_leaf() { + assert_snapshot!(inspect(" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "), @"(0/1)..1"); +} + +#[test] +fn test_raise_in_second_argument() { + assert_snapshot!(inspect(" + def write(hash, key) + hash[key] = raise rescue true + hash + end + write({}, :warmup) + write({}, :ok) + "), @"{ok: true}"); +} + +#[test] +fn test_struct_set() { + assert_snapshot!(inspect(" + C = Struct.new(:foo).new(1) + def test + C.foo = Object.new + 42 + end + r = [test, test] + C.freeze + r << begin + test + rescue FrozenError + :frozen_error + end + "), @"[42, 42, :frozen_error]"); +} + +#[test] +fn test_opt_case_dispatch() { + eval(" + def test(x) + case x + when :foo + true + else + false + end + end + test(:warmup) + "); + assert_contains_opcode("test", YARVINSN_opt_case_dispatch); + assert_snapshot!(assert_compiles("[test(:foo), test(1)]"), @"[true, false]"); +} + +#[test] +fn test_checkmatch_case() { + eval(r#" + def test(o) + case o + in Integer + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles(r#"[test(1), test(2), test("3")]"#), @"[1, 1, 2]"); +} + +#[test] +fn test_checkmatch_case_splat_array() { + eval(r#" + def test(o) + case o + when *[1, 2] + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles("[test(1), test(2), test(3)]"), @"[1, 1, 2]"); +} + +#[test] +fn test_checkmatch_when_splat_array() { + eval(r#" + def test + case + when *[1, 2] + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_checkmatch_rescue() { + // Rescue behavior is tested functionally here. It still side-exits because + // JIT exception handling is not supported yet. + eval(r#" + def test + begin + raise TypeError + rescue TypeError + 1 + end + end + "#); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_checkmatch_rescue_splat_array() { + eval(r#" + def test + begin + raise TypeError + rescue *[TypeError, ArgumentError] + 1 + end + end + "#); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_stack_overflow() { + assert_snapshot!(inspect(" + def recurse(n) + return if n == 0 + recurse(n-1) + nil + end + recurse(2) + recurse(2) + begin + recurse(20_000) + rescue SystemStackError + end + "), @"nil"); +} + +#[test] +fn test_invokeblock() { + eval(" + def test + yield + end + test { 41 } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test { 42 }"), @"42"); +} + +#[test] +fn test_invokeblock_with_args() { + eval(" + def test(x, y) + yield x, y + end + test(1, 2) { |a, b| a + b } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test(1, 2) { |a, b| a + b }"), @"3"); +} + +#[test] +fn test_invokeblock_no_block_given() { + eval(" + def test + yield rescue :error + end + test { } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test"), @":error"); +} + +#[test] +fn test_invokeblock_multiple_yields() { + eval(" + def test + yield 1 + yield 2 + yield 3 + end + test { |x| x } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles(" + results = [] + test { |x| results << x } + results + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokeblock_ifunc_map() { + eval(" + class MyList + include Enumerable + def each + yield 1 + yield 2 + yield 3 + end + end + def test = MyList.new.map { |x| x * 2 } + test + "); + assert_snapshot!(assert_compiles("test"), @"[2, 4, 6]"); +} + +#[test] +fn test_ccall_variadic_with_multiple_args() { + eval(" + def test + a = [] + a.push(1, 2, 3) + a + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @"[1, 2, 3]"); +} + +#[test] +fn test_ccall_variadic_with_no_args() { + eval(" + def test + a = [1] + a.push + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @"[1]"); +} + +#[test] +fn test_ccall_variadic_with_no_args_causing_argument_error() { + eval(" + def test + format + rescue ArgumentError + :error + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @":error"); +} + +#[test] +fn test_allocating_in_hir_c_method_is() { + eval(" + def a(f) = test(f) + def test(f) = (f.new if f) + def second = third + def third = nil + a(nil) + a(nil) + class Foo + def self.new = :k + end + second + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles_allowing_exits("a(Foo)"), @":k"); +} + +#[test] +fn test_singleton_class_invalidation_annotated_ccall() { + assert_snapshot!(inspect(" + def define_singleton(obj, define) + if define + [nil].reverse_each do + class << obj + def ==(_) + true + end + end + end + end + false + end + def test(define) + obj = BasicObject.new + obj == define_singleton(obj, define) + end + result = [] + result << test(false) + result << test(true) + result + "), @"[false, true]"); +} + +#[test] +fn test_singleton_class_invalidation_optimized_variadic_ccall() { + assert_snapshot!(inspect(" + def define_singleton(arr, define) + if define + [nil].reverse_each do + class << arr + def push(x) + super(x * 1000) + end + end + end + end + 1 + end + def test(define) + arr = [] + val = define_singleton(arr, define) + arr.push(val) + arr[0] + end + result = [] + result << test(false) + result << test(true) + result + "), @"[1, 1000]"); +} + +#[test] +fn test_is_a_string_special_case() { + assert_snapshot!(inspect(r#" + def test(x) + x.is_a?(String) + end + test("foo") + [test("bar"), test(1), test(false), test(:foo), test([]), test({})] + "#), @"[true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_array_special_case() { + assert_snapshot!(inspect(" + def test(x) + x.is_a?(Array) + end + test([]) + [test([1,2,3]), test([]), test(1), test(false), test(:foo), test('foo'), test({})] + "), @"[true, true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_hash_special_case() { + assert_snapshot!(inspect(" + def test(x) + x.is_a?(Hash) + end + test({}) + [test({:a => 'b'}), test({}), test(1), test(false), test(:foo), test([]), test('foo')] + "), @"[true, true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_hash_subclass() { + assert_snapshot!(inspect(" + class MyHash < Hash + end + def test(x) + x.is_a?(Hash) + end + test({}) + test(MyHash.new) + "), @"true"); +} + +#[test] +fn test_is_a_normal_case() { + assert_snapshot!(inspect(r#" + class MyClass + end + def test(x) + x.is_a?(MyClass) + end + test("a") + [test(MyClass.new), test("a")] + "#), @"[true, false]"); +} + +#[test] +fn test_fixnum_div_zero() { + eval(" + def test(n) + n / 0 + rescue ZeroDivisionError => e + e.message + end + test(0) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles_allowing_exits(r#"test(0)"#), @r#""divided by 0""#); +} + +#[test] +fn test_invokesuper_with_local_written_by_blockiseq() { + assert_snapshot!(inspect(r#" + class A + def foo = "A" + end + class B < A + def foo + x = nil + [nil].each do |_| + x = super + end + x + end + end + def test = B.new.foo + test + test + "#), @r#""A""#); +} + +#[test] +fn test_max_iseq_versions() { + let max_versions = max_iseq_versions(); + eval(&format!(" + TEST = -1 + def test = TEST + + # compile and invalidate MAX+1 times + i = 0 + while i < {max_versions} + 1 + test; test # compile a version + + Object.send(:remove_const, :TEST) + TEST = i + + i += 1 + end + ")); + + // It should not exceed MAX_ISEQ_VERSIONS + let iseq = get_method_iseq("self", "test"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(payload.versions.len(), max_iseq_versions()); + + // The last call should not discard the JIT code + assert!(matches!(unsafe { payload.versions.last().unwrap().as_ref() }.status, IseqStatus::Compiled(_))); +} + +#[test] +fn test_optional_arguments_side_exit() { + assert_snapshot!(inspect(" + def test(a = (def foo = nil)) = a + test + [test, (undef :foo), test(1)] + "), @"[:foo, nil, 1]"); +} + +#[test] +fn test_call_a_forwardable_method() { + assert_snapshot!(inspect(" + def test_root = forwardable + def forwardable(...) = Array.[](...) + test_root + test_root + "), @"[]"); +} + +#[test] +fn test_send_on_heap_object_in_spilled_arg() { + assert_snapshot!(inspect(" + def entry(a1, a2, a3, a4, a5, a6, a7, a8, a9) + a9.itself.class + end + entry(1, 2, 3, 4, 5, 6, 7, 8, {}) + entry(1, 2, 3, 4, 5, 6, 7, 8, {}) + "), @"Hash"); +} + +#[test] +fn test_send_splat() { + assert_snapshot!(inspect(" + def test(a, b) = [a, b] + def entry(arr) = test(*arr) + entry([1, 2]) + entry([1, 2]) + "), @"[1, 2]"); +} + +#[test] +fn test_send_kwarg() { + assert_snapshot!(inspect(" + def test(a:, b:) = [a, b] + def entry = test(b: 2, a: 1) + entry + entry + "), @"[1, 2]"); +} + +#[test] +fn test_spilled_method_args() { + assert_snapshot!(inspect(" + def foo(n1, n2, n3, n4, n5, n6, n7, n8, n9, n10) + n1 + n2 + n3 + n4 + n5 + n6 + n7 + n8 + n9 + n10 + end + def test + foo(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + end + test + test + "), @"55"); +} + +#[test] +fn test_spilled_method_args_first_and_last() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8,n9) = n1+n9 + a(2,0,0,0,0,0,0,0,-1) + a(2,0,0,0,0,0,0,0,-1) + "), @"1"); +} + +#[test] +fn test_spilled_method_args_last() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = n8 + a(1,1,1,1,1,1,1,0) + a(1,1,1,1,1,1,1,0) + "), @"0"); +} + +#[test] +fn test_spilled_method_args_self() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = self + a(1,0,0,0,0,0,0,0).to_s + a(1,0,0,0,0,0,0,0).to_s + "), @r#""main""#); +} + +#[test] +fn test_spilled_param_new_array() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = [n8] + a(0,0,0,0,0,0,0, :ok) + a(0,0,0,0,0,0,0, :ok) + "), @"[:ok]"); +} + +#[test] +fn test_forty_param_method() { + assert_snapshot!(inspect(" + def foo(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,n40) = n40 + foo(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1) + foo(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1) + "), @"1"); +} + +#[test] +fn test_toplevel_local_after_eval() { + assert_snapshot!(inspect(" + a = 1 + b = 2 + eval('b = 3') + c = 4 + [a, b, c] + "), @"[1, 3, 4]"); +} + +#[test] +fn test_send_exit_with_uninitialized_locals() { + assert_snapshot!(inspect(" + def entry(init) + function_stub_exit(init) + end + + def function_stub_exit(init) + uninitialized_local = 1 if init + uninitialized_local + end + + entry(true) + entry(false) + "), @"nil"); +} + +#[test] +fn test_invokebuiltin_dir_glob() { + assert_snapshot!(inspect(r#" + def test = Dir.glob(".") + test + test + "#), @r#"["."]"#); +} + +#[test] +fn test_profiled_fact() { + assert_snapshot!(inspect(" + def fact(n) + if n == 0 + return 1 + end + return n * fact(n-1) + end + fact(1) + [fact(0), fact(3), fact(6)] + "), @"[1, 6, 720]"); +} + +#[test] +fn test_profiled_fib() { + assert_snapshot!(inspect(" + def fib(n) + if n < 2 + return n + end + return fib(n-1) + fib(n-2) + end + fib(3) + [fib(0), fib(3), fib(4)] + "), @"[0, 2, 3]"); +} + +#[test] +fn test_single_ractor_mode_invalidation() { + assert_snapshot!(inspect(r#" + C = Object.new + + def test + C + rescue Ractor::IsolationError + "errored but not crashed" + end + + test + test + + Ractor.new { + test + }.value + "#), @r#""errored but not crashed""#); +} + +#[test] +fn test_ivar_attr_reader_optimization_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + class << self + attr_accessor :bar + + def get_bar + bar + rescue Ractor::IsolationError + 42 + end + end + end + + Foo.bar = [] + + def test + Foo.get_bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_ivar_get_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.set_bar + @bar = [] + end + + def self.bar + @bar + rescue Ractor::IsolationError + 42 + end + end + + Foo.set_bar + + def test + Foo.bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_ivar_get_with_already_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.set_bar + @bar = [] + end + + def self.bar + @bar + rescue Ractor::IsolationError + 42 + end + end + + Foo.set_bar + r = Ractor.new { + Ractor.receive + Foo.bar + } + + Foo.bar + Foo.bar + + r << :go + r.value + "), @"42"); +} + +#[test] +fn test_ivar_set_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.bar + _foo = 1 + _bar = 2 + begin + @bar = _foo + _bar + rescue Ractor::IsolationError + 42 + end + end + end + + def test + Foo.bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_global_tracepoint() { + assert_snapshot!(inspect(" + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |event| + if event.method_id == :foo + called = true + end + } + tp.enable do + foo + end + called + "), @"true"); +} + +#[test] +fn test_local_tracepoint() { + assert_snapshot!(inspect(" + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |_| called = true } + tp.enable(target: method(:foo)) do + foo + end + called + "), @"true"); +} + +// Regression test: TracePoint return value for methods with rescue that use `return`. +// ZJIT's send fallback uses rb_vm_opt_send_without_block which calls VM_EXEC, +// setting FLAG_FINISH on the callee frame. This changes how throw TAG_RETURN is +// handled, causing the return value to be nil instead of the actual value. +#[test] +fn test_tracepoint_return_value_with_rescue() { + assert_snapshot!(inspect(" + def f_raise + raise + rescue + return :f_raise_return + end + + ary = [] + TracePoint.new(:return, :b_return){|tp| + ary << [tp.event, tp.method_id, tp.return_value] + }.enable{ + send :f_raise + } + ary.pop # last b_return event is not required + ary + "), @"[[:return, :f_raise, :f_raise_return]]"); +} + +// Regression test: polymorphic getivar must not return nil for too-complex shapes. +// Too-complex shapes use hash tables for ivar storage, and rb_shape_get_iv_index() +// doesn't work for them. The polymorphic path must fall through to GetIvar instead. +#[test] +fn test_polymorphic_getivar_complex_shape() { + // Need threshold >= 3 so both shapes get profiled before compilation + set_call_threshold(3); + assert_snapshot!(inspect(r#" + class C + def initialize(foo) + @foo = foo + end + def foo = @foo + end + + # Create a normal object and a too-complex object of the same class + normal = C.new(:normal) + complex = C.new(:complex) + 1001.times { |i| complex.instance_variable_set(:"@v#{i}", i) } + 1001.times { |i| complex.remove_instance_variable(:"@v#{i}") } + + # Profile with both shapes before compilation triggers at call 3 + normal.foo # call 1: profile normal shape + complex.foo # call 2: profile too-complex shape + + # The too-complex object should still return :complex, not nil + [normal.foo, complex.foo] + "#), @"[:normal, :complex]"); +} + +/// When a method with keyword defaults contains a block that creates a lambda, +/// the lambda causes EP escape, which globally patches NoEPEscape PatchPoints. +/// On subsequent calls the PatchPoint side exit (which uses without_locals()) +/// must not leave stale keyword default values in the frame. We solve this by +/// invalidating the ISEQ version on EP escape so the interpreter takes over. +#[test] +fn test_ep_escape_preserves_keyword_default() { + set_call_threshold(1); + assert_snapshot!(inspect(r#" + def target(dumped, additional_methods: []) + dumped.class + additional_methods.each { |m| ->{ m } } + additional_methods + end + + def forwarder(x, **kwargs) + target(x, **kwargs) + end + + 5.times { forwarder("z") } + forwarder("y", additional_methods: [:to_s]) + target("x") + "#), @"[]"); +} + +#[test] +fn test_send_block_to_accepts_no_block() { + // Methods with &nil should raise ArgumentError when called with a block + assert_snapshot!(inspect(" + def m(a, &nil); a end + + def test + m(1) {} + rescue ArgumentError => e + e.message + end + + test + test + "), @r#""no block accepted""#); +} + +#[test] +fn test_send_block_to_method_not_using_block() { + // Passing a block to a method that doesn't use it should still work correctly. + // ZJIT falls back to the interpreter for this case so that unused block + // warnings are properly emitted. + assert_snapshot!(inspect(" + def m_no_block = 42 + + def test + m_no_block {} + end + + test + test + "), @"42"); +} + +#[test] +fn test_send_block_unused_warning_emitted_from_jit() { + // When ZJIT compiles a send with a block as a dynamic dispatch fallback + // (gen_send -> rb_vm_send), warn_unused_block uses cfp->pc for the dedup + // key. We save cfp->pc before calling rb_vm_send so the key is stable + // and won't spuriously collide with prior entries in the dedup table. + assert_snapshot!(inspect(r#" + $warnings = [] + module Warning + def warn(message, category: nil) + $warnings << message + end + end + + def m_unused_block_warn_test = 42 + + def test + $VERBOSE = true + m_unused_block_warn_test {} + $warnings.any? { |w| w.include?("may be ignored") } + end + + test + test + "#), @"true"); +} + +#[test] +fn test_load_immediates_into_registers_before_masking() { + // See https://github.com/ruby/ruby/pull/16669 -- this is a reduced reproduction from a Ruby + // spec. + set_call_threshold(2); + assert_snapshot!(inspect(r#" + def test + klass = Class.new do + def ===(o) + true + end + end + + case 1 + when klass.new + :called + end == :called + end + + test + test + "#), @"true"); +} + +#[test] +fn test_loop_terminates() { + set_call_threshold(3); + // Previous worklist-based type inference only worked for maximal SSA. This is a regression + // test for hanging. + assert_snapshot!(inspect(r#" + class TheClass + def set_value_loop + i = 0 + while i < 10 + @levar ||= i + i += 1 + end + end + end + + 3.times do |i| + TheClass.new.set_value_loop + end + "#), @"3"); +} + +// Regression test: getlocal with level=0 after setlocal_WC_0 was loading stale EP +// memory, causing Array#pack with buffer: keyword to receive the wrong buffer VALUE. +// See https://github.com/ruby/ruby/pull/16736 +#[test] +fn test_getlocal_level_zero_after_setlocal_wc_0() { + assert_snapshot!(inspect(r#" + def test + b = +"x" + v = 2 + [v].pack("C*", buffer: b) + b.size + end + test + "#), @"2"); +} diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs new file mode 100644 index 0000000000..eb3241b0a2 --- /dev/null +++ b/zjit/src/cruby.rs @@ -0,0 +1,1627 @@ +//! This module deals with making relevant C functions available to Rust ZJIT. +//! Some C functions we use we maintain, some are public C extension APIs, +//! some are internal CRuby APIs. +//! +//! ## General notes about linking +//! +//! The ZJIT crate compiles to a native static library, which for our purposes +//! we can understand as a collection of object files. On ELF platforms at least, +//! object files can refer to "external symbols" which we could take some +//! liberty and understand as assembly labels that refer to code defined in other +//! object files resolved when linking. When we are linking, say to produce miniruby, +//! the linker resolves and put concrete addresses for each usage of C function in +//! the Rust static library. +//! +//! By declaring external functions and using them, we are asserting the symbols +//! we use have definition in one of the object files we pass to the linker. Declaring +//! a function here that has no definition anywhere causes a linking error. +//! +//! There are more things going on during linking and this section makes a lot of +//! simplifications but hopefully this gives a good enough working mental model. +//! +//! ## Difference from example in the Rustonomicon +//! +//! You might be wondering about why this is different from the [FFI example] +//! in the Nomicon, an official book about Unsafe Rust. +//! +//! There is no `#[link]` attribute because we are not linking against an external +//! library, but rather implicitly asserting that we'll supply a concrete definition +//! for all C functions we call, similar to how pure C projects put functions +//! across different compilation units and link them together. +//! +//! TODO(alan): is the model different enough on Windows that this setup is unworkable? +//! Seems prudent to at least learn more about Windows binary tooling before +//! committing to a design. +//! +//! Alan recommends reading the Nomicon cover to cover as he thinks the book is +//! not very long in general and especially for something that can save hours of +//! debugging Undefined Behavior (UB) down the road. +//! +//! UBs can cause Safe Rust to crash, at which point it's hard to tell which +//! usage of `unsafe` in the codebase invokes UB. Providing safe Rust interface +//! wrapping `unsafe` Rust is a good technique, but requires practice and knowledge +//! about what's well defined and what's undefined. +//! +//! For an extremely advanced example of building safe primitives using Unsafe Rust, +//! see the [GhostCell] paper. Some parts of the paper assume less background knowledge +//! than other parts, so there should be learning opportunities in it for all experience +//! levels. +//! +//! ## Binding generation +//! +//! For the moment declarations on the Rust side are hand written. The code is boilerplate +//! and could be generated automatically with a custom tooling that depend on +//! rust-lang/rust-bindgen. The output Rust code could be checked in to version control +//! and verified on CI like `make update-deps`. +//! +//! Upsides for this design: +//! - the ZJIT static lib that links with miniruby and friends will not need bindgen +//! as a dependency at all. This is an important property so Ruby end users can +//! build a ZJIT enabled Ruby with no internet connection using a release tarball +//! - Less hand-typed boilerplate +//! - Helps reduce risk of C definitions and Rust declaration going out of sync since +//! CI verifies synchronicity +//! +//! Downsides and known unknowns: +//! - Using rust-bindgen this way seems unusual. We might be depending on parts +//! that the project is not committed to maintaining +//! - This setup assumes rust-bindgen gives deterministic output, which can't be taken +//! for granted +//! - ZJIT contributors will need to install libclang on their system to get rust-bindgen +//! to work if they want to run the generation tool locally +//! +//! The elephant in the room is that we'll still need to use Unsafe Rust to call C functions, +//! and the binding generation can't magically save us from learning Unsafe Rust. +//! +//! +//! [FFI example]: https://doc.rust-lang.org/nomicon/ffi.html +//! [GhostCell]: http://plv.mpi-sws.org/rustbelt/ghostcell/ + +// CRuby types use snake_case. Allow them so we use one name across languages. +#![allow(non_camel_case_types)] +// A lot of imported CRuby globals aren't all-caps +#![allow(non_upper_case_globals)] +#![allow(clippy::upper_case_acronyms)] + +// Some of this code may not be used yet +#![allow(dead_code)] +#![allow(unused_macros)] +#![allow(unused_imports)] + +use std::convert::From; +use std::ffi::{c_void, CString, CStr}; +use std::fmt::{Debug, Display, Formatter}; +use std::os::raw::{c_char, c_int, c_uint}; +use std::panic::{catch_unwind, UnwindSafe}; + +use crate::cast::IntoUsize as _; + +// We check that we can do this with the configure script and a couple of +// static asserts. u64 and not usize to play nice with lowering to x86. +pub type size_t = u64; + +/// A type alias for the redefinition flags coming from CRuby. These are just +/// shifted 1s but not explicitly an enum. +pub type RedefinitionFlag = u32; + +#[allow(unsafe_op_in_unsafe_fn)] +#[allow(dead_code)] +#[allow(clippy::all)] // warning meant to help with reading; not useful for generated code +mod autogened { + use super::*; + // Textually include output from rust-bindgen as suggested by its user guide. + include!("cruby_bindings.inc.rs"); +} +pub use autogened::*; + +// TODO: For #defines that affect memory layout, we need to check for them +// on build and fail if they're wrong. e.g. USE_FLONUM *must* be true. + +// These are functions we expose from C files, not in any header. +// Parsing it would result in a lot of duplicate definitions. +// Use bindgen for functions that are defined in headers or in zjit.c. +#[cfg_attr(test, allow(unused))] // We don't link against C code when testing +unsafe extern "C" { + pub fn rb_check_overloaded_cme( + me: *const rb_callable_method_entry_t, + ci: *const rb_callinfo, + ) -> *const rb_callable_method_entry_t; + + // Floats within range will be encoded without creating objects in the heap. + // (Range is 0x3000000000000001 to 0x4fffffffffffffff (1.7272337110188893E-77 to 2.3158417847463237E+77). + pub fn rb_float_new(d: f64) -> VALUE; + + pub fn rb_hash_empty_p(hash: VALUE) -> VALUE; + pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE; + pub fn rb_str_getbyte(str: VALUE, index: VALUE) -> VALUE; + pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE; + pub fn rb_jit_fix_div_fix(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_jit_fix_mod_fix(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE; + pub fn rb_vm_get_special_object(reg_ep: *const VALUE, value_type: vm_special_object_type) -> VALUE; + pub fn rb_vm_concat_to_array(ary1: VALUE, ary2st: VALUE) -> VALUE; + pub fn rb_vm_defined( + ec: EcPtr, + reg_cfp: CfpPtr, + op_type: rb_num_t, + obj: VALUE, + v: VALUE, + ) -> bool; + pub fn rb_vm_set_ivar_id(obj: VALUE, idx: u32, val: VALUE) -> VALUE; + pub fn rb_vm_setinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, val: VALUE, ic: IVC); + pub fn rb_vm_getinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, ic: IVC) -> VALUE; + pub fn rb_aliased_callable_method_entry( + me: *const rb_callable_method_entry_t, + ) -> *const rb_callable_method_entry_t; + pub fn rb_vm_getclassvariable(iseq: IseqPtr, cfp: CfpPtr, id: ID, ic: ICVARC) -> VALUE; + pub fn rb_vm_setclassvariable( + iseq: IseqPtr, + cfp: CfpPtr, + id: ID, + val: VALUE, + ic: ICVARC, + ) -> VALUE; + pub fn rb_vm_ic_hit_p(ic: IC, reg_ep: *const VALUE) -> bool; + pub fn rb_vm_stack_canary() -> VALUE; + pub fn rb_vm_push_cfunc_frame(cme: *const rb_callable_method_entry_t, recv_idx: c_int); + pub fn rb_obj_class(klass: VALUE) -> VALUE; + pub fn rb_vm_objtostring(reg_cfp: CfpPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; +} + +// Renames +pub use rb_insn_name as raw_insn_name; +pub use rb_get_ec_cfp as get_ec_cfp; +pub use rb_get_cfp_iseq as get_cfp_iseq; +pub use rb_get_cfp_pc as get_cfp_pc; +pub use rb_get_cfp_sp as get_cfp_sp; +pub use rb_get_cfp_self as get_cfp_self; +pub use rb_get_cfp_ep as get_cfp_ep; +pub use rb_get_cfp_ep_level as get_cfp_ep_level; +pub use rb_vm_base_ptr as get_cfp_bp; +pub use rb_get_cme_def_type as get_cme_def_type; +pub use rb_get_cme_def_body_attr_id as get_cme_def_body_attr_id; +pub use rb_get_cme_def_body_optimized_type as get_cme_def_body_optimized_type; +pub use rb_get_cme_def_body_optimized_index as get_cme_def_body_optimized_index; +pub use rb_get_cme_def_body_cfunc as get_cme_def_body_cfunc; +pub use rb_get_def_method_serial as get_def_method_serial; +pub use rb_get_def_original_id as get_def_original_id; +pub use rb_get_mct_argc as get_mct_argc; +pub use rb_get_mct_func as get_mct_func; +pub use rb_get_def_iseq_ptr as get_def_iseq_ptr; +pub use rb_iseq_encoded_size as get_iseq_encoded_size; +pub use rb_get_iseq_body_local_iseq as get_iseq_body_local_iseq; +pub use rb_get_iseq_body_iseq_encoded as get_iseq_body_iseq_encoded; +pub use rb_get_iseq_body_stack_max as get_iseq_body_stack_max; +pub use rb_get_iseq_body_type as get_iseq_body_type; +pub use rb_get_iseq_body_local_table_size as get_iseq_body_local_table_size; +pub use rb_get_cikw_keyword_len as get_cikw_keyword_len; +pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx; +pub use rb_get_call_data_ci as get_call_data_ci; +pub use rb_FL_TEST as FL_TEST; +pub use rb_FL_TEST_RAW as FL_TEST_RAW; +pub use rb_RB_TYPE_P as RB_TYPE_P; +pub use rb_BASIC_OP_UNREDEFINED_P as BASIC_OP_UNREDEFINED_P; +pub use rb_vm_ci_argc as vm_ci_argc; +pub use rb_vm_ci_mid as vm_ci_mid; +pub use rb_vm_ci_flag as vm_ci_flag; +pub use rb_vm_ci_kwarg as vm_ci_kwarg; +pub use rb_METHOD_ENTRY_VISI as METHOD_ENTRY_VISI; +pub use rb_RCLASS_ORIGIN as RCLASS_ORIGIN; +pub use rb_vm_get_special_object as vm_get_special_object; +pub use rb_jit_fix_div_fix as rb_fix_div_fix; +pub use rb_jit_fix_mod_fix as rb_fix_mod_fix; + +/// Helper so we can get a Rust string for insn_name() +pub fn insn_name(opcode: usize) -> String { + if opcode >= VM_INSTRUCTION_SIZE.try_into().unwrap() { + return "<unknown>".into(); + } + unsafe { + // Look up Ruby's NULL-terminated insn name string + let op_name = raw_insn_name(VALUE(opcode)); + + // Convert the op name C string to a Rust string and concat + let op_name = CStr::from_ptr(op_name).to_str().unwrap(); + + // Convert into an owned string + op_name.to_string() + } +} + +pub fn insn_len(opcode: usize) -> u32 { + unsafe { + rb_insn_len(VALUE(opcode)).try_into().unwrap() + } +} + +/// We avoid using bindgen for `rb_iseq_constant_body` since its definition changes depending +/// on build configuration while we need one bindgen file that works for all configurations. +/// Use an opaque type for it instead. +/// See: <https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs> +#[repr(C)] +pub struct rb_iseq_constant_body { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// An object handle similar to VALUE in the C code. Our methods assume +/// that this is a handle. Sometimes the C code briefly uses VALUE as +/// an unsigned integer type and don't necessarily store valid handles but +/// thankfully those cases are rare and don't cross the FFI boundary. +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +#[repr(transparent)] // same size and alignment as simply `usize` +pub struct VALUE(pub usize); + +/// An interned string. See [ids] and methods this type. +/// `0` is a sentinal value for IDs. +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub struct ID(pub ::std::os::raw::c_ulong); + +/// Pointer to an ISEQ +pub type IseqPtr = *const rb_iseq_t; + +/// Index of a YARV instruction within an ISEQ's bytecode array. +pub type YarvInsnIdx = usize; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ShapeId(pub u32); + +pub const INVALID_SHAPE_ID: ShapeId = ShapeId(rb_invalid_shape_id); + +impl ShapeId { + pub fn is_valid(self) -> bool { + self != INVALID_SHAPE_ID + } + + pub fn is_complex(self) -> bool { + unsafe { rb_jit_shape_complex_p(self.0) } + } + + pub fn is_frozen(self) -> bool { + (self.0 & SHAPE_ID_FL_FROZEN) != 0 + } +} + +// Given an ISEQ pointer, convert PC to insn_idx +pub fn iseq_pc_to_insn_idx(iseq: IseqPtr, pc: *mut VALUE) -> Option<u16> { + let pc_zero = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + unsafe { pc.offset_from(pc_zero) }.try_into().ok() +} + +/// Given an ISEQ pointer and an instruction index, return an opcode. +pub fn iseq_opcode_at_idx(iseq: IseqPtr, insn_idx: u32) -> u32 { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + unsafe { rb_iseq_opcode_at_pc(iseq, pc) as u32 } +} + +/// Return true if a given ISEQ is known to escape EP to the heap on entry. +/// +/// As of vm_push_frame(), EP is always equal to BP. However, after pushing +/// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + match unsafe { get_iseq_body_type(iseq) } { + // The EP of the <main> frame points to TOPLEVEL_BINDING + ISEQ_TYPE_MAIN | + // eval frames point to the EP of another frame or scope + ISEQ_TYPE_EVAL => true, + _ => false, + } +} + +/// Index of the local variable that has a rest parameter if any +pub fn iseq_rest_param_idx(params: &IseqParameters) -> Option<i32> { + // TODO(alan): replace with `params.rest_start` + if params.flags.has_rest() != 0 { + Some(params.opt_num + params.lead_num) + } else { + None + } +} + +/// Iterate over all existing ISEQs +pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { + unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { + // SAFETY: points to the local below + let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast(); + unsafe { (**callback)(iseq) }; + } + let mut data: *mut dyn FnMut(IseqPtr) = &raw mut callback; + let data: *mut *mut dyn FnMut(IseqPtr) = &raw mut data; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), data.cast()) }; +} + +/// Return a poison value to be set above the stack top to verify leafness. +#[cfg(not(test))] +pub fn vm_stack_canary() -> u64 { + unsafe { rb_vm_stack_canary() }.as_u64() +} + +/// Avoid linking the C function in `cargo test` +#[cfg(test)] +pub fn vm_stack_canary() -> u64 { + 0 +} + +/// Opaque execution-context type from vm_core.h +#[repr(C)] +pub struct rb_execution_context_struct { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} +/// Alias for rb_execution_context_struct used by CRuby sometimes +pub type rb_execution_context_t = rb_execution_context_struct; + +/// Pointer to an execution context (rb_execution_context_struct) +pub type EcPtr = *const rb_execution_context_struct; + +// From method.h +#[repr(C)] +pub struct rb_method_definition_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} +type rb_method_definition_struct = rb_method_definition_t; + +/// Opaque cfunc type from method.h +#[repr(C)] +pub struct rb_method_cfunc_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Opaque call-cache type from vm_callinfo.h +#[repr(C)] +pub struct rb_callcache { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Pointer to a control frame pointer (CFP) +pub type CfpPtr = *mut rb_control_frame_struct; + +/// Opaque struct from vm_core.h +#[repr(C)] +pub struct rb_cref_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +#[derive(PartialEq)] +pub enum ClassRelationship { + Subclass, + Superclass, + NoRelation, +} + +/// A print adapator for debug info about a [VALUE]. Includes info +/// the GC knows about the handle. Example: `println!("{}", value.obj_info());`. +pub struct ObjInfoPrinter(VALUE); + +impl Display for ObjInfoPrinter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use std::mem::MaybeUninit; + const BUFFER_SIZE: usize = 0x100; + let mut buffer: MaybeUninit<[c_char; BUFFER_SIZE]> = MaybeUninit::uninit(); + let info = unsafe { + rb_raw_obj_info(buffer.as_mut_ptr().cast(), BUFFER_SIZE, self.0); + CStr::from_ptr(buffer.as_ptr().cast()).to_string_lossy() + }; + write!(f, "{info}") + } +} + +impl VALUE { + /// Get a printer for raw debug info from `rb_obj_info()` about the value. + pub fn obj_info(self) -> ObjInfoPrinter { + ObjInfoPrinter(self) + } + + /// Return whether the value is truthy or falsy in Ruby -- only nil and false are falsy. + pub fn test(self) -> bool { + let VALUE(cval) = self; + let VALUE(qnilval) = Qnil; + (cval & !qnilval) != 0 + } + + /// Return true if the number is an immediate integer, flonum or static symbol + fn immediate_p(self) -> bool { + let VALUE(cval) = self; + let mask = RUBY_IMMEDIATE_MASK as usize; + (cval & mask) != 0 + } + + /// Return true if the value is a Ruby immediate integer, flonum, static symbol, nil or false + pub fn special_const_p(self) -> bool { + self.immediate_p() || !self.test() + } + + /// Return true if the value is a heap object + pub fn heap_object_p(self) -> bool { + !self.special_const_p() + } + + /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`. + pub fn shareable_p(self) -> bool { + (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0 + } + + /// Return true if the value is a Ruby Fixnum (immediate-size integer) + pub fn fixnum_p(self) -> bool { + let VALUE(cval) = self; + let flag = RUBY_FIXNUM_FLAG as usize; + (cval & flag) == flag + } + + /// Return true if the value is an immediate Ruby floating-point number (flonum) + pub fn flonum_p(self) -> bool { + let VALUE(cval) = self; + let mask = RUBY_FLONUM_MASK as usize; + let flag = RUBY_FLONUM_FLAG as usize; + (cval & mask) == flag + } + + /// Return true if the value is a Ruby symbol (RB_SYMBOL_P) + pub fn symbol_p(self) -> bool { + self.static_sym_p() || self.dynamic_sym_p() + } + + pub fn instance_can_have_singleton_class(self) -> bool { + if self == unsafe { rb_cInteger } || self == unsafe { rb_cFloat } || + self == unsafe { rb_cSymbol } || self == unsafe { rb_cNilClass } || + self == unsafe { rb_cTrueClass } || self == unsafe { rb_cFalseClass } { + + return false + } + true + } + + /// A metaclass is the singleton class of an object that is a `Module`. + /// This is internal terminology from `class.c`. + pub fn is_metaclass(self) -> bool { + unsafe { + if RB_TYPE_P(self, RUBY_T_CLASS) && rb_zjit_singleton_class_p(self) { + let attached = rb_class_attached_object(self); + RB_TYPE_P(attached, RUBY_T_CLASS) || RB_TYPE_P(attached, RUBY_T_MODULE) + } else { + false + } + } + } + + pub fn is_singleton_class(self) -> bool { + // TODO(alan): clean up one of double check on T_CLASS + unsafe { RB_TYPE_P(self, RUBY_T_CLASS) && rb_zjit_singleton_class_p(self) } + } + + /// Return true for a static (non-heap) Ruby symbol (RB_STATIC_SYM_P) + pub fn static_sym_p(self) -> bool { + let VALUE(cval) = self; + let flag = RUBY_SYMBOL_FLAG as usize; + (cval & 0xff) == flag + } + + /// Return true for a dynamic Ruby symbol (RB_DYNAMIC_SYM_P) + fn dynamic_sym_p(self) -> bool { + if self.special_const_p() { + false + } else { + self.builtin_type() == RUBY_T_SYMBOL + } + } + + /// Returns true if the value is T_HASH + pub fn hash_p(self) -> bool { + !self.special_const_p() && self.builtin_type() == RUBY_T_HASH + } + + /// Returns true or false depending on whether the value is nil + pub fn nil_p(self) -> bool { + self == Qnil + } + + pub fn string_p(self) -> bool { + self.class_of() == unsafe { rb_cString } + } + + /// Read the flags bits from the RBasic object, then return a Ruby type enum (e.g. RUBY_T_ARRAY) + pub fn builtin_type(self) -> ruby_value_type { + (self.builtin_flags() & (RUBY_T_MASK as usize)) as ruby_value_type + } + + pub fn builtin_flags(self) -> usize { + assert!(!self.special_const_p()); + + let VALUE(cval) = self; + let rbasic_ptr = cval as *const RBasic; + let flags_bits: usize = unsafe { (*rbasic_ptr).flags }.as_usize(); + flags_bits + } + + pub fn class_of(self) -> VALUE { + if !self.special_const_p() { + let builtin_type = self.builtin_type(); + assert!( + builtin_type != RUBY_T_NONE && builtin_type != RUBY_T_MOVED, + "ZJIT saw a dead object. T_type={builtin_type}, {}", + self.obj_info() + ); + } + + unsafe { rb_yarv_class_of(self) } + } + + /// Check if `self` is a subclass of `other`. Assumes both `self` and `other` are class + /// objects. Returns [`ClassRelationship::Subclass`] if `self <= other`, + /// [`ClassRelationship::Superclass`] if `other < self`, and [`ClassRelationship::NoRelation`] + /// otherwise. + pub fn is_subclass_of(self, other: VALUE) -> ClassRelationship { + assert!(unsafe { RB_TYPE_P(self, RUBY_T_CLASS) }); + assert!(unsafe { RB_TYPE_P(other, RUBY_T_CLASS) }); + match unsafe { rb_class_inherited_p(self, other) } { + Qtrue => ClassRelationship::Subclass, + Qfalse => ClassRelationship::Superclass, + Qnil => ClassRelationship::NoRelation, + // The API specifies that it will return Qnil in this case + _ => panic!("Unexpected return value from rb_class_inherited_p"), + } + } + + /// Borrow the string contents of `self`. Rust unsafe because of possible mutation and GC + /// interactions. + pub unsafe fn as_rstring_byte_slice<'a>(self) -> Option<&'a [u8]> { + if !unsafe { RB_TYPE_P(self, RUBY_T_STRING) } { + None + } else { + let str_ptr = unsafe { rb_RSTRING_PTR(self) } as *const u8; + let str_len: usize = unsafe { rb_RSTRING_LEN(self) }.try_into().ok()?; + Some(unsafe { std::slice::from_raw_parts(str_ptr, str_len) }) + } + } + + pub fn is_frozen(self) -> bool { + unsafe { rb_obj_frozen_p(self) != VALUE(0) } + } + + pub fn shape_id_of(self) -> ShapeId { + if self.special_const_p() { + INVALID_SHAPE_ID + } else { + ShapeId(unsafe { rb_obj_shape_id(self) }) + } + } + + pub fn embedded_p(self) -> bool { + unsafe { + FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0) + } + } + + pub fn struct_embedded_p(self) -> bool { + unsafe { + RB_TYPE_P(self, RUBY_T_STRUCT) && + FL_TEST_RAW(self, VALUE(RSTRUCT_EMBED_LEN_MASK)) != VALUE(0) + } + } + + pub fn class_fields_embedded_p(self) -> bool { + unsafe { rb_jit_class_fields_embedded_p(self) } + } + + pub fn data_p(self) -> bool { + !self.special_const_p() && + self.builtin_type() == RUBY_T_DATA + } + + pub fn data_fields_embedded_p(self) -> bool { + unsafe { rb_jit_data_fields_embedded_p(self) } + } + + pub fn as_fixnum(self) -> i64 { + assert!(self.fixnum_p()); + (self.0 as i64) >> 1 + } + + pub fn as_isize(self) -> isize { + let VALUE(is) = self; + is as isize + } + + pub fn as_i32(self) -> i32 { + self.as_i64().try_into().unwrap() + } + + pub fn as_u32(self) -> u32 { + let VALUE(i) = self; + i.try_into().unwrap() + } + + pub fn as_i64(self) -> i64 { + let VALUE(i) = self; + i as i64 + } + + pub fn as_u64(self) -> u64 { + let VALUE(i) = self; + i.try_into().unwrap() + } + + pub fn as_usize(self) -> usize { + let VALUE(us) = self; + us + } + + pub fn as_ptr<T>(self) -> *const T { + let VALUE(us) = self; + us as *const T + } + + pub fn as_mut_ptr<T>(self) -> *mut T { + let VALUE(us) = self; + us as *mut T + } + + /// For working with opaque pointers and encoding null check. + /// Similar to [std::ptr::NonNull], but for `*const T`. `NonNull<T>` + /// is for `*mut T` while our C functions are setup to use `*const T`. + /// Casting from `NonNull<T>` to `*const T` is too noisy. + pub fn as_optional_ptr<T>(self) -> Option<*const T> { + let ptr: *const T = self.as_ptr(); + + if ptr.is_null() { + None + } else { + Some(ptr) + } + } + + /// Assert that `self` is an iseq in debug builds + pub fn as_iseq(self) -> IseqPtr { + let ptr: IseqPtr = self.as_ptr(); + + #[cfg(debug_assertions)] + if !ptr.is_null() { + unsafe { rb_assert_iseq_handle(self) } + } + + ptr + } + + pub fn cme_p(self) -> bool { + if self == VALUE(0) { return false; } + unsafe { rb_IMEMO_TYPE_P(self, imemo_ment) == 1 } + } + + /// Assert that `self` is a method entry in debug builds + pub fn as_cme(self) -> *const rb_callable_method_entry_t { + let ptr: *const rb_callable_method_entry_t = self.as_ptr(); + + #[cfg(debug_assertions)] + if !ptr.is_null() { + unsafe { rb_assert_cme_handle(self) } + } + + ptr + } + + pub const fn fixnum_from_usize(item: usize) -> Self { + assert!(item <= (RUBY_FIXNUM_MAX as usize)); // An unsigned will always be greater than RUBY_FIXNUM_MIN + let k: usize = item.wrapping_add(item.wrapping_add(1)); + VALUE(k) + } + + pub const fn fixnum_from_isize(item: isize) -> Self { + assert!(item >= RUBY_FIXNUM_MIN); + assert!(item <= RUBY_FIXNUM_MAX); + let k: isize = item.wrapping_add(item.wrapping_add(1)); + VALUE(k as usize) + } + + /// Call the write barrier after separately writing val to self. + pub fn write_barrier(self, val: VALUE) { + // rb_gc_writebarrier() asserts it is not called with a special constant + if !val.special_const_p() { + unsafe { rb_gc_writebarrier(self, val) }; + } + } +} + +pub type IseqParameters = rb_iseq_constant_body_rb_iseq_parameters; + +/// Extension trait to enable method calls on [`IseqPtr`] +pub trait IseqAccess { + unsafe fn params<'a>(self) -> &'a IseqParameters; +} + +impl IseqAccess for IseqPtr { + /// Get a description of the ISEQ's signature. Analogous to `ISEQ_BODY(iseq)->param` in C. + unsafe fn params<'a>(self) -> &'a IseqParameters { + use crate::cast::IntoUsize; + unsafe { &*((*self).body.byte_add(ISEQ_BODY_OFFSET_PARAM.to_usize()) as *const IseqParameters) } + } +} + +impl IseqParameters { + /// The `opt_table` is a mapping where `opt_table[number_of_optional_parameters_filled]` + /// gives the YARV entry point of ISeq as an index of the iseq_encoded array. + /// This method gives over the table that additionally works when `opt_num==0`, + /// when the table is stored as `NULL` and implicit. + /// The table stores the indexes as raw VALUE integers; they are not tagged as fixnum. + pub fn opt_table_slice(&self) -> &[VALUE] { + let opt_num: usize = self.opt_num.try_into().expect("ISeq opt_num should always >=0"); + if opt_num > 0 { + // The table has size=opt_num+1 because opt_table[opt_num] is valid (all optionals filled) + unsafe { std::slice::from_raw_parts(self.opt_table, opt_num + 1) } + } else { + // The ISeq entry point is index 0 when there are no optional parameters + &[VALUE(0)] + } + } +} + +impl Debug for VALUE { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Only use rb_obj_info() when {:#?} since it dereferences the pointer so carries some risk. + if f.alternate() { + write!(f, "VALUE({})", self.obj_info()) + } else { + write!(f, "VALUE(0x{:x})", self.0) + } + } +} + +impl From<IseqPtr> for VALUE { + /// For `.into()` convenience + fn from(iseq: IseqPtr) -> Self { + VALUE(iseq as usize) + } +} + +impl From<*const rb_callable_method_entry_t> for VALUE { + /// For `.into()` convenience + fn from(cme: *const rb_callable_method_entry_t) -> Self { + VALUE(cme as usize) + } +} + +impl From<&str> for VALUE { + fn from(value: &str) -> Self { + rust_str_to_ruby(value) + } +} + +impl From<String> for VALUE { + fn from(value: String) -> Self { + rust_str_to_ruby(&value) + } +} + +impl From<VALUE> for u64 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + uimm as u64 + } +} + +impl From<VALUE> for i64 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + assert!(uimm <= (i64::MAX as usize)); + uimm as i64 + } +} + +impl From<VALUE> for i32 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + assert!(uimm <= (i32::MAX as usize)); + uimm.try_into().unwrap() + } +} + +impl From<VALUE> for u16 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + uimm.try_into().unwrap() + } +} + +impl ID { + // Get a debug representation of the contents of the ID. Since `str` is UTF-8 + // and IDs have encodings that are not, this is a lossy representation. + pub fn contents_lossy(&self) -> std::borrow::Cow<'_, str> { + use std::borrow::Cow; + if self.0 == 0 { + Cow::Borrowed("ID(0)") + } else { + // Get the contents as a byte slice. IDs can have internal NUL bytes so rb_id2name, + // which returns a C string is more lossy than this approach. + let contents = unsafe { rb_id2str(*self) }; + if contents == Qfalse { + Cow::Borrowed("ID(0)") + } else { + let slice = unsafe { contents.as_rstring_byte_slice() } + .expect("rb_id2str() returned truthy non-string"); + String::from_utf8_lossy(slice) + } + } + } +} + +impl Display for ID { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.contents_lossy()) + } +} + +/// Produce a Ruby string from a Rust string slice +pub fn rust_str_to_ruby(str: &str) -> VALUE { + unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) } +} + +/// Produce a Ruby ID from a Rust string slice +pub fn rust_str_to_id(str: &str) -> ID { + let c_str = CString::new(str).unwrap(); + let c_ptr: *const c_char = c_str.as_ptr(); + unsafe { rb_intern(c_ptr) } +} + +/// Produce a Ruby symbol from a Rust string slice +pub fn rust_str_to_sym(str: &str) -> VALUE { + let id = rust_str_to_id(str); + unsafe { rb_id2sym(id) } +} + +/// Produce an owned Rust String from a C char pointer +pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { + assert!(!c_char_ptr.is_null()); + + let c_str: &CStr = unsafe { CStr::from_ptr(c_char_ptr) }; + + match c_str.to_str() { + Ok(rust_str) => Some(rust_str.to_string()), + Err(_) => None + } +} + +pub fn iseq_name(iseq: IseqPtr) -> String { + if iseq.is_null() { + return "<NULL>".to_string(); + } + let iseq_label = unsafe { rb_iseq_label(iseq) }; + if iseq_label == Qnil { + "None".to_string() + } else { + ruby_str_to_rust_string(iseq_label) + } +} + +// Equivalent of get_lvar_level() in compile.c +pub fn get_lvar_level(mut iseq: IseqPtr) -> u32 { + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let mut level = 0; + while iseq != local_iseq { + iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; + level += 1; + } + + level +} + +// Location is the file defining the method, colon, method name. +// Filenames are sometimes internal strings supplied to eval, +// so be careful with them. +pub fn iseq_get_location(iseq: IseqPtr, pos: u32) -> String { + let iseq_path = unsafe { rb_iseq_path(iseq) }; + let iseq_lineno = unsafe { rb_iseq_line_no(iseq, pos as usize) }; + + let mut s = iseq_name(iseq); + s.push('@'); + if iseq_path == Qnil { + s.push_str("None"); + } else { + s.push_str(&ruby_str_to_rust_string(iseq_path)); + } + s.push(':'); + s.push_str(&iseq_lineno.to_string()); + s +} + +pub fn ruby_str_to_rust_string_result(v: VALUE) -> Result<String, std::string::FromUtf8Error> { + let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; + let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); + let str_slice: &[u8] = unsafe { std::slice::from_raw_parts(str_ptr, str_len) }; + String::from_utf8(str_slice.to_vec()) +} + +// Convert a CRuby UTF-8-encoded RSTRING into a Rust string. +// This should work fine on ASCII strings and anything else +// that is considered legal UTF-8, including embedded nulls. +pub fn ruby_str_to_rust_string(v: VALUE) -> String { + ruby_str_to_rust_string_result(v).unwrap_or_default() +} + +pub fn ruby_sym_to_rust_string(v: VALUE) -> String { + let ruby_str = unsafe { rb_sym2str(v) }; + ruby_str_to_rust_string(ruby_str) +} + +pub fn ruby_call_method_id(cd: *const rb_call_data) -> ID { + let call_info = unsafe { rb_get_call_data_ci(cd) }; + unsafe { rb_vm_ci_mid(call_info) } +} + +pub fn ruby_call_method_name(cd: *const rb_call_data) -> String { + let mid = ruby_call_method_id(cd); + mid.contents_lossy().to_string() +} + +/// A location in Rust code for integrating with debugging facilities defined in C. +/// Use the [src_loc!] macro to crate an instance. +pub struct SourceLocation { + pub file: &'static CStr, + pub line: c_int, +} + +impl Debug for SourceLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}:{}", self.file.to_string_lossy(), self.line)) + } +} + +/// Make a [SourceLocation] at the current spot. +macro_rules! src_loc { + () => { + { + // Nul-terminated string with static lifetime, make a CStr out of it safely. + let file: &'static str = concat!(file!(), '\0'); + $crate::cruby::SourceLocation { + file: unsafe { std::ffi::CStr::from_ptr(file.as_ptr().cast()) }, + line: line!().try_into().unwrap(), + } + } + }; +} + +pub(crate) use src_loc; + +/// Run GC write barrier. Required after making a new edge in the object reference +/// graph from `old` to `young`. +macro_rules! obj_written { + ($old: expr, $young: expr) => { + let (old, young): (VALUE, VALUE) = ($old, $young); + let src_loc = $crate::cruby::src_loc!(); + unsafe { rb_yjit_obj_written(old, young, src_loc.file.as_ptr(), src_loc.line) }; + }; +} +pub(crate) use obj_written; + +/// Acquire the VM lock, make sure all other Ruby threads are asleep then run +/// some code while holding the lock. Returns whatever `func` returns. +/// Use with [src_loc!]. +/// +/// Required for code patching in the presence of ractors. +pub fn with_vm_lock<F, R>(loc: SourceLocation, func: F) -> R +where + F: FnOnce() -> R + UnwindSafe, +{ + let file = loc.file.as_ptr(); + let line = loc.line; + let mut recursive_lock_level: c_uint = 0; + + unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + // Ensure GC is off while we have the VM lock because: + // 1. We create many transient Rust collections that hold VALUEs during compilation. + // It's extremely tricky to properly marked and reference update these, not to + // mention the overhead and ergonomics issues. + // 2. If we yield to the GC while compiling, it re-enters our mark and update functions. + // This breaks `&mut` exclusivity since mark functions derive fresh `&mut` from statics + // while there is a stack frame below it that has an overlapping `&mut`. That's UB. + let gc_disabled_pre_call = unsafe { rb_gc_disable() }.test(); + + let ret = match catch_unwind(func) { + Ok(result) => result, + Err(_) => { + // Theoretically we can recover from some of these panics, + // but it's too late if the unwind reaches here. + + let _ = catch_unwind(|| { + // IO functions can panic too. + eprintln!( + "ZJIT panicked while holding VM lock acquired at {}:{}. Aborting...", + loc.file.to_string_lossy(), + line, + ); + }); + std::process::abort(); + } + }; + + unsafe { + if !gc_disabled_pre_call { + rb_gc_enable(); + } + rb_jit_vm_unlock(&mut recursive_lock_level, file, line); + }; + + ret +} + +/// At the moment, we abort in all cases we panic. +/// To aid with getting diagnostics in the wild without requiring people to set +/// RUST_BACKTRACE=1, register a panic hook that crash using rb_bug() for release builds. +/// rb_bug() might not be as good at printing a call trace as Rust's stdlib, but +/// it dumps some other info that might be relevant. +/// +/// In case we want to start doing fancier exception handling with panic=unwind, +/// we can revisit this later. For now, this helps to get us good bug reports. +pub fn rb_bug_panic_hook() { + use std::env; + use std::panic; + use std::io::{stderr, Write}; + + // Probably the default hook. We do this very early during process boot. + let previous_hook = panic::take_hook(); + + panic::set_hook(Box::new(move |panic_info| { + // Not using `eprintln` to avoid double panic. + let _ = stderr().write_all(b"ruby: ZJIT has panicked. More info to follow...\n"); + + // Always show a Rust backtrace for release builds. + // You should set RUST_BACKTRACE=1 for dev builds. + let release_build = cfg!(not(debug_assertions)); + if release_build { + unsafe { env::set_var("RUST_BACKTRACE", "1"); } + } + previous_hook(panic_info); + + // Dump information about the interpreter for release builds. + // You may also use ZJIT_RB_BUG=1 to trigger this on dev builds. + if release_build || env::var("ZJIT_RB_BUG").is_ok() { + // Abort with rb_bug(). It has a length limit on the message. + let panic_message = &format!("{panic_info}")[..]; + let len = std::cmp::min(0x100, panic_message.len()) as c_int; + unsafe { rb_bug(b"ZJIT: %*s\0".as_ref().as_ptr() as *const c_char, len, panic_message.as_ptr()); } + } else { + eprintln!("note: run with `ZJIT_RB_BUG=1` environment variable to display a Ruby backtrace"); + } + })); +} + +// Non-idiomatic capitalization for consistency with CRuby code +#[allow(non_upper_case_globals)] +pub const Qfalse: VALUE = VALUE(RUBY_Qfalse as usize); +#[allow(non_upper_case_globals)] +pub const Qnil: VALUE = VALUE(RUBY_Qnil as usize); +#[allow(non_upper_case_globals)] +pub const Qtrue: VALUE = VALUE(RUBY_Qtrue as usize); +#[allow(non_upper_case_globals)] +pub const Qundef: VALUE = VALUE(RUBY_Qundef as usize); + +#[allow(unused)] +mod manual_defs { + use super::*; + + pub const SIZEOF_VALUE: usize = 8; + pub const BITS_PER_BYTE: usize = 8; + pub const SIZEOF_VALUE_I32: i32 = SIZEOF_VALUE as i32; + pub const VALUE_BITS: u8 = BITS_PER_BYTE as u8 * SIZEOF_VALUE as u8; + + pub const RUBY_LONG_MIN: isize = std::os::raw::c_long::MIN as isize; + pub const RUBY_LONG_MAX: isize = std::os::raw::c_long::MAX as isize; + + pub const RUBY_FIXNUM_MIN: isize = RUBY_LONG_MIN / 2; + pub const RUBY_FIXNUM_MAX: isize = RUBY_LONG_MAX / 2; + + // From vm_callinfo.h - uses calculation that seems to confuse bindgen + pub const VM_CALL_ARGS_SIMPLE: u32 = 1 << VM_CALL_ARGS_SIMPLE_bit; + pub const VM_CALL_ARGS_SPLAT: u32 = 1 << VM_CALL_ARGS_SPLAT_bit; + pub const VM_CALL_ARGS_SPLAT_MUT: u32 = 1 << VM_CALL_ARGS_SPLAT_MUT_bit; + pub const VM_CALL_ARGS_BLOCKARG: u32 = 1 << VM_CALL_ARGS_BLOCKARG_bit; + pub const VM_CALL_FORWARDING: u32 = 1 << VM_CALL_FORWARDING_bit; + pub const VM_CALL_FCALL: u32 = 1 << VM_CALL_FCALL_bit; + pub const VM_CALL_KWARG: u32 = 1 << VM_CALL_KWARG_bit; + pub const VM_CALL_KW_SPLAT: u32 = 1 << VM_CALL_KW_SPLAT_bit; + pub const VM_CALL_KW_SPLAT_MUT: u32 = 1 << VM_CALL_KW_SPLAT_MUT_bit; + pub const VM_CALL_TAILCALL: u32 = 1 << VM_CALL_TAILCALL_bit; + pub const VM_CALL_SUPER : u32 = 1 << VM_CALL_SUPER_bit; + pub const VM_CALL_ZSUPER : u32 = 1 << VM_CALL_ZSUPER_bit; + pub const VM_CALL_OPT_SEND : u32 = 1 << VM_CALL_OPT_SEND_bit; + + // From internal/struct.h - in anonymous enum, so we can't easily import it + pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; + + // From iseq.h - via a different constant, which seems to confuse bindgen + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + + // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to + // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants. + pub const RUBY_OFFSET_RBASIC_FLAGS: i32 = 0; // struct RBasic, field "flags" + pub const RUBY_OFFSET_RBASIC_KLASS: i32 = 8; // struct RBasic, field "klass" + pub const RUBY_OFFSET_RARRAY_AS_HEAP_LEN: i32 = 16; // struct RArray, subfield "as.heap.len" + pub const RUBY_OFFSET_RARRAY_AS_HEAP_PTR: i32 = 32; // struct RArray, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RARRAY_AS_ARY: i32 = 16; // struct RArray, subfield "as.ary" + + pub const RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR: i32 = 24; // struct RStruct, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RSTRUCT_AS_ARY: i32 = 16; // struct RStruct, subfield "as.ary" + + pub const RUBY_OFFSET_RSTRING_AS_HEAP_PTR: i32 = 24; // struct RString, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RSTRING_AS_ARY: i32 = 24; // struct RString, subfield "as.embed.ary" + + // Constants from rb_control_frame_t vm_core.h + pub const RUBY_OFFSET_CFP_PC: i32 = 0; + pub const RUBY_OFFSET_CFP_SP: i32 = 8; + pub const RUBY_OFFSET_CFP_ISEQ: i32 = 16; + pub const RUBY_OFFSET_CFP_SELF: i32 = 24; + pub const RUBY_OFFSET_CFP_EP: i32 = 32; + pub const RUBY_OFFSET_CFP_BLOCK_CODE: i32 = 40; + pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; + pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; + + // Constants from rb_thread_t in vm_core.h + pub const RUBY_OFFSET_THREAD_SELF: i32 = 16; + + // Constants from iseq_inline_constant_cache (IC) and iseq_inline_constant_cache_entry (ICE) in vm_core.h + pub const RUBY_OFFSET_IC_ENTRY: i32 = 0; + pub const RUBY_OFFSET_ICE_VALUE: i32 = 8; +} +pub use manual_defs::*; + +#[cfg(test)] +pub mod test_utils { + use std::{ptr::null, sync::Once}; + + use crate::{options::{rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold, DEFAULT_CALL_THRESHOLD}, state::{rb_zjit_entry, ZJITState}}; + + use super::*; + + static RUBY_VM_INIT: Once = Once::new(); + + /// Boot and initialize the Ruby VM for Rust testing + fn boot_rubyvm() { + // Boot the VM + unsafe { + // TODO(alan): this init_stack call is incorrect. It sets the stack bottom, but + // when we return from this function will be be deeper in the stack. + // The callback for with_rubyvm() should run on a frame higher than this frame + // so the GC scans all the VALUEs on the stack. + // Consequently with_rubyvm() can only be used once per process, i.e. you can't + // boot and then run a few callbacks, because that risks putting VALUE outside + // the marked stack memory range. + // + // Need to also address the ergnomic issues addressed by + // <https://github.com/Shopify/zjit/pull/37>, though + let mut var: VALUE = Qnil; + ruby_init_stack(&mut var as *mut VALUE as *mut _); + rb_zjit_prepare_options(); // enable `#with_jit` on builtins + ruby_init(); + + // The default rb_zjit_profile_threshold is too high, so lower it for HIR tests. + if rb_zjit_call_threshold == DEFAULT_CALL_THRESHOLD { + set_call_threshold(2); + } + + // Pass command line options so the VM loads core library methods defined in + // ruby such as from `kernel.rb`. + // We drive ZJIT manually in tests, so disable heuristic compilation triggers. + // (Also, pass this in case we offer a -DFORCE_ENABLE_ZJIT option which turns + // ZJIT on by default.) + let cmdline = [c"--disable-all".as_ptr().cast_mut(), c"-e0".as_ptr().cast_mut()]; + let options_ret = ruby_options(2, cmdline.as_ptr().cast_mut()); + assert_ne!(0, ruby_executable_node(options_ret, std::ptr::null_mut()), "command-line parsing failed"); + + crate::cruby::ids::init(); // for ID! usages in tests + } + + // Call ZJIT hooks to install Ruby implementations of builtins like Array#each + unsafe { + let zjit = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + rb_funcallv(zjit, rust_str_to_id("call_jit_hooks"), 0, std::ptr::null()); + } + + // Set up globals for convenience + let zjit_entry = ZJITState::init(); + + // Enable zjit_* instructions + unsafe { rb_zjit_entry = zjit_entry; } + } + + /// Make sure the Ruby VM is set up and run a given callback with rb_protect() + pub fn with_rubyvm<T>(mut func: impl FnMut() -> T) -> T { + RUBY_VM_INIT.call_once(boot_rubyvm); + + // Invoke callback through rb_protect() so exceptions don't crash the process. + // "Fun" double pointer dance to get a thin function pointer to pass through C + unsafe extern "C" fn callback_wrapper(data: VALUE) -> VALUE { + // SAFETY: shorter lifetime than the data local in the caller frame + let callback: *const *mut dyn FnMut() = std::ptr::with_exposed_provenance_mut(data.0); + unsafe { (**callback)() }; + Qnil + } + + // Set up a callback wrapper to store the return value + let mut result: Option<T> = None; + let mut func_wrapper = || { + result.replace(func()); + }; + let data: *mut dyn FnMut() = &raw mut func_wrapper; + let data: *const *mut dyn FnMut() = &raw const data; + let mut state: c_int = 0; + unsafe { super::rb_protect(Some(callback_wrapper), VALUE(data.expose_provenance()), &mut state) }; + if state != 0 { + unsafe { rb_zjit_print_exception(); } + assert_eq!(0, state, "Exceptional unwind in callback. Ruby exception?"); + } + + result.expect("Callback did not set result") + } + + /// Compile an ISeq via `RubyVM::InstructionSequence.compile`. + pub fn compile_to_iseq(program: &str) -> *const rb_iseq_t { + with_rubyvm(|| { + let wrapped_iseq = compile_to_wrapped_iseq(program); + unsafe { rb_iseqw_to_iseq(wrapped_iseq) } + }) + } + + pub fn define_class(name: &str, superclass: VALUE) -> VALUE { + let name = CString::new(name).unwrap(); + unsafe { rb_define_class(name.as_ptr(), superclass) } + } + + /// Evaluate a given Ruby program + pub fn eval(program: &str) -> VALUE { + with_rubyvm(|| { + let wrapped_iseq = compile_to_wrapped_iseq(&unindent(program, false)); + unsafe { rb_funcallv(wrapped_iseq, ID!(eval), 0, null()) } + }) + } + + /// Evaluate a given Ruby program with compile options + pub fn eval_with_options(program: &str, options_expr: &str) -> VALUE { + with_rubyvm(|| { + let options = eval(options_expr); + let wrapped_iseq = compile_to_wrapped_iseq_with_options(&unindent(program, false), options); + unsafe { rb_funcallv(wrapped_iseq, ID!(eval), 0, null()) } + }) + } + + /// Get the #inspect of a given Ruby program in Rust string + pub fn inspect(program: &str) -> String { + let inspect = format!("({program}).inspect"); + ruby_str_to_rust_string(eval(&inspect)) + } + + /// Like inspect, but also asserts that all compilations triggered by this program succeed. + pub fn assert_compiles_allowing_exits(program: &str) -> String { + use crate::state::ZJITState; + ZJITState::enable_assert_compiles(); + let result = inspect(program); + ZJITState::disable_assert_compiles(); + result + } + + /// Like inspect, but also asserts that all compilations triggered by this program succeed and + /// no side exits occurr during the program. + pub fn assert_compiles(program: &str) -> String { + use crate::state::ZJITState; + let exits_before = crate::stats::total_exit_count(); + ZJITState::enable_assert_compiles(); + let result = inspect(program); + ZJITState::disable_assert_compiles(); + assert_eq!(exits_before, crate::stats::total_exit_count(), "Program side-exited"); + result + } + + /// Get IseqPtr for a specified method + pub fn get_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified instance method + pub fn get_instance_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.instance_method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified Proc object + pub fn get_proc_iseq(obj: &str) -> *const rb_iseq_t { + let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of({obj})")); + unsafe { rb_iseqw_to_iseq(wrapped_iseq) } + } + + /// Remove the minimum indent from every line, skipping the first and last lines if `trim_lines`. + pub fn unindent(string: &str, trim_lines: bool) -> String { + // Break up a string into multiple lines + let mut lines: Vec<String> = string.split_inclusive("\n").map(|s| s.to_string()).collect(); + if trim_lines { // raw string literals come with extra lines + lines.remove(0); + lines.remove(lines.len() - 1); + } + + // Count the minimum number of spaces + let spaces = lines.iter().filter_map(|line| { + for (i, ch) in line.as_bytes().iter().enumerate() { + if *ch != b' ' { + return Some(i); + } + } + None + }).min().unwrap_or(0); + + // Join lines, removing spaces + let mut unindented: Vec<u8> = vec![]; + for line in lines.iter() { + if line.len() > spaces { + unindented.extend_from_slice(&line.as_bytes()[spaces..]); + } else { + unindented.extend_from_slice(line.as_bytes()); + } + } + String::from_utf8(unindented).unwrap() + } + + /// Compile a program into a RubyVM::InstructionSequence object + fn compile_to_wrapped_iseq(program: &str) -> VALUE { + compile_to_wrapped_iseq_with_options(program, Qnil) + } + + fn compile_to_wrapped_iseq_with_options(program: &str, options: VALUE) -> VALUE { + let bytes = program.as_bytes().as_ptr() as *const c_char; + unsafe { + let program_str = rb_utf8_str_new(bytes, program.len().try_into().unwrap()); + let args = [program_str, Qnil, Qnil, VALUE(1_usize.wrapping_shl(1) | 1), options]; + rb_funcallv(rb_cISeq, ID!(compile), args.len() as c_int, args.as_ptr()) + } + } + + #[test] + fn boot_vm() { + // Test that we loaded kernel.rb and have Kernel#class + eval("1.class"); + } + + #[test] + #[should_panic] + fn ruby_exception_causes_panic() { + eval("raise"); + } + + #[test] + fn value_from_fixnum_in_range() { + assert_eq!(VALUE::fixnum_from_usize(2), VALUE(5)); + assert_eq!(VALUE::fixnum_from_usize(0), VALUE(1)); + assert_eq!(VALUE::fixnum_from_isize(-1), VALUE(0xffffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(-2), VALUE(0xfffffffffffffffd)); + assert_eq!(VALUE::fixnum_from_usize(RUBY_FIXNUM_MAX as usize), VALUE(0x7fffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX), VALUE(0x7fffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN), VALUE(0x8000000000000001)); + } + + #[test] + fn value_as_fixnum() { + assert_eq!(VALUE::fixnum_from_usize(2).as_fixnum(), 2); + assert_eq!(VALUE::fixnum_from_usize(0).as_fixnum(), 0); + assert_eq!(VALUE::fixnum_from_isize(-1).as_fixnum(), -1); + assert_eq!(VALUE::fixnum_from_isize(-2).as_fixnum(), -2); + assert_eq!(VALUE::fixnum_from_usize(RUBY_FIXNUM_MAX as usize).as_fixnum(), RUBY_FIXNUM_MAX.try_into().unwrap()); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX).as_fixnum(), RUBY_FIXNUM_MAX.try_into().unwrap()); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN).as_fixnum(), RUBY_FIXNUM_MIN.try_into().unwrap()); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_big_usize() { + assert_eq!(VALUE::fixnum_from_usize((RUBY_FIXNUM_MAX+1) as usize), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_big_isize() { + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX+1), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_small_usize() { + assert_eq!(VALUE::fixnum_from_usize((RUBY_FIXNUM_MIN-1) as usize), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_small_isize() { + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN-1), VALUE(1)); + } + + #[test] + fn value_fmt_debug() { + assert_eq!("VALUE(0xcafe)", format!("{:?}", VALUE(0xcafe))); + let alternate = format!("{:#?}", eval("::Hash")); + assert!(alternate.contains("Hash"), "'Hash' not substring of '{alternate}'"); + } +} +#[cfg(test)] +pub use test_utils::*; + +/// Get class name from a class pointer. For anonymous classes, includes the +/// superclass name for context (e.g. `#<Class(String):0x00007f...>`). +pub fn get_class_name(class: VALUE) -> String { + // type checks for rb_class2name() + let name = if unsafe { RB_TYPE_P(class, RUBY_T_MODULE) || RB_TYPE_P(class, RUBY_T_CLASS) } { + Some(class) + } else { + None + }.and_then(|class| unsafe { + cstr_to_rust_string(rb_class2name(class)) + }).unwrap_or_else(|| "Unknown".to_string()); + + // For anonymous classes, include the superclass name for context. + // Use rb_class_real to resolve through iclasses (internal include/prepend + // wrappers) before checking rb_mod_name, which returns Qnil for anonymous classes. + // e.g. "#<Class:0x7f...>" with superclass String => "#<Class(String):0x7f...>" + if unsafe { RB_TYPE_P(class, RUBY_T_CLASS) && rb_mod_name(rb_class_real(class)) == Qnil } { + let super_class = unsafe { rb_class_get_superclass(class) }; + if super_class != Qnil { + let super_name = get_class_name(super_class); + return format!("#<Class({super_name}):{:#x}>", class.0); + } + } + + name +} + +// Return the module name for a given module or class. For anonymous modules, returns None since +// rb_mod_name returns Qnil. +pub fn get_module_name(module: VALUE) -> Option<String> { + // type checks for rb_mod_name() + assert!(unsafe { RB_TYPE_P(module, RUBY_T_MODULE) || RB_TYPE_P(module, RUBY_T_CLASS) }, "Expected class or module"); + let name = unsafe { rb_mod_name(module) }; + if name == Qnil { + None + } else { + Some(ruby_str_to_rust_string(name)) + } +} + + +#[cfg(test)] +mod class_name_tests { + use super::*; + use test_utils::{eval, with_rubyvm}; + + #[test] + fn named_class() { + with_rubyvm(|| { + assert_eq!(get_class_name(eval("String")), "String"); + }); + } + + #[test] + fn named_module() { + with_rubyvm(|| { + assert_eq!(get_class_name(eval("Kernel")), "Kernel"); + }); + } + + #[test] + fn anonymous_class_includes_superclass() { + with_rubyvm(|| { + let name = get_class_name(eval("Class.new(String)")); + assert!(name.starts_with("#<Class(String):0x"), "got: {name}"); + }); + } + + #[test] + fn anonymous_class_nested_superclass() { + with_rubyvm(|| { + let name = get_class_name(eval("Class.new(Class.new(String))")); + assert!(name.starts_with("#<Class(#<Class(String):0x"), "got: {name}"); + }); + } + + #[test] + fn anonymous_module_unchanged() { + with_rubyvm(|| { + let name = get_class_name(eval("Module.new")); + assert!(name.starts_with("#<Module:0x"), "got: {name}"); + }); + } + +} + +pub fn class_has_leaf_allocator(class: VALUE) -> bool { + // We need to check if the class is initialized and not a singleton before + // trying to read the allocator, otherwise it will raise. + // Because of this they should be considered non-leaf anyways. + if !unsafe { rb_zjit_class_initialized_p(class) } { return false; } + if unsafe { rb_zjit_singleton_class_p(class) } { return false; } + + // empty_hash_alloc + if class == unsafe { rb_cHash } { return true; } + // empty_ary_alloc + if class == unsafe { rb_cArray } { return true; } + // empty_str_alloc + if class == unsafe { rb_cString } { return true; } + // rb_reg_s_alloc + if class == unsafe { rb_cRegexp } { return true; } + // rb_class_allocate_instance + unsafe { rb_zjit_class_has_default_allocator(class) } +} + +/// Interned ID values for Ruby symbols and method names. +/// See [type@crate::cruby::ID] and usages outside of ZJIT. +pub(crate) mod ids { + use std::sync::atomic::AtomicU64; + /// Globals to cache IDs on boot. Atomic to use with relaxed ordering + /// so reads can happen without `unsafe`. Synchronization done through + /// the VM lock. + macro_rules! def_ids { + ($(name: $name:ident $(content: $content:literal)?)*) => { + $( + pub static $name: AtomicU64 = AtomicU64::new(0); + )* + + pub(crate) fn init() { + $( + let content = stringify!($name); + _ = content; + $(let content = &$content;)? + let ptr: *const u8 = content.as_ptr(); + + // Lookup and cache each ID + $name.store( + unsafe { $crate::cruby::rb_intern2(ptr.cast(), content.len() as _) }.0, + std::sync::atomic::Ordering::Relaxed + ); + )* + + } + } + } + + def_ids! { + name: NULL content: b"" + name: respond_to_missing content: b"respond_to_missing?" + name: eq content: b"==" + name: string_eq content: b"String#==" + name: include_p content: b"include?" + name: to_ary + name: to_s + name: compile + name: eval + name: plus content: b"+" + name: minus content: b"-" + name: mult content: b"*" + name: div content: b"/" + name: modulo content: b"%" + name: neq content: b"!=" + name: lt content: b"<" + name: le content: b"<=" + name: gt content: b">" + name: ge content: b">=" + name: and content: b"&" + name: or content: b"|" + name: xor content: b"^" + name: freeze + name: minusat content: b"-@" + name: aref content: b"[]" + name: rb_ivar_get_at_no_ractor_check + name: RUBY_FL_FREEZE + name: RUBY_ELTS_SHARED + name: RubyVM + name: ZJIT + name: induce_side_exit_bang content: b"induce_side_exit!" + name: induce_compile_failure_bang content: b"induce_compile_failure!" + name: induce_breakpoint_bang content: b"induce_breakpoint!" + } + + /// Get an CRuby `ID` to an interned string, e.g. a particular method name. + macro_rules! ID { + ($id_name:ident) => {{ + let id = $crate::cruby::ids::$id_name.load(std::sync::atomic::Ordering::Relaxed); + debug_assert_ne!(0, id, "ids module should be initialized"); + $crate::cruby::ID(id) + }} + } + pub(crate) use ID; +} +pub(crate) use ids::ID; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs new file mode 100644 index 0000000000..08c502b0d8 --- /dev/null +++ b/zjit/src/cruby_bindings.inc.rs @@ -0,0 +1,2327 @@ +/* automatically generated by rust-bindgen 0.71.1 */ + +#[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit<Storage> { + storage: Storage, +} +impl<Storage> __BindgenBitfieldUnit<Storage> { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl<Storage> __BindgenBitfieldUnit<Storage> +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + fn extract_bit(byte: u8, index: usize) -> bool { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + Self::extract_bit(byte, index) + } + #[inline] + pub unsafe fn raw_get_bit(this: *const Self, index: usize) -> bool { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = *(core::ptr::addr_of!((*this).storage) as *const u8).offset(byte_index as isize); + Self::extract_bit(byte, index) + } + #[inline] + fn change_bit(byte: u8, index: usize, val: bool) -> u8 { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + byte | mask + } else { + byte & !mask + } + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub unsafe fn raw_set_bit(this: *mut Self, index: usize, val: bool) { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = + (core::ptr::addr_of_mut!((*this).storage) as *mut u8).offset(byte_index as isize); + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub unsafe fn raw_get(this: *const Self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if Self::raw_get_bit(this, i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } + #[inline] + pub unsafe fn raw_set(this: *mut Self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + Self::raw_set_bit(this, index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]); +impl<T> __IncompleteArrayField<T> { + #[inline] + pub const fn new() -> Self { + __IncompleteArrayField(::std::marker::PhantomData, []) + } + #[inline] + pub fn as_ptr(&self) -> *const T { + self as *const _ as *const T + } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::std::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} +impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__IncompleteArrayField") + } +} +#[repr(C)] +pub struct __BindgenUnionField<T>(::std::marker::PhantomData<T>); +impl<T> __BindgenUnionField<T> { + #[inline] + pub const fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_ref(&self) -> &T { + ::std::mem::transmute(self) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl<T> ::std::default::Default for __BindgenUnionField<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl<T> ::std::clone::Clone for __BindgenUnionField<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} +impl<T> ::std::marker::Copy for __BindgenUnionField<T> {} +impl<T> ::std::fmt::Debug for __BindgenUnionField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} +impl<T> ::std::hash::Hash for __BindgenUnionField<T> { + fn hash<H: ::std::hash::Hasher>(&self, _state: &mut H) {} +} +impl<T> ::std::cmp::PartialEq for __BindgenUnionField<T> { + fn eq(&self, _other: &__BindgenUnionField<T>) -> bool { + true + } +} +impl<T> ::std::cmp::Eq for __BindgenUnionField<T> {} +pub const ONIG_OPTION_IGNORECASE: u32 = 1; +pub const ONIG_OPTION_EXTEND: u32 = 2; +pub const ONIG_OPTION_MULTILINE: u32 = 4; +pub const ARG_ENCODING_FIXED: u32 = 16; +pub const ARG_ENCODING_NONE: u32 = 32; +pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1; +pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2; +pub const STRING_REDEFINED_OP_FLAG: u32 = 4; +pub const ARRAY_REDEFINED_OP_FLAG: u32 = 8; +pub const HASH_REDEFINED_OP_FLAG: u32 = 16; +pub const SYMBOL_REDEFINED_OP_FLAG: u32 = 64; +pub const TIME_REDEFINED_OP_FLAG: u32 = 128; +pub const REGEXP_REDEFINED_OP_FLAG: u32 = 256; +pub const NIL_REDEFINED_OP_FLAG: u32 = 512; +pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024; +pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048; +pub const PROC_REDEFINED_OP_FLAG: u32 = 4096; +pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31; +pub const VM_ENV_DATA_SIZE: u32 = 3; +pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2; +pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1; +pub const VM_ENV_DATA_INDEX_FLAGS: u32 = 0; +pub const VM_BLOCK_HANDLER_NONE: u32 = 0; +pub const SHAPE_ID_NUM_BITS: u32 = 32; +pub const ZJIT_JIT_RETURN_C_FRAME: u32 = 1; +pub const ZJIT_JIT_RETURN_POISON: i64 = -4981057192772781345; +pub type rb_alloc_func_t = ::std::option::Option<unsafe extern "C" fn(klass: VALUE) -> VALUE>; +pub const RUBY_Qfalse: ruby_special_consts = 0; +pub const RUBY_Qnil: ruby_special_consts = 4; +pub const RUBY_Qtrue: ruby_special_consts = 20; +pub const RUBY_Qundef: ruby_special_consts = 36; +pub const RUBY_IMMEDIATE_MASK: ruby_special_consts = 7; +pub const RUBY_FIXNUM_FLAG: ruby_special_consts = 1; +pub const RUBY_FLONUM_MASK: ruby_special_consts = 3; +pub const RUBY_FLONUM_FLAG: ruby_special_consts = 2; +pub const RUBY_SYMBOL_FLAG: ruby_special_consts = 12; +pub const RUBY_SPECIAL_SHIFT: ruby_special_consts = 8; +pub type ruby_special_consts = u32; +#[repr(C)] +pub struct RBasic { + pub flags: VALUE, + pub klass: VALUE, +} +pub const RUBY_T_NONE: ruby_value_type = 0; +pub const RUBY_T_OBJECT: ruby_value_type = 1; +pub const RUBY_T_CLASS: ruby_value_type = 2; +pub const RUBY_T_MODULE: ruby_value_type = 3; +pub const RUBY_T_FLOAT: ruby_value_type = 4; +pub const RUBY_T_STRING: ruby_value_type = 5; +pub const RUBY_T_REGEXP: ruby_value_type = 6; +pub const RUBY_T_ARRAY: ruby_value_type = 7; +pub const RUBY_T_HASH: ruby_value_type = 8; +pub const RUBY_T_STRUCT: ruby_value_type = 9; +pub const RUBY_T_BIGNUM: ruby_value_type = 10; +pub const RUBY_T_FILE: ruby_value_type = 11; +pub const RUBY_T_DATA: ruby_value_type = 12; +pub const RUBY_T_MATCH: ruby_value_type = 13; +pub const RUBY_T_COMPLEX: ruby_value_type = 14; +pub const RUBY_T_RATIONAL: ruby_value_type = 15; +pub const RUBY_T_NIL: ruby_value_type = 17; +pub const RUBY_T_TRUE: ruby_value_type = 18; +pub const RUBY_T_FALSE: ruby_value_type = 19; +pub const RUBY_T_SYMBOL: ruby_value_type = 20; +pub const RUBY_T_FIXNUM: ruby_value_type = 21; +pub const RUBY_T_UNDEF: ruby_value_type = 22; +pub const RUBY_T_IMEMO: ruby_value_type = 26; +pub const RUBY_T_NODE: ruby_value_type = 27; +pub const RUBY_T_ICLASS: ruby_value_type = 28; +pub const RUBY_T_ZOMBIE: ruby_value_type = 29; +pub const RUBY_T_MOVED: ruby_value_type = 30; +pub const RUBY_T_MASK: ruby_value_type = 31; +pub type ruby_value_type = u32; +pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12; +pub type ruby_fl_ushift = u32; +pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; +pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; +pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; +pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; +pub const RUBY_FL_EXIVAR: ruby_fl_type = 0; +pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; +pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512; +pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024; +pub const RUBY_FL_FREEZE: ruby_fl_type = 2048; +pub const RUBY_FL_USER0: ruby_fl_type = 4096; +pub const RUBY_FL_USER1: ruby_fl_type = 8192; +pub const RUBY_FL_USER2: ruby_fl_type = 16384; +pub const RUBY_FL_USER3: ruby_fl_type = 32768; +pub const RUBY_FL_USER4: ruby_fl_type = 65536; +pub const RUBY_FL_USER5: ruby_fl_type = 131072; +pub const RUBY_FL_USER6: ruby_fl_type = 262144; +pub const RUBY_FL_USER7: ruby_fl_type = 524288; +pub const RUBY_FL_USER8: ruby_fl_type = 1048576; +pub const RUBY_FL_USER9: ruby_fl_type = 2097152; +pub const RUBY_FL_USER10: ruby_fl_type = 4194304; +pub const RUBY_FL_USER11: ruby_fl_type = 8388608; +pub const RUBY_FL_USER12: ruby_fl_type = 16777216; +pub const RUBY_FL_USER13: ruby_fl_type = 33554432; +pub const RUBY_FL_USER14: ruby_fl_type = 67108864; +pub const RUBY_FL_USER15: ruby_fl_type = 134217728; +pub const RUBY_FL_USER16: ruby_fl_type = 268435456; +pub const RUBY_FL_USER17: ruby_fl_type = 536870912; +pub const RUBY_FL_USER18: ruby_fl_type = 1073741824; +pub const RUBY_FL_USER19: ruby_fl_type = -2147483648; +pub const RUBY_ELTS_SHARED: ruby_fl_type = 4096; +pub const RUBY_FL_SINGLETON: ruby_fl_type = 8192; +pub type ruby_fl_type = i32; +pub const RSTRING_NOEMBED: ruby_rstring_flags = 8192; +pub const RSTRING_FSTR: ruby_rstring_flags = 536870912; +pub type ruby_rstring_flags = u32; +pub type st_data_t = ::std::os::raw::c_ulong; +pub type st_index_t = st_data_t; +pub const ST_CONTINUE: st_retval = 0; +pub const ST_STOP: st_retval = 1; +pub const ST_DELETE: st_retval = 2; +pub const ST_CHECK: st_retval = 3; +pub const ST_REPLACE: st_retval = 4; +pub type st_retval = u32; +pub type st_foreach_callback_func = ::std::option::Option< + unsafe extern "C" fn( + arg1: st_data_t, + arg2: st_data_t, + arg3: st_data_t, + ) -> ::std::os::raw::c_int, +>; +pub const RARRAY_EMBED_FLAG: ruby_rarray_flags = 8192; +pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536; +pub type ruby_rarray_flags = u32; +pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; +pub type ruby_rarray_consts = u32; +pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192; +pub type ruby_rmodule_flags = u32; +pub const ROBJECT_HEAP: ruby_robject_flags = 65536; +pub type ruby_robject_flags = u32; +pub const RUBY_TYPED_FREE_IMMEDIATELY: rbimpl_typeddata_flags = 1; +pub const RUBY_TYPED_EMBEDDABLE: rbimpl_typeddata_flags = 2; +pub const RUBY_TYPED_FROZEN_SHAREABLE: rbimpl_typeddata_flags = 256; +pub const RUBY_TYPED_WB_PROTECTED: rbimpl_typeddata_flags = 32; +pub const RUBY_TYPED_DECL_MARKING: rbimpl_typeddata_flags = 16384; +pub type rbimpl_typeddata_flags = u32; +pub type rb_event_flag_t = u32; +pub type rb_block_call_func = ::std::option::Option< + unsafe extern "C" fn( + yielded_arg: VALUE, + callback_arg: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + blockarg: VALUE, + ) -> VALUE, +>; +pub type rb_block_call_func_t = rb_block_call_func; +pub const RUBY_ENCODING_INLINE_MAX: ruby_encoding_consts = 127; +pub const RUBY_ENCODING_SHIFT: ruby_encoding_consts = 22; +pub const RUBY_ENCODING_MASK: ruby_encoding_consts = 532676608; +pub const RUBY_ENCODING_MAXNAMELEN: ruby_encoding_consts = 42; +pub type ruby_encoding_consts = u32; +pub const RUBY_ENCINDEX_ASCII_8BIT: ruby_preserved_encindex = 0; +pub const RUBY_ENCINDEX_UTF_8: ruby_preserved_encindex = 1; +pub const RUBY_ENCINDEX_US_ASCII: ruby_preserved_encindex = 2; +pub const RUBY_ENCINDEX_UTF_16BE: ruby_preserved_encindex = 3; +pub const RUBY_ENCINDEX_UTF_16LE: ruby_preserved_encindex = 4; +pub const RUBY_ENCINDEX_UTF_32BE: ruby_preserved_encindex = 5; +pub const RUBY_ENCINDEX_UTF_32LE: ruby_preserved_encindex = 6; +pub const RUBY_ENCINDEX_UTF_16: ruby_preserved_encindex = 7; +pub const RUBY_ENCINDEX_UTF_32: ruby_preserved_encindex = 8; +pub const RUBY_ENCINDEX_UTF8_MAC: ruby_preserved_encindex = 9; +pub const RUBY_ENCINDEX_EUC_JP: ruby_preserved_encindex = 10; +pub const RUBY_ENCINDEX_Windows_31J: ruby_preserved_encindex = 11; +pub const RUBY_ENCINDEX_BUILTIN_MAX: ruby_preserved_encindex = 12; +pub type ruby_preserved_encindex = u32; +pub const BOP_PLUS: ruby_basic_operators = 0; +pub const BOP_MINUS: ruby_basic_operators = 1; +pub const BOP_MULT: ruby_basic_operators = 2; +pub const BOP_DIV: ruby_basic_operators = 3; +pub const BOP_MOD: ruby_basic_operators = 4; +pub const BOP_EQ: ruby_basic_operators = 5; +pub const BOP_EQQ: ruby_basic_operators = 6; +pub const BOP_LT: ruby_basic_operators = 7; +pub const BOP_LE: ruby_basic_operators = 8; +pub const BOP_LTLT: ruby_basic_operators = 9; +pub const BOP_AREF: ruby_basic_operators = 10; +pub const BOP_ASET: ruby_basic_operators = 11; +pub const BOP_LENGTH: ruby_basic_operators = 12; +pub const BOP_SIZE: ruby_basic_operators = 13; +pub const BOP_EMPTY_P: ruby_basic_operators = 14; +pub const BOP_NIL_P: ruby_basic_operators = 15; +pub const BOP_SUCC: ruby_basic_operators = 16; +pub const BOP_GT: ruby_basic_operators = 17; +pub const BOP_GE: ruby_basic_operators = 18; +pub const BOP_GTGT: ruby_basic_operators = 19; +pub const BOP_NOT: ruby_basic_operators = 20; +pub const BOP_NEQ: ruby_basic_operators = 21; +pub const BOP_MATCH: ruby_basic_operators = 22; +pub const BOP_FREEZE: ruby_basic_operators = 23; +pub const BOP_UMINUS: ruby_basic_operators = 24; +pub const BOP_MAX: ruby_basic_operators = 25; +pub const BOP_MIN: ruby_basic_operators = 26; +pub const BOP_HASH: ruby_basic_operators = 27; +pub const BOP_CALL: ruby_basic_operators = 28; +pub const BOP_AND: ruby_basic_operators = 29; +pub const BOP_OR: ruby_basic_operators = 30; +pub const BOP_CMP: ruby_basic_operators = 31; +pub const BOP_DEFAULT: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 33; +pub const BOP_INCLUDE_P: ruby_basic_operators = 34; +pub const BOP_LAST_: ruby_basic_operators = 35; +pub type ruby_basic_operators = u32; +pub type rb_serial_t = ::std::os::raw::c_ulonglong; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_id_item { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_id_table { + pub capa: ::std::os::raw::c_int, + pub num: ::std::os::raw::c_int, + pub used: ::std::os::raw::c_int, + pub items: *mut rb_id_item, +} +pub const imemo_env: imemo_type = 0; +pub const imemo_cref: imemo_type = 1; +pub const imemo_svar: imemo_type = 2; +pub const imemo_throw_data: imemo_type = 3; +pub const imemo_ifunc: imemo_type = 4; +pub const imemo_memo: imemo_type = 5; +pub const imemo_ment: imemo_type = 6; +pub const imemo_iseq: imemo_type = 7; +pub const imemo_tmpbuf: imemo_type = 8; +pub const imemo_cvar_entry: imemo_type = 9; +pub const imemo_callinfo: imemo_type = 10; +pub const imemo_callcache: imemo_type = 11; +pub const imemo_constcache: imemo_type = 12; +pub const imemo_fields: imemo_type = 13; +pub const imemo_subclasses: imemo_type = 14; +pub const imemo_cdhash: imemo_type = 15; +pub type imemo_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vm_ifunc_argc { + pub min: ::std::os::raw::c_int, + pub max: ::std::os::raw::c_int, +} +#[repr(C)] +pub struct vm_ifunc { + pub flags: VALUE, + pub svar_lep: *mut VALUE, + pub func: rb_block_call_func_t, + pub data: *const ::std::os::raw::c_void, + pub argc: vm_ifunc_argc, +} +pub type rb_atomic_t = ::std::os::raw::c_uint; +pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; +pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; +pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2; +pub const METHOD_VISI_PROTECTED: rb_method_visibility_t = 3; +pub const METHOD_VISI_MASK: rb_method_visibility_t = 3; +pub type rb_method_visibility_t = u32; +#[repr(C)] +pub struct rb_method_entry_struct { + pub flags: VALUE, + pub defined_class: VALUE, + pub def: *mut rb_method_definition_struct, + pub called_id: ID, + pub owner: VALUE, +} +pub type rb_method_entry_t = rb_method_entry_struct; +#[repr(C)] +pub struct rb_callable_method_entry_struct { + pub flags: VALUE, + pub defined_class: VALUE, + pub def: *mut rb_method_definition_struct, + pub called_id: ID, + pub owner: VALUE, +} +pub type rb_callable_method_entry_t = rb_callable_method_entry_struct; +pub const VM_METHOD_TYPE_ISEQ: rb_method_type_t = 0; +pub const VM_METHOD_TYPE_CFUNC: rb_method_type_t = 1; +pub const VM_METHOD_TYPE_ATTRSET: rb_method_type_t = 2; +pub const VM_METHOD_TYPE_IVAR: rb_method_type_t = 3; +pub const VM_METHOD_TYPE_BMETHOD: rb_method_type_t = 4; +pub const VM_METHOD_TYPE_ZSUPER: rb_method_type_t = 5; +pub const VM_METHOD_TYPE_ALIAS: rb_method_type_t = 6; +pub const VM_METHOD_TYPE_UNDEF: rb_method_type_t = 7; +pub const VM_METHOD_TYPE_NOTIMPLEMENTED: rb_method_type_t = 8; +pub const VM_METHOD_TYPE_OPTIMIZED: rb_method_type_t = 9; +pub const VM_METHOD_TYPE_MISSING: rb_method_type_t = 10; +pub const VM_METHOD_TYPE_REFINED: rb_method_type_t = 11; +pub type rb_method_type_t = u32; +pub type rb_iseq_t = rb_iseq_struct; +pub type rb_cfunc_t = ::std::option::Option<unsafe extern "C" fn() -> VALUE>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_method_cfunc_struct { + pub func: rb_cfunc_t, + pub invoker: ::std::option::Option< + unsafe extern "C" fn( + recv: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + func: ::std::option::Option<unsafe extern "C" fn() -> VALUE>, + ) -> VALUE, + >, + pub argc: ::std::os::raw::c_int, +} +pub const OPTIMIZED_METHOD_TYPE_SEND: method_optimized_type = 0; +pub const OPTIMIZED_METHOD_TYPE_CALL: method_optimized_type = 1; +pub const OPTIMIZED_METHOD_TYPE_BLOCK_CALL: method_optimized_type = 2; +pub const OPTIMIZED_METHOD_TYPE_STRUCT_AREF: method_optimized_type = 3; +pub const OPTIMIZED_METHOD_TYPE_STRUCT_ASET: method_optimized_type = 4; +pub const OPTIMIZED_METHOD_TYPE__MAX: method_optimized_type = 5; +pub type method_optimized_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_position_struct { + pub lineno: ::std::os::raw::c_int, + pub column: ::std::os::raw::c_int, +} +pub type rb_code_position_t = rb_code_position_struct; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_location_struct { + pub beg_pos: rb_code_position_t, + pub end_pos: rb_code_position_t, +} +pub type rb_code_location_t = rb_code_location_struct; +pub type rb_num_t = ::std::os::raw::c_ulong; +pub type rb_snum_t = ::std::os::raw::c_long; +pub const RUBY_TAG_NONE: ruby_tag_type = 0; +pub const RUBY_TAG_RETURN: ruby_tag_type = 1; +pub const RUBY_TAG_BREAK: ruby_tag_type = 2; +pub const RUBY_TAG_NEXT: ruby_tag_type = 3; +pub const RUBY_TAG_RETRY: ruby_tag_type = 4; +pub const RUBY_TAG_REDO: ruby_tag_type = 5; +pub const RUBY_TAG_RAISE: ruby_tag_type = 6; +pub const RUBY_TAG_THROW: ruby_tag_type = 7; +pub const RUBY_TAG_FATAL: ruby_tag_type = 8; +pub const RUBY_TAG_MASK: ruby_tag_type = 15; +pub type ruby_tag_type = u32; +pub const VM_THROW_NO_ESCAPE_FLAG: ruby_vm_throw_flags = 32768; +pub const VM_THROW_STATE_MASK: ruby_vm_throw_flags = 255; +pub type ruby_vm_throw_flags = u32; +#[repr(C)] +pub struct iseq_inline_constant_cache_entry { + pub flags: VALUE, + pub value: VALUE, + pub ic_cref: *const rb_cref_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_inline_constant_cache { + pub entry: *mut iseq_inline_constant_cache_entry, + pub segments: *const ID, +} +#[repr(C)] +pub struct iseq_inline_iv_cache_entry { + pub value: u64, + pub iv_set_name: ID, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_inline_cvar_cache_entry { + pub entry: *mut rb_cvar_class_tbl_entry, +} +#[repr(C)] +#[repr(align(8))] +#[derive(Copy, Clone)] +pub struct iseq_inline_storage_entry { + pub _bindgen_opaque_blob: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_location_struct { + pub pathobj: VALUE, + pub base_label: VALUE, + pub label: VALUE, + pub first_lineno: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub code_location: rb_code_location_t, +} +pub type rb_iseq_location_t = rb_iseq_location_struct; +pub type iseq_bits_t = usize; +pub const ISEQ_TYPE_TOP: rb_iseq_type = 0; +pub const ISEQ_TYPE_METHOD: rb_iseq_type = 1; +pub const ISEQ_TYPE_BLOCK: rb_iseq_type = 2; +pub const ISEQ_TYPE_CLASS: rb_iseq_type = 3; +pub const ISEQ_TYPE_RESCUE: rb_iseq_type = 4; +pub const ISEQ_TYPE_ENSURE: rb_iseq_type = 5; +pub const ISEQ_TYPE_EVAL: rb_iseq_type = 6; +pub const ISEQ_TYPE_MAIN: rb_iseq_type = 7; +pub const ISEQ_TYPE_PLAIN: rb_iseq_type = 8; +pub type rb_iseq_type = u32; +pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1; +pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2; +pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4; +pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8; +pub const BUILTIN_ATTR_WITHOUT_INTERRUPTS: rb_builtin_attr = 16; +pub type rb_builtin_attr = u32; +pub type rb_jit_func_t = ::std::option::Option< + unsafe extern "C" fn( + arg1: *mut rb_execution_context_struct, + arg2: *mut rb_control_frame_struct, + ) -> VALUE, +>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters { + pub flags: rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1, + pub size: ::std::os::raw::c_uint, + pub lead_num: ::std::os::raw::c_int, + pub opt_num: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub post_start: ::std::os::raw::c_int, + pub post_num: ::std::os::raw::c_int, + pub block_start: ::std::os::raw::c_int, + pub opt_table: *const VALUE, + pub keyword: *const rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword, +} +#[repr(C)] +#[repr(align(4))] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 2usize]>, + pub __bindgen_padding_0: u16, +} +impl rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + #[inline] + pub fn has_lead(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_lead(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_lead_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_lead_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_opt(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_opt(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_opt_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_opt_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_post(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_post(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_post_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 3usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_post_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 3usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kw(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kw(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kw_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 4usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kw_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 4usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 5usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 5usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 6usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 6usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ambiguous_param0(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u32) } + } + #[inline] + pub fn set_ambiguous_param0(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ambiguous_param0_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 7usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ambiguous_param0_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 7usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_kwarg(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_kwarg(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_kwarg_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 8usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_kwarg_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 8usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ruby2_keywords(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u32) } + } + #[inline] + pub fn set_ruby2_keywords(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ruby2_keywords_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 9usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ruby2_keywords_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 9usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 10usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 10usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 11usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 11usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn use_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u32) } + } + #[inline] + pub fn set_use_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn use_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 12usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_use_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 12usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn forwardable(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u32) } + } + #[inline] + pub fn set_forwardable(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn forwardable_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 13usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_forwardable_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 13usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(14usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(14usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 14usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 14usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + has_lead: ::std::os::raw::c_uint, + has_opt: ::std::os::raw::c_uint, + has_rest: ::std::os::raw::c_uint, + has_post: ::std::os::raw::c_uint, + has_kw: ::std::os::raw::c_uint, + has_kwrest: ::std::os::raw::c_uint, + has_block: ::std::os::raw::c_uint, + ambiguous_param0: ::std::os::raw::c_uint, + accepts_no_kwarg: ::std::os::raw::c_uint, + ruby2_keywords: ::std::os::raw::c_uint, + anon_rest: ::std::os::raw::c_uint, + anon_kwrest: ::std::os::raw::c_uint, + use_block: ::std::os::raw::c_uint, + forwardable: ::std::os::raw::c_uint, + accepts_no_block: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 2usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 2usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let has_lead: u32 = unsafe { ::std::mem::transmute(has_lead) }; + has_lead as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let has_opt: u32 = unsafe { ::std::mem::transmute(has_opt) }; + has_opt as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let has_rest: u32 = unsafe { ::std::mem::transmute(has_rest) }; + has_rest as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let has_post: u32 = unsafe { ::std::mem::transmute(has_post) }; + has_post as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let has_kw: u32 = unsafe { ::std::mem::transmute(has_kw) }; + has_kw as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let has_kwrest: u32 = unsafe { ::std::mem::transmute(has_kwrest) }; + has_kwrest as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let has_block: u32 = unsafe { ::std::mem::transmute(has_block) }; + has_block as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let ambiguous_param0: u32 = unsafe { ::std::mem::transmute(ambiguous_param0) }; + ambiguous_param0 as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let accepts_no_kwarg: u32 = unsafe { ::std::mem::transmute(accepts_no_kwarg) }; + accepts_no_kwarg as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let ruby2_keywords: u32 = unsafe { ::std::mem::transmute(ruby2_keywords) }; + ruby2_keywords as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let anon_rest: u32 = unsafe { ::std::mem::transmute(anon_rest) }; + anon_rest as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let anon_kwrest: u32 = unsafe { ::std::mem::transmute(anon_kwrest) }; + anon_kwrest as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let use_block: u32 = unsafe { ::std::mem::transmute(use_block) }; + use_block as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let forwardable: u32 = unsafe { ::std::mem::transmute(forwardable) }; + forwardable as u64 + }); + __bindgen_bitfield_unit.set(14usize, 1u8, { + let accepts_no_block: u32 = unsafe { ::std::mem::transmute(accepts_no_block) }; + accepts_no_block as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword { + pub num: ::std::os::raw::c_int, + pub required_num: ::std::os::raw::c_int, + pub bits_start: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub table: *const ID, + pub default_values: *mut VALUE, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_iseq_insn_info { + pub body: *const iseq_insn_info_entry, + pub positions: *mut ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub succ_index_table: *mut succ_index_table, +} +pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; +pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; +pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; +pub type rb_iseq_constant_body_lvar_state = u32; +#[repr(C)] +pub struct rb_iseq_constant_body__bindgen_ty_1 { + pub flip_count: rb_snum_t, + pub script_lines: VALUE, + pub coverage: VALUE, + pub pc2branchindex: VALUE, + pub original_iseq: *mut VALUE, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union rb_iseq_constant_body__bindgen_ty_2 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +pub struct rb_iseq_struct { + pub flags: VALUE, + pub wrapper: VALUE, + pub body: *mut rb_iseq_constant_body, + pub aux: rb_iseq_struct__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1 { + pub compile_data: __BindgenUnionField<*mut iseq_compile_data>, + pub loader: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_1>, + pub exec: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_2>, + pub bindgen_union_field: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_1 { + pub obj: VALUE, + pub index: ::std::os::raw::c_int, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_2 { + pub local_hooks_cnt: ::std::os::raw::c_uint, + pub global_trace_events: rb_event_flag_t, +} +#[repr(C)] +pub struct rb_captured_block { + pub self_: VALUE, + pub ep: *const VALUE, + pub code: rb_captured_block__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_captured_block__bindgen_ty_1 { + pub iseq: __BindgenUnionField<*const rb_iseq_t>, + pub ifunc: __BindgenUnionField<*const vm_ifunc>, + pub val: __BindgenUnionField<VALUE>, + pub bindgen_union_field: u64, +} +pub const block_type_iseq: rb_block_type = 0; +pub const block_type_ifunc: rb_block_type = 1; +pub const block_type_symbol: rb_block_type = 2; +pub const block_type_proc: rb_block_type = 3; +pub type rb_block_type = u32; +#[repr(C)] +pub struct rb_block { + pub as_: rb_block__bindgen_ty_1, + pub type_: rb_block_type, +} +#[repr(C)] +pub struct rb_block__bindgen_ty_1 { + pub captured: __BindgenUnionField<rb_captured_block>, + pub symbol: __BindgenUnionField<VALUE>, + pub proc_: __BindgenUnionField<VALUE>, + pub bindgen_union_field: [u64; 3usize], +} +#[repr(C)] +pub struct rb_control_frame_struct { + pub pc: *const VALUE, + pub sp: *mut VALUE, + pub _iseq: *const rb_iseq_t, + pub self_: VALUE, + pub ep: *const VALUE, + pub block_code: *const ::std::os::raw::c_void, + pub jit_return: *mut ::std::os::raw::c_void, +} +pub type rb_control_frame_t = rb_control_frame_struct; +#[repr(C)] +pub struct rb_proc_t { + pub block: rb_block, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +impl rb_proc_t { + #[inline] + pub fn is_from_method(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_from_method(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_from_method_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_from_method_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_lambda(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_lambda(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_lambda_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_lambda_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_isolated(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_isolated(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_isolated_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_isolated_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + is_from_method: ::std::os::raw::c_uint, + is_lambda: ::std::os::raw::c_uint, + is_isolated: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let is_from_method: u32 = unsafe { ::std::mem::transmute(is_from_method) }; + is_from_method as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let is_lambda: u32 = unsafe { ::std::mem::transmute(is_lambda) }; + is_lambda as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let is_isolated: u32 = unsafe { ::std::mem::transmute(is_isolated) }; + is_isolated as u64 + }); + __bindgen_bitfield_unit + } +} +pub const VM_CHECKMATCH_TYPE_WHEN: vm_check_match_type = 1; +pub const VM_CHECKMATCH_TYPE_CASE: vm_check_match_type = 2; +pub const VM_CHECKMATCH_TYPE_RESCUE: vm_check_match_type = 3; +pub type vm_check_match_type = u32; +pub const VM_OPT_NEWARRAY_SEND_MAX: vm_opt_newarray_send_type = 1; +pub const VM_OPT_NEWARRAY_SEND_MIN: vm_opt_newarray_send_type = 2; +pub const VM_OPT_NEWARRAY_SEND_HASH: vm_opt_newarray_send_type = 3; +pub const VM_OPT_NEWARRAY_SEND_PACK: vm_opt_newarray_send_type = 4; +pub const VM_OPT_NEWARRAY_SEND_PACK_BUFFER: vm_opt_newarray_send_type = 5; +pub const VM_OPT_NEWARRAY_SEND_INCLUDE_P: vm_opt_newarray_send_type = 6; +pub type vm_opt_newarray_send_type = u32; +pub const VM_SPECIAL_OBJECT_VMCORE: vm_special_object_type = 1; +pub const VM_SPECIAL_OBJECT_CBASE: vm_special_object_type = 2; +pub const VM_SPECIAL_OBJECT_CONST_BASE: vm_special_object_type = 3; +pub type vm_special_object_type = u32; +pub type IC = *mut iseq_inline_constant_cache; +pub type IVC = *mut iseq_inline_iv_cache_entry; +pub type ICVARC = *mut iseq_inline_cvar_cache_entry; +pub const VM_FRAME_MAGIC_METHOD: vm_frame_env_flags = 286326785; +pub const VM_FRAME_MAGIC_BLOCK: vm_frame_env_flags = 572653569; +pub const VM_FRAME_MAGIC_CLASS: vm_frame_env_flags = 858980353; +pub const VM_FRAME_MAGIC_TOP: vm_frame_env_flags = 1145307137; +pub const VM_FRAME_MAGIC_CFUNC: vm_frame_env_flags = 1431633921; +pub const VM_FRAME_MAGIC_IFUNC: vm_frame_env_flags = 1717960705; +pub const VM_FRAME_MAGIC_EVAL: vm_frame_env_flags = 2004287489; +pub const VM_FRAME_MAGIC_RESCUE: vm_frame_env_flags = 2022178817; +pub const VM_FRAME_MAGIC_DUMMY: vm_frame_env_flags = 2040070145; +pub const VM_FRAME_MAGIC_MASK: vm_frame_env_flags = 2147418113; +pub const VM_FRAME_FLAG_FINISH: vm_frame_env_flags = 32; +pub const VM_FRAME_FLAG_BMETHOD: vm_frame_env_flags = 64; +pub const VM_FRAME_FLAG_CFRAME: vm_frame_env_flags = 128; +pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256; +pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512; +pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024; +pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048; +pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096; +pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2; +pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4; +pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8; +pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; +pub type vm_frame_env_flags = u32; +pub type attr_index_t = u8; +pub type shape_id_t = u32; +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 7864320; +pub const SHAPE_ID_FL_COMPLEX: shape_id_fl_type = 8388608; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 16777216; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 33554432; +pub const SHAPE_ID_LAYOUT_ROBJECT: shape_id_fl_type = 0; +pub const SHAPE_ID_LAYOUT_RCLASS: shape_id_fl_type = 67108864; +pub const SHAPE_ID_LAYOUT_RDATA: shape_id_fl_type = 134217728; +pub const SHAPE_ID_LAYOUT_OTHER: shape_id_fl_type = 201326592; +pub const SHAPE_ID_LAYOUT_MASK: shape_id_fl_type = 201326592; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 50331648; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 267911168; +pub type shape_id_fl_type = u32; +pub const CONST_DEPRECATED: rb_const_flag_t = 256; +pub const CONST_VISIBILITY_MASK: rb_const_flag_t = 255; +pub const CONST_PUBLIC: rb_const_flag_t = 0; +pub const CONST_PRIVATE: rb_const_flag_t = 1; +pub const CONST_VISIBILITY_MAX: rb_const_flag_t = 2; +pub type rb_const_flag_t = u32; +#[repr(C)] +pub struct rb_const_entry_struct { + pub flag: rb_const_flag_t, + pub line: ::std::os::raw::c_int, + pub value: VALUE, + pub file: VALUE, +} +pub type rb_const_entry_t = rb_const_entry_struct; +#[repr(C)] +pub struct rb_cvar_class_tbl_entry { + pub imemo_flags: VALUE, + pub index: u32, + pub global_cvar_state: rb_serial_t, + pub cref: *const rb_cref_t, + pub class_value: VALUE, +} +pub const VM_CALL_ARGS_SPLAT_bit: vm_call_flag_bits = 0; +pub const VM_CALL_ARGS_BLOCKARG_bit: vm_call_flag_bits = 1; +pub const VM_CALL_FCALL_bit: vm_call_flag_bits = 2; +pub const VM_CALL_VCALL_bit: vm_call_flag_bits = 3; +pub const VM_CALL_ARGS_SIMPLE_bit: vm_call_flag_bits = 4; +pub const VM_CALL_KWARG_bit: vm_call_flag_bits = 5; +pub const VM_CALL_KW_SPLAT_bit: vm_call_flag_bits = 6; +pub const VM_CALL_TAILCALL_bit: vm_call_flag_bits = 7; +pub const VM_CALL_SUPER_bit: vm_call_flag_bits = 8; +pub const VM_CALL_ZSUPER_bit: vm_call_flag_bits = 9; +pub const VM_CALL_OPT_SEND_bit: vm_call_flag_bits = 10; +pub const VM_CALL_KW_SPLAT_MUT_bit: vm_call_flag_bits = 11; +pub const VM_CALL_ARGS_SPLAT_MUT_bit: vm_call_flag_bits = 12; +pub const VM_CALL_FORWARDING_bit: vm_call_flag_bits = 13; +pub const VM_CALL__END: vm_call_flag_bits = 14; +pub type vm_call_flag_bits = u32; +#[repr(C)] +pub struct rb_callinfo_kwarg { + pub keyword_len: ::std::os::raw::c_int, + pub references: rb_atomic_t, + pub keywords: __IncompleteArrayField<VALUE>, +} +#[repr(C)] +pub struct rb_callinfo { + pub flags: VALUE, + pub kwarg: *const rb_callinfo_kwarg, + pub mid: VALUE, + pub flag: ::std::os::raw::c_uint, + pub argc: ::std::os::raw::c_uint, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_call_data { + pub ci: *const rb_callinfo, + pub cc: *const rb_callcache, +} +pub const RSTRING_CHILLED: ruby_rstring_private_flags = 49152; +pub type ruby_rstring_private_flags = u32; +pub const RHASH_PASS_AS_KEYWORDS: ruby_rhash_flags = 8192; +pub const RHASH_PROC_DEFAULT: ruby_rhash_flags = 16384; +pub const RHASH_ST_TABLE_FLAG: ruby_rhash_flags = 32768; +pub const RHASH_AR_TABLE_SIZE_MASK: ruby_rhash_flags = 983040; +pub const RHASH_AR_TABLE_SIZE_SHIFT: ruby_rhash_flags = 16; +pub const RHASH_AR_TABLE_BOUND_MASK: ruby_rhash_flags = 15728640; +pub const RHASH_AR_TABLE_BOUND_SHIFT: ruby_rhash_flags = 20; +pub const RHASH_LEV_SHIFT: ruby_rhash_flags = 25; +pub const RHASH_LEV_MAX: ruby_rhash_flags = 127; +pub type ruby_rhash_flags = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_builtin_function { + pub func_ptr: *const ::std::os::raw::c_void, + pub argc: ::std::os::raw::c_int, + pub index: ::std::os::raw::c_int, + pub name: *const ::std::os::raw::c_char, +} +pub const YARVINSN_nop: ruby_vminsn_type = 0; +pub const YARVINSN_getlocal: ruby_vminsn_type = 1; +pub const YARVINSN_setlocal: ruby_vminsn_type = 2; +pub const YARVINSN_getblockparam: ruby_vminsn_type = 3; +pub const YARVINSN_setblockparam: ruby_vminsn_type = 4; +pub const YARVINSN_getblockparamproxy: ruby_vminsn_type = 5; +pub const YARVINSN_getspecial: ruby_vminsn_type = 6; +pub const YARVINSN_setspecial: ruby_vminsn_type = 7; +pub const YARVINSN_getinstancevariable: ruby_vminsn_type = 8; +pub const YARVINSN_setinstancevariable: ruby_vminsn_type = 9; +pub const YARVINSN_getclassvariable: ruby_vminsn_type = 10; +pub const YARVINSN_setclassvariable: ruby_vminsn_type = 11; +pub const YARVINSN_opt_getconstant_path: ruby_vminsn_type = 12; +pub const YARVINSN_getconstant: ruby_vminsn_type = 13; +pub const YARVINSN_setconstant: ruby_vminsn_type = 14; +pub const YARVINSN_getglobal: ruby_vminsn_type = 15; +pub const YARVINSN_setglobal: ruby_vminsn_type = 16; +pub const YARVINSN_putnil: ruby_vminsn_type = 17; +pub const YARVINSN_putself: ruby_vminsn_type = 18; +pub const YARVINSN_putobject: ruby_vminsn_type = 19; +pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20; +pub const YARVINSN_dupstring: ruby_vminsn_type = 21; +pub const YARVINSN_dupchilledstring: ruby_vminsn_type = 22; +pub const YARVINSN_concatstrings: ruby_vminsn_type = 23; +pub const YARVINSN_anytostring: ruby_vminsn_type = 24; +pub const YARVINSN_toregexp: ruby_vminsn_type = 25; +pub const YARVINSN_intern: ruby_vminsn_type = 26; +pub const YARVINSN_newarray: ruby_vminsn_type = 27; +pub const YARVINSN_pushtoarraykwsplat: ruby_vminsn_type = 28; +pub const YARVINSN_duparray: ruby_vminsn_type = 29; +pub const YARVINSN_duphash: ruby_vminsn_type = 30; +pub const YARVINSN_expandarray: ruby_vminsn_type = 31; +pub const YARVINSN_concatarray: ruby_vminsn_type = 32; +pub const YARVINSN_concattoarray: ruby_vminsn_type = 33; +pub const YARVINSN_pushtoarray: ruby_vminsn_type = 34; +pub const YARVINSN_splatarray: ruby_vminsn_type = 35; +pub const YARVINSN_splatkw: ruby_vminsn_type = 36; +pub const YARVINSN_newhash: ruby_vminsn_type = 37; +pub const YARVINSN_newrange: ruby_vminsn_type = 38; +pub const YARVINSN_pop: ruby_vminsn_type = 39; +pub const YARVINSN_dup: ruby_vminsn_type = 40; +pub const YARVINSN_dupn: ruby_vminsn_type = 41; +pub const YARVINSN_swap: ruby_vminsn_type = 42; +pub const YARVINSN_opt_reverse: ruby_vminsn_type = 43; +pub const YARVINSN_topn: ruby_vminsn_type = 44; +pub const YARVINSN_setn: ruby_vminsn_type = 45; +pub const YARVINSN_adjuststack: ruby_vminsn_type = 46; +pub const YARVINSN_defined: ruby_vminsn_type = 47; +pub const YARVINSN_definedivar: ruby_vminsn_type = 48; +pub const YARVINSN_checkmatch: ruby_vminsn_type = 49; +pub const YARVINSN_checkkeyword: ruby_vminsn_type = 50; +pub const YARVINSN_checktype: ruby_vminsn_type = 51; +pub const YARVINSN_defineclass: ruby_vminsn_type = 52; +pub const YARVINSN_definemethod: ruby_vminsn_type = 53; +pub const YARVINSN_definesmethod: ruby_vminsn_type = 54; +pub const YARVINSN_send: ruby_vminsn_type = 55; +pub const YARVINSN_sendforward: ruby_vminsn_type = 56; +pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 57; +pub const YARVINSN_opt_new: ruby_vminsn_type = 58; +pub const YARVINSN_objtostring: ruby_vminsn_type = 59; +pub const YARVINSN_opt_ary_freeze: ruby_vminsn_type = 60; +pub const YARVINSN_opt_hash_freeze: ruby_vminsn_type = 61; +pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 62; +pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 63; +pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 64; +pub const YARVINSN_opt_duparray_send: ruby_vminsn_type = 65; +pub const YARVINSN_opt_newarray_send: ruby_vminsn_type = 66; +pub const YARVINSN_invokesuper: ruby_vminsn_type = 67; +pub const YARVINSN_invokesuperforward: ruby_vminsn_type = 68; +pub const YARVINSN_invokeblock: ruby_vminsn_type = 69; +pub const YARVINSN_leave: ruby_vminsn_type = 70; +pub const YARVINSN_throw: ruby_vminsn_type = 71; +pub const YARVINSN_jump: ruby_vminsn_type = 72; +pub const YARVINSN_branchif: ruby_vminsn_type = 73; +pub const YARVINSN_branchunless: ruby_vminsn_type = 74; +pub const YARVINSN_branchnil: ruby_vminsn_type = 75; +pub const YARVINSN_jump_without_ints: ruby_vminsn_type = 76; +pub const YARVINSN_branchif_without_ints: ruby_vminsn_type = 77; +pub const YARVINSN_branchunless_without_ints: ruby_vminsn_type = 78; +pub const YARVINSN_branchnil_without_ints: ruby_vminsn_type = 79; +pub const YARVINSN_once: ruby_vminsn_type = 80; +pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 81; +pub const YARVINSN_opt_plus: ruby_vminsn_type = 82; +pub const YARVINSN_opt_minus: ruby_vminsn_type = 83; +pub const YARVINSN_opt_mult: ruby_vminsn_type = 84; +pub const YARVINSN_opt_div: ruby_vminsn_type = 85; +pub const YARVINSN_opt_mod: ruby_vminsn_type = 86; +pub const YARVINSN_opt_eq: ruby_vminsn_type = 87; +pub const YARVINSN_opt_neq: ruby_vminsn_type = 88; +pub const YARVINSN_opt_lt: ruby_vminsn_type = 89; +pub const YARVINSN_opt_le: ruby_vminsn_type = 90; +pub const YARVINSN_opt_gt: ruby_vminsn_type = 91; +pub const YARVINSN_opt_ge: ruby_vminsn_type = 92; +pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 93; +pub const YARVINSN_opt_and: ruby_vminsn_type = 94; +pub const YARVINSN_opt_or: ruby_vminsn_type = 95; +pub const YARVINSN_opt_aref: ruby_vminsn_type = 96; +pub const YARVINSN_opt_aset: ruby_vminsn_type = 97; +pub const YARVINSN_opt_length: ruby_vminsn_type = 98; +pub const YARVINSN_opt_size: ruby_vminsn_type = 99; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 100; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 101; +pub const YARVINSN_opt_not: ruby_vminsn_type = 102; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 103; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 104; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 105; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 106; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 107; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 108; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 109; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 110; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 111; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 112; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 113; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 114; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 115; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 116; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 117; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 118; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 119; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 120; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 121; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 122; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 123; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 124; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 125; +pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 126; +pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 127; +pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 128; +pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 129; +pub const YARVINSN_trace_putnil: ruby_vminsn_type = 130; +pub const YARVINSN_trace_putself: ruby_vminsn_type = 131; +pub const YARVINSN_trace_putobject: ruby_vminsn_type = 132; +pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 133; +pub const YARVINSN_trace_dupstring: ruby_vminsn_type = 134; +pub const YARVINSN_trace_dupchilledstring: ruby_vminsn_type = 135; +pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 136; +pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 137; +pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 138; +pub const YARVINSN_trace_intern: ruby_vminsn_type = 139; +pub const YARVINSN_trace_newarray: ruby_vminsn_type = 140; +pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 141; +pub const YARVINSN_trace_duparray: ruby_vminsn_type = 142; +pub const YARVINSN_trace_duphash: ruby_vminsn_type = 143; +pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 144; +pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 145; +pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 146; +pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 147; +pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 148; +pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 149; +pub const YARVINSN_trace_newhash: ruby_vminsn_type = 150; +pub const YARVINSN_trace_newrange: ruby_vminsn_type = 151; +pub const YARVINSN_trace_pop: ruby_vminsn_type = 152; +pub const YARVINSN_trace_dup: ruby_vminsn_type = 153; +pub const YARVINSN_trace_dupn: ruby_vminsn_type = 154; +pub const YARVINSN_trace_swap: ruby_vminsn_type = 155; +pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 156; +pub const YARVINSN_trace_topn: ruby_vminsn_type = 157; +pub const YARVINSN_trace_setn: ruby_vminsn_type = 158; +pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 159; +pub const YARVINSN_trace_defined: ruby_vminsn_type = 160; +pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 161; +pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 162; +pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 163; +pub const YARVINSN_trace_checktype: ruby_vminsn_type = 164; +pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 165; +pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 166; +pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 167; +pub const YARVINSN_trace_send: ruby_vminsn_type = 168; +pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 171; +pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 175; +pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 176; +pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 177; +pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 178; +pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 179; +pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 180; +pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 181; +pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 182; +pub const YARVINSN_trace_leave: ruby_vminsn_type = 183; +pub const YARVINSN_trace_throw: ruby_vminsn_type = 184; +pub const YARVINSN_trace_jump: ruby_vminsn_type = 185; +pub const YARVINSN_trace_branchif: ruby_vminsn_type = 186; +pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 187; +pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 188; +pub const YARVINSN_trace_jump_without_ints: ruby_vminsn_type = 189; +pub const YARVINSN_trace_branchif_without_ints: ruby_vminsn_type = 190; +pub const YARVINSN_trace_branchunless_without_ints: ruby_vminsn_type = 191; +pub const YARVINSN_trace_branchnil_without_ints: ruby_vminsn_type = 192; +pub const YARVINSN_trace_once: ruby_vminsn_type = 193; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 194; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 195; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 196; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 197; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 198; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 199; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 200; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 201; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 202; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 203; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 204; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 205; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 206; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 207; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 208; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 209; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 210; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 211; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 212; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 213; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 214; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 215; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 216; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 217; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 218; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 219; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 220; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 221; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259; +pub type ruby_vminsn_type = u32; +pub type rb_iseq_callback = ::std::option::Option< + unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), +>; +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data { + pub _bindgen_opaque_blob: [u64; 24usize], +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union iseq_compile_data__bindgen_ty_1 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_2 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_3 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_insn_info_entry { + pub line_no: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub events: rb_event_flag_t, +} +pub const CATCH_TYPE_RESCUE: rb_catch_type = 3; +pub const CATCH_TYPE_ENSURE: rb_catch_type = 5; +pub const CATCH_TYPE_RETRY: rb_catch_type = 7; +pub const CATCH_TYPE_BREAK: rb_catch_type = 9; +pub const CATCH_TYPE_REDO: rb_catch_type = 11; +pub const CATCH_TYPE_NEXT: rb_catch_type = 13; +pub type rb_catch_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_catch_table_entry { + pub type_: rb_catch_type, + pub iseq: *mut rb_iseq_t, + pub start: ::std::os::raw::c_uint, + pub end: ::std::os::raw::c_uint, + pub cont: ::std::os::raw::c_uint, + pub sp: ::std::os::raw::c_uint, +} +#[repr(C, packed)] +pub struct iseq_catch_table { + pub size: ::std::os::raw::c_uint, + pub entries: __IncompleteArrayField<iseq_catch_table_entry>, +} +#[repr(C)] +#[derive(Debug)] +pub struct iseq_compile_data_storage { + pub next: *mut iseq_compile_data_storage, + pub pos: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub buff: __IncompleteArrayField<::std::os::raw::c_char>, +} +pub const DEFINED_NOT_DEFINED: defined_type = 0; +pub const DEFINED_NIL: defined_type = 1; +pub const DEFINED_IVAR: defined_type = 2; +pub const DEFINED_LVAR: defined_type = 3; +pub const DEFINED_GVAR: defined_type = 4; +pub const DEFINED_CVAR: defined_type = 5; +pub const DEFINED_CONST: defined_type = 6; +pub const DEFINED_METHOD: defined_type = 7; +pub const DEFINED_YIELD: defined_type = 8; +pub const DEFINED_ZSUPER: defined_type = 9; +pub const DEFINED_SELF: defined_type = 10; +pub const DEFINED_TRUE: defined_type = 11; +pub const DEFINED_FALSE: defined_type = 12; +pub const DEFINED_ASGN: defined_type = 13; +pub const DEFINED_EXPR: defined_type = 14; +pub const DEFINED_REF: defined_type = 15; +pub const DEFINED_FUNC: defined_type = 16; +pub const DEFINED_CONST_FROM: defined_type = 17; +pub type defined_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct zjit_jit_frame { + pub pc: *const VALUE, + pub iseq: *const rb_iseq_t, + pub materialize_block_code: bool, +} +pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; +pub type zjit_struct_offsets = u32; +pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; +pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; +pub const RCLASS_OFFSET_PRIME_FIELDS_OBJ: jit_bindgen_constants = 40; +pub const TDATA_OFFSET_FIELDS_OBJ: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16; +pub const RB_SHAPE_FLAG_SHIFT: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36; +pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48; +pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64; +pub type jit_bindgen_constants = u32; +pub const rb_invalid_shape_id: shape_id_t = 524287; +pub type rb_iseq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct succ_index_table { + pub _address: u8, +} +unsafe extern "C" { + pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void); + pub fn rb_class_attached_object(klass: VALUE) -> VALUE; + pub fn rb_singleton_class(obj: VALUE) -> VALUE; + pub fn rb_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_method_basic_definition_p(klass: VALUE, mid: ID) -> ::std::os::raw::c_int; + pub fn rb_bug(fmt: *const ::std::os::raw::c_char, ...) -> !; + pub fn rb_float_new(d: f64) -> VALUE; + pub fn rb_gc_mark(obj: VALUE); + pub fn rb_gc_mark_movable(obj: VALUE); + pub fn rb_gc_location(obj: VALUE) -> VALUE; + pub fn rb_gc_enable() -> VALUE; + pub fn rb_gc_disable() -> VALUE; + pub fn rb_gc_register_mark_object(object: VALUE); + pub fn rb_gc_writebarrier(old: VALUE, young: VALUE); + pub fn rb_class_get_superclass(klass: VALUE) -> VALUE; + pub fn rb_funcallv( + recv: VALUE, + mid: ID, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + ) -> VALUE; + pub static mut rb_mKernel: VALUE; + pub static mut rb_cBasicObject: VALUE; + pub static mut rb_cObject: VALUE; + pub static mut rb_cArray: VALUE; + pub static mut rb_cClass: VALUE; + pub static mut rb_cFalseClass: VALUE; + pub static mut rb_cFloat: VALUE; + pub static mut rb_cHash: VALUE; + pub static mut rb_cIO: VALUE; + pub static mut rb_cInteger: VALUE; + pub static mut rb_cModule: VALUE; + pub static mut rb_cNilClass: VALUE; + pub static mut rb_cNumeric: VALUE; + pub static mut rb_cRange: VALUE; + pub static mut rb_cRegexp: VALUE; + pub static mut rb_cSet: VALUE; + pub static mut rb_cString: VALUE; + pub static mut rb_cSymbol: VALUE; + pub static mut rb_cThread: VALUE; + pub static mut rb_cTrueClass: VALUE; + pub fn ruby_init(); + pub fn ruby_options( + argc: ::std::os::raw::c_int, + argv: *mut *mut ::std::os::raw::c_char, + ) -> *mut ::std::os::raw::c_void; + pub fn ruby_executable_node( + n: *mut ::std::os::raw::c_void, + status: *mut ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; + pub fn ruby_init_stack(addr: *mut ::std::os::raw::c_void); + pub fn rb_define_class(name: *const ::std::os::raw::c_char, super_: VALUE) -> VALUE; + pub fn rb_obj_class(obj: VALUE) -> VALUE; + pub fn rb_ary_new_capa(capa: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_store(ary: VALUE, key: ::std::os::raw::c_long, val: VALUE); + pub fn rb_ary_dup(ary: VALUE) -> VALUE; + pub fn rb_ary_resurrect(ary: VALUE) -> VALUE; + pub fn rb_ary_cat(ary: VALUE, train: *const VALUE, len: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_push(ary: VALUE, elem: VALUE) -> VALUE; + pub fn rb_ary_pop(ary: VALUE) -> VALUE; + pub fn rb_ary_entry(ary: VALUE, off: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_clear(ary: VALUE) -> VALUE; + pub fn rb_ary_concat(lhs: VALUE, rhs: VALUE) -> VALUE; + pub fn rb_hash_new() -> VALUE; + pub fn rb_hash_aref(hash: VALUE, key: VALUE) -> VALUE; + pub fn rb_hash_aset(hash: VALUE, key: VALUE, val: VALUE) -> VALUE; + pub fn rb_hash_bulk_insert(argc: ::std::os::raw::c_long, argv: *const VALUE, hash: VALUE); + pub fn rb_obj_is_proc(recv: VALUE) -> VALUE; + pub fn rb_protect( + func: ::std::option::Option<unsafe extern "C" fn(args: VALUE) -> VALUE>, + args: VALUE, + state: *mut ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_sym2id(obj: VALUE) -> ID; + pub fn rb_id2sym(id: ID) -> VALUE; + pub fn rb_intern(name: *const ::std::os::raw::c_char) -> ID; + pub fn rb_intern2(name: *const ::std::os::raw::c_char, len: ::std::os::raw::c_long) -> ID; + pub fn rb_id2str(id: ID) -> VALUE; + pub fn rb_sym2str(symbol: VALUE) -> VALUE; + pub fn rb_class2name(klass: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_class_new_instance_pass_kw( + argc: ::std::os::raw::c_int, + argv: *const VALUE, + klass: VALUE, + ) -> VALUE; + pub fn rb_obj_is_kind_of(obj: VALUE, klass: VALUE) -> VALUE; + pub fn rb_obj_alloc(klass: VALUE) -> VALUE; + pub fn rb_obj_frozen_p(obj: VALUE) -> VALUE; + pub fn rb_class_real(klass: VALUE) -> VALUE; + pub fn rb_class_inherited_p(scion: VALUE, ascendant: VALUE) -> VALUE; + pub fn rb_backref_get() -> VALUE; + pub fn rb_range_new(beg: VALUE, end: VALUE, excl: ::std::os::raw::c_int) -> VALUE; + pub fn rb_reg_nth_match(n: ::std::os::raw::c_int, md: VALUE) -> VALUE; + pub fn rb_reg_last_match(md: VALUE) -> VALUE; + pub fn rb_reg_match_pre(md: VALUE) -> VALUE; + pub fn rb_reg_match_post(md: VALUE) -> VALUE; + pub fn rb_reg_match_last(md: VALUE) -> VALUE; + pub fn rb_utf8_str_new( + ptr: *const ::std::os::raw::c_char, + len: ::std::os::raw::c_long, + ) -> VALUE; + pub fn rb_str_buf_append(dst: VALUE, src: VALUE) -> VALUE; + pub fn rb_str_dup(str_: VALUE) -> VALUE; + pub fn rb_str_intern(str_: VALUE) -> VALUE; + pub fn rb_mod_name(mod_: VALUE) -> VALUE; + pub fn rb_ivar_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_ivar_set(obj: VALUE, name: ID, val: VALUE) -> VALUE; + pub fn rb_ivar_defined(obj: VALUE, name: ID) -> VALUE; + pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_const_get(space: VALUE, name: ID) -> VALUE; + pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; + pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; + pub fn rb_reg_new_from_values( + cnt: ::std::os::raw::c_long, + elements: *const VALUE, + opt: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_ary_tmp_new_from_values( + arg1: VALUE, + arg2: ::std::os::raw::c_long, + arg3: *const VALUE, + ) -> VALUE; + pub fn rb_ec_ary_new_from_values( + ec: *mut rb_execution_context_struct, + n: ::std::os::raw::c_long, + elts: *const VALUE, + ) -> VALUE; + pub fn rb_vm_top_self() -> VALUE; + pub static mut rb_vm_insn_count: u64; + pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t; + pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t; + pub fn rb_callable_method_entry_or_negative( + klass: VALUE, + id: ID, + ) -> *const rb_callable_method_entry_t; + pub static mut rb_cISeq: VALUE; + pub static mut rb_cRubyVM: VALUE; + pub static mut rb_mRubyVMFrozenCore: VALUE; + pub static mut rb_block_param_proxy: VALUE; + pub fn rb_vm_ep_local_ep(ep: *const VALUE) -> *const VALUE; + pub fn rb_iseq_path(iseq: *const rb_iseq_t) -> VALUE; + pub fn rb_vm_env_write(ep: *const VALUE, index: ::std::os::raw::c_int, v: VALUE); + pub fn rb_vm_bh_to_procval(ec: *const rb_execution_context_t, block_handler: VALUE) -> VALUE; + pub fn rb_vm_frame_method_entry( + cfp: *const rb_control_frame_t, + ) -> *const rb_callable_method_entry_t; + pub fn rb_obj_info(obj: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_raw_obj_info( + buff: *mut ::std::os::raw::c_char, + buff_size: usize, + obj: VALUE, + ) -> *const ::std::os::raw::c_char; + pub fn rb_ec_stack_check(ec: *mut rb_execution_context_struct) -> ::std::os::raw::c_int; + pub fn rb_gc_writebarrier_remember(obj: VALUE); + pub fn rb_shape_id_offset() -> i32; + pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t; + pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool; + pub fn rb_shape_transition_add_ivar_no_warnings( + shape_id: shape_id_t, + id: ID, + klass: VALUE, + ) -> shape_id_t; + pub fn rb_const_lookup(klass: VALUE, id: ID) -> *mut rb_const_entry_t; + pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE; + pub fn rb_gvar_get(arg1: ID) -> VALUE; + pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; + pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32); + pub fn rb_vm_barrier(); + pub fn rb_str_byte_substr(str_: VALUE, beg: VALUE, len: VALUE) -> VALUE; + pub fn rb_str_substr_two_fixnums( + str_: VALUE, + beg: VALUE, + len: VALUE, + empty: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_obj_as_string_result(str_: VALUE, obj: VALUE) -> VALUE; + pub fn rb_str_concat_literals(num: usize, strary: *const VALUE) -> VALUE; + pub fn rb_ec_str_resurrect( + ec: *mut rb_execution_context_struct, + str_: VALUE, + chilled: bool, + ) -> VALUE; + pub fn rb_to_hash_type(obj: VALUE) -> VALUE; + pub fn rb_hash_stlike_foreach( + hash: VALUE, + func: st_foreach_callback_func, + arg: st_data_t, + ) -> ::std::os::raw::c_int; + pub fn rb_hash_new_with_size(size: st_index_t) -> VALUE; + pub fn rb_hash_new_with_bulk_insert(argc: ::std::os::raw::c_long, argv: *const VALUE) -> VALUE; + pub fn rb_hash_resurrect(hash: VALUE) -> VALUE; + pub fn rb_hash_stlike_lookup( + hash: VALUE, + key: st_data_t, + pval: *mut st_data_t, + ) -> ::std::os::raw::c_int; + pub fn rb_insn_len(insn: VALUE) -> ::std::os::raw::c_int; + pub fn rb_vm_insn_decode(encoded: VALUE) -> ::std::os::raw::c_int; + pub fn rb_float_plus(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_minus(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_mul(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_div(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_flo_to_i(num: VALUE) -> VALUE; + pub fn rb_fix_aref(fix: VALUE, idx: VALUE) -> VALUE; + pub fn rb_vm_insn_addr2opcode(addr: *const ::std::os::raw::c_void) -> ::std::os::raw::c_int; + pub fn rb_iseq_line_no(iseq: *const rb_iseq_t, pos: usize) -> ::std::os::raw::c_uint; + pub fn rb_iseqw_to_iseq(iseqw: VALUE) -> *const rb_iseq_t; + pub fn rb_iseq_label(iseq: *const rb_iseq_t) -> VALUE; + pub fn rb_iseq_defined_string(type_: defined_type) -> VALUE; + pub fn rb_profile_frames( + start: ::std::os::raw::c_int, + limit: ::std::os::raw::c_int, + buff: *mut VALUE, + lines: *mut ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; + pub fn rb_profile_frame_path(frame: VALUE) -> VALUE; + pub fn rb_profile_frame_absolute_path(frame: VALUE) -> VALUE; + pub fn rb_profile_frame_full_label(frame: VALUE) -> VALUE; + pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); + pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; + pub fn rb_zjit_iseq_insn_set( + iseq: *const rb_iseq_t, + insn_idx: ::std::os::raw::c_uint, + bare_insn: ruby_vminsn_type, + ); + pub fn rb_iseq_get_zjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; + pub fn rb_iseq_set_zjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); + pub fn rb_zjit_print_exception(); + pub fn rb_zjit_singleton_class_p(klass: VALUE) -> bool; + pub fn rb_zjit_defined_ivar(obj: VALUE, id: ID, pushval: VALUE) -> VALUE; + pub fn rb_zjit_method_tracing_currently_enabled() -> bool; + pub fn rb_zjit_iseq_tracing_currently_enabled() -> bool; + pub fn rb_zjit_insn_leaf(insn: ::std::os::raw::c_int, opes: *const VALUE) -> bool; + pub fn rb_zjit_local_id(iseq: *const rb_iseq_t, idx: ::std::os::raw::c_uint) -> ID; + pub fn rb_zjit_cme_is_cfunc( + me: *const rb_callable_method_entry_t, + func: *const ::std::os::raw::c_void, + ) -> bool; + pub fn rb_zjit_vm_search_method( + cd_owner: VALUE, + cd: *mut rb_call_data, + recv: VALUE, + ) -> *const rb_callable_method_entry_struct; + pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; + pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; + pub fn rb_vm_untag_block_handler(block_handler: VALUE) -> VALUE; + pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; + pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; + pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; + pub fn rb_iseq_bare_opcode_at_pc( + iseq: *const rb_iseq_t, + pc: *const VALUE, + ) -> ::std::os::raw::c_int; + pub fn rb_RSTRING_LEN(str_: VALUE) -> ::std::os::raw::c_ulong; + pub fn rb_RSTRING_PTR(str_: VALUE) -> *mut ::std::os::raw::c_char; + pub fn rb_insn_name(insn: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_vm_ci_argc(ci: *const rb_callinfo) -> ::std::os::raw::c_uint; + pub fn rb_vm_ci_mid(ci: *const rb_callinfo) -> ID; + pub fn rb_vm_ci_flag(ci: *const rb_callinfo) -> ::std::os::raw::c_uint; + pub fn rb_vm_ci_kwarg(ci: *const rb_callinfo) -> *const rb_callinfo_kwarg; + pub fn rb_get_cikw_keyword_len(cikw: *const rb_callinfo_kwarg) -> ::std::os::raw::c_int; + pub fn rb_get_cikw_keywords_idx( + cikw: *const rb_callinfo_kwarg, + idx: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_METHOD_ENTRY_VISI(me: *const rb_callable_method_entry_t) -> rb_method_visibility_t; + pub fn rb_get_cme_def_type(cme: *const rb_callable_method_entry_t) -> rb_method_type_t; + pub fn rb_get_cme_def_body_attr_id(cme: *const rb_callable_method_entry_t) -> ID; + pub fn rb_get_cme_def_body_optimized_type( + cme: *const rb_callable_method_entry_t, + ) -> method_optimized_type; + pub fn rb_get_cme_def_body_optimized_index( + cme: *const rb_callable_method_entry_t, + ) -> ::std::os::raw::c_uint; + pub fn rb_get_cme_def_body_cfunc( + cme: *const rb_callable_method_entry_t, + ) -> *mut rb_method_cfunc_t; + pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize; + pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID; + pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; + pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; + pub fn rb_optimized_call( + recv: VALUE, + ec: *mut rb_execution_context_t, + argc: ::std::os::raw::c_int, + argv: *mut VALUE, + kw_splat: ::std::os::raw::c_int, + block_handler: VALUE, + ) -> VALUE; + pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int; + pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; + pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_local_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_parent_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_local_table_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_iseq_encoded(iseq: *const rb_iseq_t) -> *mut VALUE; + pub fn rb_get_iseq_body_stack_max(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_type(iseq: *const rb_iseq_t) -> rb_iseq_type; + pub fn rb_get_iseq_flags_has_lead(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_opt(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_kw(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_post(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_kwrest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_anon_kwrest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_rest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_ruby2_keywords(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_block(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_ambiguous_param0(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_accepts_no_kwarg(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_forwardable(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_body_param_keyword( + iseq: *const rb_iseq_t, + ) -> *const rb_iseq_param_keyword_struct; + pub fn rb_get_iseq_body_param_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_param_lead_num(iseq: *const rb_iseq_t) -> ::std::os::raw::c_int; + pub fn rb_get_iseq_body_param_opt_num(iseq: *const rb_iseq_t) -> ::std::os::raw::c_int; + pub fn rb_get_iseq_body_param_opt_table(iseq: *const rb_iseq_t) -> *const VALUE; + pub fn rb_get_ec_cfp(ec: *const rb_execution_context_t) -> *mut rb_control_frame_struct; + pub fn rb_get_cfp_iseq(cfp: *mut rb_control_frame_struct) -> *const rb_iseq_t; + pub fn rb_get_cfp_pc(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_sp(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_self(cfp: *mut rb_control_frame_struct) -> VALUE; + pub fn rb_get_cfp_ep(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_ep_level(cfp: *mut rb_control_frame_struct, lv: u32) -> *const VALUE; + pub fn rb_yarv_class_of(obj: VALUE) -> VALUE; + pub fn rb_FL_TEST(obj: VALUE, flags: VALUE) -> VALUE; + pub fn rb_FL_TEST_RAW(obj: VALUE, flags: VALUE) -> VALUE; + pub fn rb_RB_TYPE_P(obj: VALUE, t: ruby_value_type) -> bool; + pub fn rb_get_call_data_ci(cd: *const rb_call_data) -> *const rb_callinfo; + pub fn rb_BASIC_OP_UNREDEFINED_P(bop: ruby_basic_operators, klass: u32) -> bool; + pub fn rb_RCLASS_ORIGIN(c: VALUE) -> VALUE; + pub fn rb_assert_iseq_handle(handle: VALUE); + pub fn rb_assert_holding_vm_lock(); + pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int; + pub fn rb_assert_cme_handle(handle: VALUE); + pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; + pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); + pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); + pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; + pub fn rb_jit_multi_ractor_p() -> bool; + pub fn rb_jit_class_fields_embedded_p(klass: VALUE) -> bool; + pub fn rb_jit_data_fields_embedded_p(obj: VALUE) -> bool; + pub fn rb_jit_vm_lock_then_barrier( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_jit_vm_unlock( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); + pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; + pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE); + pub fn rb_jit_shape_capacity(shape_id: shape_id_t) -> attr_index_t; +} diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs new file mode 100644 index 0000000000..05b0055032 --- /dev/null +++ b/zjit/src/cruby_methods.rs @@ -0,0 +1,1040 @@ +/*! This module contains assertions we make about runtime properties of core library methods. + * Some properties that influence codegen: + * - Whether the method has been redefined since boot + * - Whether the C method can yield to the GC + * - Whether the C method makes any method calls + * + * For Ruby methods, many of these properties can be inferred through analyzing the + * bytecode, but for C methods we resort to annotation and validation in debug builds. + */ + +use crate::cruby::*; +use std::collections::HashMap; +use std::ffi::c_void; +use crate::hir_type::{types, Type}; +use crate::hir::{self, FieldName}; + +// Array iteration builtin functions (defined in array.c) +unsafe extern "C" { + fn rb_jit_ary_at_end(ec: EcPtr, self_: VALUE, index: VALUE) -> VALUE; + fn rb_jit_ary_at(ec: EcPtr, self_: VALUE, index: VALUE) -> VALUE; + fn rb_jit_fixnum_inc(ec: EcPtr, self_: VALUE, num: VALUE) -> VALUE; + fn rb_str_equal(str1: VALUE, str2: VALUE) -> VALUE; +} + +pub struct Annotations { + cfuncs: HashMap<*mut c_void, FnProperties>, + builtin_funcs: HashMap<*mut c_void, FnProperties>, +} + +/// Runtime behaviors of C functions that implement a Ruby method +#[derive(Clone, Copy)] +pub struct FnProperties { + /// Whether it's possible for the function to yield to the GC + pub no_gc: bool, + /// Whether it's possible for the function to make a ruby call + pub leaf: bool, + /// What Type the C function returns + pub return_type: Type, + /// Whether it's legal to remove the call if the result is unused + pub elidable: bool, + pub inline: fn(&mut hir::Function, hir::BlockId, hir::InsnId, &[hir::InsnId], hir::InsnId) -> Option<hir::InsnId>, +} + +/// A safe default for un-annotated Ruby methods: we can't optimize them or their returned values. +impl Default for FnProperties { + fn default() -> Self { + Self { + no_gc: false, + leaf: false, + return_type: types::BasicObject, + elidable: false, + inline: no_inline, + } + } +} + +impl Annotations { + /// Query about properties of a C method + pub fn get_cfunc_properties(&self, method: *const rb_callable_method_entry_t) -> Option<FnProperties> { + let fn_ptr = unsafe { + if VM_METHOD_TYPE_CFUNC != get_cme_def_type(method) { + return None; + } + get_mct_func(get_cme_def_body_cfunc(method.cast())) + }; + self.cfuncs.get(&fn_ptr).copied() + } + + /// Query about properties of a builtin function by its pointer + pub fn get_builtin_properties(&self, bf: *const rb_builtin_function) -> Option<FnProperties> { + let func_ptr = unsafe { (*bf).func_ptr as *mut c_void }; + self.builtin_funcs.get(&func_ptr).copied() + } +} + +fn annotate_c_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { + // Lookup function pointer of the C method + let fn_ptr = unsafe { + // TODO(alan): (side quest) make rust methods and clean up glue code for rb_method_cfunc_t and + // rb_method_definition_t. + let method_id = rb_intern2(method_name.as_ptr().cast(), method_name.len() as _); + let method = rb_method_entry_at(class, method_id); + assert!(!method.is_null()); + // ME-to-CME cast is fine due to identical layout + debug_assert_eq!(VM_METHOD_TYPE_CFUNC, get_cme_def_type(method.cast())); + get_mct_func(get_cme_def_body_cfunc(method.cast())) + }; + + props_map.insert(fn_ptr, props); +} + +/// Look up a method and find its builtin function pointer by parsing its ISEQ +/// We currently only support methods with exactly one invokebuiltin instruction +fn annotate_builtin_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { + unsafe { + let method_id = rb_intern2(method_name.as_ptr().cast(), method_name.len().try_into().unwrap()); + let method = rb_method_entry_at(class, method_id); + if method.is_null() { + panic!("Method {}#{} not found", std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), method_name); + } + + // Cast ME to CME - they have identical layout + let cme = method.cast::<rb_callable_method_entry_t>(); + let def_type = get_cme_def_type(cme); + + if def_type != VM_METHOD_TYPE_ISEQ { + panic!("Method {}#{} is not an ISEQ method (type: {})", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name, def_type); + } + + // Get the ISEQ from the method definition + let iseq = get_def_iseq_ptr((*cme).def); + if iseq.is_null() { + panic!("Failed to get ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + + // Get the size of the ISEQ in instruction units + let encoded_size = rb_iseq_encoded_size(iseq); + + // Scan through the ISEQ to find invokebuiltin instructions + let mut insn_idx: u32 = 0; + let mut func_ptr = std::ptr::null_mut::<c_void>(); + + while insn_idx < encoded_size { + // Get the PC for this instruction index + let pc = rb_iseq_pc_at_idx(iseq, insn_idx); + + // Get the opcode using the proper decoder + let opcode = rb_iseq_opcode_at_pc(iseq, pc); + + if opcode == YARVINSN_invokebuiltin as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate_leave as i32 { + // The first operand is the builtin function pointer + let bf_value = *pc.add(1); + let bf_ptr: *const rb_builtin_function = bf_value.as_ptr(); + + if func_ptr.is_null() { + func_ptr = (*bf_ptr).func_ptr as *mut c_void; + } else { + panic!("Multiple invokebuiltin instructions found in ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + } + + // Move to the next instruction using the proper length + insn_idx = insn_idx.saturating_add(rb_insn_len(VALUE(opcode as usize)).try_into().unwrap()); + } + + // Only insert the properties if its iseq has exactly one invokebuiltin instruction + props_map.insert(func_ptr, props); + } +} + +/// Gather annotations. Run this right after boot since the annotations +/// are about the stock versions of methods. +pub fn init() -> Annotations { + let cfuncs = &mut HashMap::new(); + let builtin_funcs = &mut HashMap::new(); + + macro_rules! annotate { + ($module:ident, $method_name:literal, $inline:ident) => { + let mut props = FnProperties::default(); + props.inline = $inline; + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + } + } + + macro_rules! annotate_builtin { + ($module:ident, $method_name:literal, $return_type:expr) => { + annotate_builtin!($module, $method_name, $return_type, no_gc, leaf, elidable) + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + } + } + + annotate!(rb_mKernel, "itself", inline_kernel_itself); + annotate!(rb_mKernel, "block_given?", inline_kernel_block_given_p); + annotate!(rb_mKernel, "===", inline_eqq); + annotate!(rb_mKernel, "is_a?", inline_kernel_is_a_p); + annotate!(rb_cString, "bytesize", inline_string_bytesize); + annotate!(rb_cString, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "length", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "getbyte", inline_string_getbyte); + annotate!(rb_cString, "setbyte", inline_string_setbyte); + annotate!(rb_cString, "empty?", inline_string_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cString, "<<", inline_string_append); + annotate!(rb_cString, "==", inline_string_eq); + // Not elidable; has a side effect of setting the encoding if ENC_CODERANGE_UNKNOWN. + // TOOD(max): Turn this into a load/compare. Will need to side-exit or do the full call if + // ENC_CODERANGE_UNKNOWN. + annotate!(rb_cString, "ascii_only?", types::BoolExact, no_gc, leaf); + annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable); + annotate!(rb_cModule, "===", inline_module_eqq, types::BoolExact, no_gc, leaf); + annotate!(rb_cArray, "length", inline_array_length, types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cArray, "empty?", inline_array_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cArray, "reverse", types::ArrayExact, leaf, elidable); + annotate!(rb_cArray, "join", types::StringExact); + annotate!(rb_cArray, "[]", inline_array_aref); + annotate!(rb_cArray, "[]=", inline_array_aset); + annotate!(rb_cArray, "<<", inline_array_push); + annotate!(rb_cArray, "push", inline_array_push); + annotate!(rb_cArray, "pop", inline_array_pop); + annotate!(rb_cHash, "[]", inline_hash_aref); + annotate!(rb_cHash, "[]=", inline_hash_aset); + annotate!(rb_cHash, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cNilClass, "nil?", inline_nilclass_nil_p); + annotate!(rb_mKernel, "nil?", inline_kernel_nil_p); + annotate!(rb_mKernel, "respond_to?", inline_kernel_respond_to_p); + annotate!(rb_cBasicObject, "==", inline_basic_object_eq, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!", inline_basic_object_not, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!=", inline_basic_object_neq, types::BoolExact); + annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); + annotate!(rb_cClass, "allocate", inline_class_allocate); + annotate!(rb_cInteger, "succ", inline_integer_succ); + annotate!(rb_cInteger, "^", inline_integer_xor); + annotate!(rb_cInteger, "==", inline_integer_eq); + annotate!(rb_cInteger, "+", inline_integer_plus); + annotate!(rb_cInteger, "-", inline_integer_minus); + annotate!(rb_cInteger, "*", inline_integer_mult); + annotate!(rb_cInteger, "/", inline_integer_div); + annotate!(rb_cInteger, "%", inline_integer_mod); + annotate!(rb_cInteger, "&", inline_integer_and); + annotate!(rb_cInteger, "|", inline_integer_or); + annotate!(rb_cInteger, ">", inline_integer_gt); + annotate!(rb_cInteger, ">=", inline_integer_ge); + annotate!(rb_cInteger, "<", inline_integer_lt); + annotate!(rb_cInteger, "<=", inline_integer_le); + annotate!(rb_cInteger, "<<", inline_integer_lshift); + annotate!(rb_cInteger, ">>", inline_integer_rshift); + annotate!(rb_cInteger, "[]", inline_integer_aref); + annotate!(rb_cInteger, "to_s", types::StringExact); + annotate!(rb_cFloat, "+", inline_float_plus); + annotate!(rb_cFloat, "-", inline_float_minus); + annotate!(rb_cFloat, "*", inline_float_mul); + annotate!(rb_cFloat, "/", inline_float_div); + annotate!(rb_cFloat, "to_i", inline_float_to_i); + annotate!(rb_cFloat, "to_int", inline_float_to_i); + annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact); + annotate!(rb_cFloat, "nan?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cFloat, "finite?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cFloat, "infinite?", types::Fixnum.union(types::NilClass), no_gc, leaf, elidable); + let thread_singleton = unsafe { rb_singleton_class(rb_cThread) }; + annotate!(thread_singleton, "current", inline_thread_current, types::BasicObject, no_gc, leaf); + + annotate_builtin!(rb_cInteger, "zero?", types::BoolExact); + annotate_builtin!(rb_cInteger, "even?", types::BoolExact); + annotate_builtin!(rb_cInteger, "odd?", types::BoolExact); + annotate_builtin!(rb_cFloat, "zero?", types::BoolExact); + annotate_builtin!(rb_cFloat, "positive?", types::BoolExact); + annotate_builtin!(rb_cFloat, "negative?", types::BoolExact); + annotate_builtin!(rb_mKernel, "Float", types::Float); + annotate_builtin!(rb_mKernel, "Integer", types::Integer); + annotate_builtin!(rb_mKernel, "class", inline_kernel_class, types::Class, leaf); + annotate_builtin!(rb_mKernel, "frozen?", types::BoolExact); + annotate_builtin!(rb_cSymbol, "name", types::StringExact); + annotate_builtin!(rb_cSymbol, "to_s", types::StringExact); + + // Array iteration builtins (used in with_jit Array#each, map, select, find) + builtin_funcs.insert(rb_jit_fixnum_inc as *mut c_void, FnProperties { inline: inline_fixnum_inc, return_type: types::Fixnum, ..Default::default() }); + builtin_funcs.insert(rb_jit_ary_at as *mut c_void, FnProperties { inline: inline_ary_at, ..Default::default() }); + builtin_funcs.insert(rb_jit_ary_at_end as *mut c_void, FnProperties { inline: inline_ary_at_end, return_type: types::BoolExact, ..Default::default() }); + + Annotations { + cfuncs: std::mem::take(cfuncs), + builtin_funcs: std::mem::take(builtin_funcs), + } +} + +fn no_inline(_fun: &mut hir::Function, _block: hir::BlockId, _recv: hir::InsnId, _args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + None +} + +fn inline_string_to_s(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::StringExact, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + return Some(recv); + } + None +} + +fn inline_thread_current(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let ec = fun.push_insn(block, hir::Insn::LoadEC); + let thread_ptr = fun.push_insn(block, hir::Insn::LoadField { + recv: ec, + id: FieldName::thread_ptr, + offset: RUBY_OFFSET_EC_THREAD_PTR as i32, + return_type: types::CPtr, + }); + let thread_self = fun.push_insn(block, hir::Insn::LoadField { + recv: thread_ptr, + id: FieldName::SelfParam, + offset: RUBY_OFFSET_THREAD_SELF as i32, + // TODO(max): Add Thread type. But Thread.current is not guaranteed to be an exact Thread. + // You can make subclasses... + return_type: types::BasicObject, + }); + Some(thread_self) +} + +fn inline_kernel_itself(_fun: &mut hir::Function, _block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() { + // No need to coerce the receiver; that is done by the Send rewriting. + return Some(recv); + } + None +} + +fn inline_kernel_block_given_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(fun.iseq()) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + // Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + // Equivalent of GET_LEP() macro. + let level = crate::cruby::get_lvar_level(fun.iseq()); + let lep = fun.push_insn(block, hir::Insn::GetEP { level }); + Some(fun.push_insn(block, hir::Insn::IsBlockGiven { lep })) + } else { + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) + } +} + +fn inline_array_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index] = args { + if fun.likely_a(recv, types::Array, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::Array, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, reason: SideExitReason::GuardLess, state }); + let index = fun.push_insn(block, hir::Insn::AdjustBounds { index, length }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + return Some(result); + } + } + None +} + +fn inline_array_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index, val] = args { + if fun.likely_a(recv, types::ArrayExact, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::ArrayExact, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + fun.guard_not_frozen(block, recv, state); + fun.guard_not_shared(block, recv, state); + + // Bounds check: unbox Fixnum index and guard 0 <= idx < length. + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, reason: SideExitReason::GuardLess, state }); + let index = fun.push_insn(block, hir::Insn::AdjustBounds { index, length }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + + let _ = fun.push_insn(block, hir::Insn::ArrayAset { array: recv, index, val }); + fun.push_insn(block, hir::Insn::WriteBarrier { recv, val }); + return Some(val); + } + } + None +} + +fn inline_array_push(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Inline only the case of `<<` or `push` when called with a single argument. + if let &[val] = args { + if !fun.likely_a(recv, types::Array, state) { return None; } + let recv = fun.coerce_to(block, recv, types::Array, state); + fun.guard_not_frozen(block, recv, state); + let _ = fun.push_insn(block, hir::Insn::ArrayPush { array: recv, val, state }); + return Some(recv); + } + None +} + +fn inline_array_pop(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Only inline the case of no arguments. + let &[] = args else { return None; }; + if !fun.likely_a(recv, types::Array, state) { return None; } + let recv = fun.coerce_to(block, recv, types::Array, state); + fun.guard_not_frozen(block, recv, state); + fun.guard_not_shared(block, recv, state); + Some(fun.push_insn(block, hir::Insn::ArrayPop { array: recv, state })) +} + +fn inline_hash_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let result = fun.push_insn(block, hir::Insn::HashAref { hash: recv, key, state }); + Some(result) + } else { + None + } +} + +fn inline_hash_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key, val] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let _ = fun.push_insn(block, hir::Insn::HashAset { hash: recv, key, val, state }); + // Hash#[]= returns the value, not the hash + Some(val) + } else { + None + } +} + +fn inline_string_bytesize(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + + let result = fun.push_insn(block, hir::Insn::BoxFixnum { + val: len, + state, + }); + + return Some(result); + } + None +} + +fn inline_string_getbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) { + // String#getbyte with a Fixnum is leaf and nogc; otherwise it may run arbitrary Ruby code + // when converting the index to a C integer. + let index = fun.coerce_to(block, index, types::Fixnum, state); + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + // TODO(max): Find a way to mark these guards as not needed for correctness... as in, once + // the data dependency is gone (say, the StringGetbyte is elided), they can also be elided. + // + // This is unlike most other guards. + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, reason: SideExitReason::GuardLess, state }); + let unboxed_index = fun.push_insn(block, hir::Insn::AdjustBounds { index: unboxed_index, length: len }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + let result = fun.push_insn(block, hir::Insn::StringGetbyte { string: recv, index: unboxed_index }); + return Some(result); + } + None +} + +fn inline_string_setbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index, value] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) && fun.likely_a(value, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let value = fun.coerce_to(block, value, types::Fixnum, state); + + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, reason: SideExitReason::GuardLess, state }); + let unboxed_index = fun.push_insn(block, hir::Insn::AdjustBounds { index: unboxed_index, length: len }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + // We know that all String are HeapObject, so no need to insert a GuardType(HeapObject). + fun.guard_not_frozen(block, recv, state); + let _ = fun.push_insn(block, hir::Insn::StringSetbyteFixnum { string: recv, index, value }); + // String#setbyte returns the fixnum provided as its `value` argument back to the caller. + Some(value) + } else { + None + } +} + +fn inline_string_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let is_zero = fun.push_insn(block, hir::Insn::IsBitEqual { left: len, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: is_zero }); + Some(result) +} + +fn inline_string_append(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Inline only StringExact << String, which matches original type check from + // `vm_opt_ltlt`, which checks `RB_TYPE_P(obj, T_STRING)`. + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::String, state); + let _ = fun.push_insn(block, hir::Insn::StringAppend { recv, other, state }); + return Some(recv); + } + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::Fixnum, state); + let _ = fun.push_insn(block, hir::Insn::StringAppendCodepoint { recv, other, state }); + return Some(recv); + } + None +} + +fn try_inline_string_equal(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if fun.likely_a(recv, types::String, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let other = fun.coerce_to(block, other, types::String, state); + let result = fun.push_insn(block, hir::Insn::StringEqual { left: recv, right: other }); + return Some(result); + } + None +} + +fn inline_string_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_string_equal(fun, block, recv, other, state) +} + +fn inline_module_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(recv, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: other, class: recv }); + return Some(result); + } + None +} + +fn inline_array_length(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let result = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + return Some(result); + } + None +} + +fn inline_array_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let result_c = fun.push_insn(block, hir::Insn::IsBitEqual { left: length_cint, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: result_c }); + return Some(result); + } + None +} + +fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + if fun.likely_a(recv, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left, right, state }); + return Some(result); + } + None +} + +fn inline_integer_xor(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[right] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumXor { left, right }); + return Some(result); + } + None +} + +fn try_inline_float_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, FLOAT_REDEFINED_OP_FLAG) } { + return None; + } + // Receiver must be Flonum (cheap tag check: (val & 3) == 2). + // The other operand can be Flonum or Fixnum since rb_float_plus/minus/mul/div + // handle both via fast paths (FIXNUM_P check + cast to double). + // HeapFloat falls back to CCallWithFrame via the default Send path. + if fun.likely_a(recv, types::Flonum, state) + && (fun.likely_a(other, types::Flonum, state) || fun.likely_a(other, types::Fixnum, state)) + { + let recv = fun.coerce_to(block, recv, types::Flonum, state); + let other_type = if fun.likely_a(other, types::Flonum, state) { types::Flonum } else { types::Fixnum }; + let other = fun.coerce_to(block, other, other_type, state); + return Some(fun.push_insn(block, f(recv, other))); + } + None +} + +fn inline_float_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatAdd { recv, other, state }, BOP_PLUS, recv, other, state) +} + +fn inline_float_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatSub { recv, other, state }, BOP_MINUS, recv, other, state) +} + +fn inline_float_mul(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatMul { recv, other, state }, BOP_MULT, recv, other, state) +} + +fn inline_float_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatDiv { recv, other, state }, BOP_DIV, recv, other, state) +} + +fn inline_float_to_i(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Flonum, state) { + let recv = fun.coerce_to(block, recv, types::Flonum, state); + return Some(fun.push_insn(block, hir::Insn::FloatToInt { recv, state })); + } + None +} + +fn try_inline_fixnum_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, left: hir::InsnId, right: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + return None; + } + if fun.likely_a(left, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + if bop == BOP_NEQ { + // For opt_neq, the interpreter checks that both neq and eq are unchanged. + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); + } + // Rely on the MethodRedefined PatchPoint for other bops. + let left = fun.coerce_to(block, left, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + return Some(fun.push_insn(block, f(left, right))); + } + None +} + +fn inline_integer_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumEq { left, right }, BOP_EQ, recv, other, state) +} + +fn inline_integer_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, other, state) +} + +fn inline_integer_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, other, state) +} + +fn inline_integer_mult(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMult { left, right, state }, BOP_MULT, recv, other, state) +} + +fn inline_integer_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, other, state) +} + +fn inline_integer_mod(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMod { left, right, state }, BOP_MOD, recv, other, state) +} + +fn inline_integer_and(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAnd { left, right, }, BOP_AND, recv, other, state) +} + +fn inline_integer_or(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumOr { left, right, }, BOP_OR, recv, other, state) +} + +fn inline_integer_gt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGt { left, right }, BOP_GT, recv, other, state) +} + +fn inline_integer_ge(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGe { left, right }, BOP_GE, recv, other, state) +} + +fn inline_integer_lt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLt { left, right }, BOP_LT, recv, other, state) +} + +fn inline_integer_le(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLe { left, right }, BOP_LE, recv, other, state) +} + +fn inline_integer_lshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLShift { left, right, state }, BOP_LTLT, recv, other, state) +} + +fn inline_integer_rshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + // TODO(max): If other_value > 63, rewrite to constant zero. + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumRShift { left, right }, BOP_GTGT, recv, other, state) +} + +fn inline_integer_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(index, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::Fixnum, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumAref { recv, index }); + return Some(result); + } + None +} + +fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_not(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.type_of(recv).is_known_truthy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) }); + return Some(result); + } + if fun.type_of(recv).is_known_falsy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + return Some(result); + } + None +} + +fn try_inline_string_not_equal(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !fun.likely_a(recv, types::String, state) || !fun.likely_a(other, types::String, state) { + return None; + } + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // String#!= is lowered to #==. Keep this specialization only while #== + // still resolves to rb_str_equal. + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_str_equal as _, state) { + return None; + } + + let eq_result = try_inline_string_equal(fun, block, recv, other, state)?; + // StringEqual always returns a Ruby boolean (Qtrue/Qfalse), + // so `!=` can be lowered to `eq_result != Qtrue`. + let true_val = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + let not_equal = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: eq_result, right: true_val }); + Some(fun.push_insn(block, hir::Insn::BoxBool { val: not_equal })) +} + +fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if let Some(result) = try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumNeq { left, right }, BOP_NEQ, recv, other, state) { + return Some(result); + } + + if let Some(result) = try_inline_string_not_equal(fun, block, recv, other, state) { + return Some(result); + } + + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_class_allocate(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + + // Inline only in the case we have a leaf allocator + fun.try_inline_object_alloc(block, recv, state) +} + +fn inline_basic_object_initialize(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qnil) }); + Some(result) +} + +fn inline_nilclass_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) })) +} + +fn inline_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_kernel_is_a_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(other, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: recv, class: other }); + return Some(result); + } + None +} + +fn inline_kernel_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) +} + +fn inline_kernel_respond_to_p( + fun: &mut hir::Function, + block: hir::BlockId, + recv: hir::InsnId, + args: &[hir::InsnId], + state: hir::InsnId, +) -> Option<hir::InsnId> { + // Parse arguments: respond_to?(method_name, allow_priv = false) + let (method_name, allow_priv) = match *args { + [method_name] => (method_name, false), + [method_name, arg] => match fun.type_of(arg) { + t if t.is_known_truthy() => (method_name, true), + t if t.is_known_falsy() => (method_name, false), + // Unknown type; bail out + _ => return None, + }, + // Unknown args; bail out + _ => return None, + }; + + // Method name must be a static symbol + let method_name = fun.type_of(method_name).ruby_object()?; + if !method_name.static_sym_p() { + return None; + } + + // The receiver must have a known class to call `respond_to?` on + // TODO: This is technically overly strict. This would also work if all of the + // observed objects at this point agree on `respond_to?` and we can add many patchpoints. + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // Get the method ID and its corresponding callable method entry + let mid = unsafe { rb_sym2id(method_name) }; + let target_cme = unsafe { rb_callable_method_entry_or_negative(recv_class, mid) }; + assert!( + !target_cme.is_null(), + "Should never be null, as in that case we will be returned a \"negative CME\"" + ); + + let cme_def_type = unsafe { get_cme_def_type(target_cme) }; + + // Cannot inline a refined method, since their refinement depends on lexical scope + if cme_def_type == VM_METHOD_TYPE_REFINED { + return None; + } + + let visibility = match cme_def_type { + VM_METHOD_TYPE_UNDEF => METHOD_VISI_UNDEF, + _ => unsafe { METHOD_ENTRY_VISI(target_cme) }, + }; + + let result = match (visibility, allow_priv) { + // Method undefined; check `respond_to_missing?` + (METHOD_VISI_UNDEF, _) => { + let respond_to_missing = ID!(respond_to_missing); + if unsafe { rb_method_basic_definition_p(recv_class, respond_to_missing) } == 0 { + return None; // Custom definition of respond_to_missing?, so cannot inline + } + let respond_to_missing_cme = + unsafe { rb_callable_method_entry(recv_class, respond_to_missing) }; + // Protect against redefinition of `respond_to_missing?` + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::NoTracePoint, + state, + }, + ); + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: respond_to_missing, + cme: respond_to_missing_cme, + }, + state, + }, + ); + Qfalse + } + // Private method with allow priv=false, so `respond_to?` returns false + (METHOD_VISI_PRIVATE, false) => Qfalse, + // Public method or allow_priv=true: check if implemented + (METHOD_VISI_PUBLIC, _) | (_, true) => { + if cme_def_type == VM_METHOD_TYPE_NOTIMPLEMENTED { + // C method with rb_f_notimplement(). `respond_to?` returns false + // without consulting `respond_to_missing?`. See also: rb_add_method_cfunc() + Qfalse + } else { + Qtrue + } + } + (_, _) => return None, // not public and include_all not known, can't compile + }; + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + return None; + } + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::NoTracePoint, state }); + fun.push_insn(block, hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: mid, + cme: target_cme + }, state + }); + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(result) })) +} + +fn inline_kernel_class(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + let real_class = unsafe { rb_class_real(recv_class) }; + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(real_class) })) +} + +/// Inline `fixnum_inc(ec, self, num)` implies FixnumAdd(num, 1). +/// num is always a Fixnum (starts at 0 and is incremented by fixnum_inc). +fn inline_fixnum_inc(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[_self, num] = args else { return None; }; + let one = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left: num, right: one, state }); + Some(result) +} + +/// Inline `ary_at(ec, self, index)` implies ArrayAref. +/// Called from Array#each etc. where self is Array and index is in bounds. +fn inline_ary_at(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv, index] = args else { return None; }; + let recv = fun.push_insn(block, hir::Insn::RefineType { val: recv, new_type: types::Array }); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + Some(result) +} + +/// Inline `ary_at_end(ec, self, index)` implies index >= ArrayLength(self). +/// Called from Array#each etc. where self is Array and index is Fixnum. +fn inline_ary_at_end(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv, index] = args else { return None; }; + let recv = fun.push_insn(block, hir::Insn::RefineType { val: recv, new_type: types::Array }); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let length = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + let result = fun.push_insn(block, hir::Insn::FixnumGe { left: index, right: length }); + Some(result) +} diff --git a/zjit/src/disasm.rs b/zjit/src/disasm.rs new file mode 100644 index 0000000000..36bb90cff7 --- /dev/null +++ b/zjit/src/disasm.rs @@ -0,0 +1,72 @@ +use crate::{asm::CodeBlock, options::DumpDisasm, virtualmem::CodePtr}; + +/// Dump disassembly for a range in a [CodeBlock]. +pub fn dump_disasm_addr_range(cb: &CodeBlock, start_addr: CodePtr, end_addr: CodePtr, dump_disasm: &DumpDisasm) { + let disasm = disasm_addr_range(cb, start_addr.raw_ptr(cb) as usize, end_addr.raw_ptr(cb) as usize); + if disasm.is_empty() { + return; + } + + match dump_disasm { + DumpDisasm::Stdout => println!("{disasm}"), + DumpDisasm::File(fd) => { + use std::io::Write; + use std::os::unix::io::{FromRawFd, IntoRawFd}; + + let mut file = unsafe { std::fs::File::from_raw_fd(*fd) }; + file.write_all(disasm.as_bytes()).unwrap(); + let _ = file.into_raw_fd(); + } + } +} + +pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> String { + use std::fmt::Write; + + let mut out = String::from(""); + + // Initialize capstone + use capstone::prelude::*; + + #[cfg(target_arch = "x86_64")] + let mut cs = Capstone::new() + .x86() + .mode(arch::x86::ArchMode::Mode64) + .syntax(arch::x86::ArchSyntax::Intel) + .build() + .unwrap(); + #[cfg(target_arch = "aarch64")] + let mut cs = Capstone::new() + .arm64() + .mode(arch::arm64::ArchMode::Arm) + .detail(true) + .build() + .unwrap(); + + cs.set_skipdata(true).unwrap(); + + // Disassemble the instructions + let code_size = end_addr - start_addr; + let code_slice = unsafe { std::slice::from_raw_parts(start_addr as _, code_size) }; + // Stabilize output for cargo test + #[cfg(test)] + let start_addr = 0; + let insns = cs.disasm_all(code_slice, start_addr as u64).unwrap(); + + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + + // For each instruction in this block + for insn in insns.as_ref() { + // Comments for this block + if let Some(comment_list) = cb.comments_at(insn.address() as usize) { + for comment in comment_list { + writeln!(&mut out, " {bold_begin}# {comment}{bold_end}").unwrap(); + } + } + writeln!(&mut out, " {}", format!("{insn}").trim()).unwrap(); + } + + out +} diff --git a/zjit/src/distribution.rs b/zjit/src/distribution.rs new file mode 100644 index 0000000000..aa4667b939 --- /dev/null +++ b/zjit/src/distribution.rs @@ -0,0 +1,282 @@ +//! Type frequency distribution tracker. + +use crate::options::NumProfiles; + +/// This implementation was inspired by the type feedback module from Google's S6, which was +/// written in C++ for use with Python. This is a new implementation in Rust created for use with +/// Ruby instead of Python. +#[derive(Debug, Clone)] +pub struct Distribution<T: Copy + PartialEq + Default, const N: usize> { + /// buckets and counts have the same length + /// `buckets[0]` is always the most common item + buckets: [T; N], + counts: [NumProfiles; N], + /// if there is no more room, increment the fallback + other: NumProfiles, + // TODO(max): Add count disparity, which can help determine when to reset the distribution +} + +impl<T: Copy + PartialEq + Default, const N: usize> Distribution<T, N> { + pub fn new() -> Self { + Self { buckets: [Default::default(); N], counts: [0; N], other: 0 } + } + + pub fn observe(&mut self, item: T) { + for (bucket, count) in self.buckets.iter_mut().zip(self.counts.iter_mut()) { + if *bucket == item || *count == 0 { + *bucket = item; + *count = count.saturating_add(1); + // Keep the most frequent item at the front + self.bubble_up(); + return; + } + } + self.other = self.other.saturating_add(1); + } + + /// Keep the highest counted bucket at index 0 + fn bubble_up(&mut self) { + if N == 0 { return; } + let max_index = self.counts.into_iter().enumerate().max_by_key(|(_, val)| *val).unwrap().0; + if max_index != 0 { + self.counts.swap(0, max_index); + self.buckets.swap(0, max_index); + } + } + + pub fn each_item(&self) -> impl Iterator<Item = T> + '_ { + self.buckets.iter().zip(self.counts.iter()) + .filter_map(|(&bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } + + pub fn each_item_mut(&mut self) -> impl Iterator<Item = &mut T> + '_ { + self.buckets.iter_mut().zip(self.counts.iter()) + .filter_map(|(bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } +} + +#[derive(PartialEq, Debug, Clone, Copy)] +enum DistributionKind { + /// No types seen + Empty, + /// One type seen + Monomorphic, + /// Between 2 and (fixed) N types seen + Polymorphic, + /// Polymorphic, but with a significant skew towards one type + SkewedPolymorphic, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic, +} + +#[derive(Debug, Clone)] +pub struct DistributionSummary<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> { + kind: DistributionKind, + buckets: [T; N], + // TODO(max): Determine if we need some notion of stability +} + +const SKEW_THRESHOLD: f64 = 0.75; + +impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> DistributionSummary<T, N> { + pub fn new(dist: &Distribution<T, N>) -> Self { + #[cfg(debug_assertions)] + { + let first_count = dist.counts[0]; + for &count in &dist.counts[1..] { + assert!(first_count >= count, "First count should be the largest"); + } + } + let num_seen = dist.counts.iter().map(|&c| usize::from(c)).sum::<usize>() + usize::from(dist.other); + let kind = if dist.other == 0 { + // Seen <= N types total + if dist.counts[0] == 0 { + DistributionKind::Empty + } else if dist.counts[1] == 0 { + DistributionKind::Monomorphic + } else if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedPolymorphic + } else { + DistributionKind::Polymorphic + } + } else { + // Seen > N types total; considered megamorphic + if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedMegamorphic + } else { + DistributionKind::Megamorphic + } + }; + Self { kind, buckets: dist.buckets } + } + + pub fn is_monomorphic(&self) -> bool { + self.kind == DistributionKind::Monomorphic + } + + pub fn is_polymorphic(&self) -> bool { + self.kind == DistributionKind::Polymorphic + } + + pub fn is_skewed_polymorphic(&self) -> bool { + self.kind == DistributionKind::SkewedPolymorphic + } + + pub fn is_megamorphic(&self) -> bool { + self.kind == DistributionKind::Megamorphic + } + + pub fn is_skewed_megamorphic(&self) -> bool { + self.kind == DistributionKind::SkewedMegamorphic + } + + pub fn bucket(&self, idx: usize) -> T { + assert!(idx < N, "index {idx} out of bounds for buckets[{N}]"); + self.buckets[idx] + } + + pub fn buckets(&self) -> &[T] { + &self.buckets + } +} + +#[cfg(test)] +mod distribution_tests { + use super::*; + + #[test] + fn start_empty() { + let dist = Distribution::<usize, 4>::new(); + assert_eq!(dist.other, 0); + assert!(dist.counts.iter().all(|&b| b == 0)); + } + + #[test] + fn observe_adds_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 1); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_increments_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_two() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + assert_eq!(dist.buckets[0], 11); + assert_eq!(dist.counts[0], 3); + assert_eq!(dist.buckets[1], 10); + assert_eq!(dist.counts[1], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_with_max_increments_other() { + let mut dist = Distribution::<usize, 0>::new(); + dist.observe(10); + assert!(dist.buckets.is_empty()); + assert!(dist.counts.is_empty()); + assert_eq!(dist.other, 1); + } + + #[test] + fn empty_distribution_returns_empty_summary() { + let dist = Distribution::<usize, 4>::new(); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Empty); + } + + #[test] + fn monomorphic_distribution_returns_monomorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Monomorphic); + assert_eq!(summary.buckets[0], 10); + } + + #[test] + fn polymorphic_distribution_returns_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Polymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn skewed_polymorphic_distribution_returns_skewed_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedPolymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn megamorphic_distribution_returns_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(12); + dist.observe(13); + dist.observe(14); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Megamorphic); + assert_eq!(summary.buckets[0], 11); + } + + #[test] + fn skewed_megamorphic_distribution_returns_skewed_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(13); + dist.observe(14); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedMegamorphic); + assert_eq!(summary.buckets[0], 12); + } +} diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs new file mode 100644 index 0000000000..7f5bc7891f --- /dev/null +++ b/zjit/src/gc.rs @@ -0,0 +1,244 @@ +//! This module is responsible for marking/moving objects on GC. + +use std::ptr::null; +use std::{ffi::c_void, ops::Range}; +use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; +use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; +use crate::stats::Counter::gc_time_ns; + +/// GC callback for marking GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_mark(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to mark + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &*(payload as *const IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_mark(payload)); +} + +/// GC callback for updating GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to update + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &mut *(payload as *mut IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_update_references(payload)); +} + +/// GC callback for finalizing an ISEQ +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_free(iseq: IseqPtr) { + if !ZJITState::has_instance() { + return; + } + + // TODO(Shopify/ruby#682): Free `IseqPayload` + let payload = get_or_create_iseq_payload(iseq); + for version in payload.versions.iter_mut() { + unsafe { version.as_mut() }.iseq = null(); + } + + let invariants = ZJITState::get_invariants(); + invariants.forget_iseq(iseq); +} + +/// GC callback for finalizing a CME +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_free(cme: *const rb_callable_method_entry_struct) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_cme(cme); +} + +/// GC callback for finalizing a class +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_klass_free(klass: VALUE) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_klass(klass); +} + +/// GC callback for updating object references after all object moves +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_update_references() { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.update_references(); + + // Update iseq pointers in all JITFrames for GC compaction. + // rb_execution_context_update only updates JITFrames currently on the stack, + // but JITFrames not on the stack also need their iseq pointers updated + // because the JIT code will reuse them on the next call. + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &mut *jit_frame }.update_references(); + } +} + +fn iseq_mark(payload: &IseqPayload) { + // Mark objects retained by profiling instructions + payload.profile.each_object(|object| { + unsafe { rb_gc_mark_movable(object); } + }); + + // Mark objects baked in JIT code + let cb = ZJITState::get_code_block(); + for version in payload.versions.iter() { + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + unsafe { + let object = value_ptr.read_unaligned(); + rb_gc_mark_movable(object); + } + } + } +} + +/// This is a mirror of [iseq_mark]. +fn iseq_update_references(payload: &mut IseqPayload) { + // Move objects retained by profiling instructions + payload.profile.each_object_mut(|old_object| { + let new_object = unsafe { rb_gc_location(*old_object) }; + if *old_object != new_object { + *old_object = new_object; + } + }); + + for &version in payload.versions.iter() { + iseq_version_update_references(version); + } +} + +fn iseq_version_update_references(mut version: IseqVersionRef) { + // Move ISEQ in the payload + unsafe { version.as_mut() }.iseq = unsafe { rb_gc_location(version.as_ref().iseq.into()) }.as_iseq(); + + // Move ISEQ references in incoming IseqCalls + for iseq_call in unsafe { version.as_mut() }.incoming.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move ISEQ references in outgoing IseqCalls + for iseq_call in unsafe { version.as_mut() }.outgoing.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move objects baked in JIT code. + // The code region is already writable because rb_zjit_mark_all_writable() was called + // before the GC update_references phase. We write directly to avoid per-page mprotect calls. + let cb = ZJITState::get_code_block(); + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + let object = unsafe { value_ptr.read_unaligned() }; + let new_addr = unsafe { rb_gc_location(object) }; + + // Only write when the VALUE moves, to be copy-on-write friendly. + if new_addr != object { + let value_ptr = value_ptr as *mut VALUE; + unsafe { value_ptr.write_unaligned(new_addr) }; + } + } +} + +/// Append a set of gc_offsets to the iseq's payload +pub fn append_gc_offsets(iseq: IseqPtr, mut version: IseqVersionRef, offsets: &Vec<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.extend(offsets); + + // Call writebarrier on each newly added value + let cb = ZJITState::get_code_block(); + for &offset in offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + let value_ptr = value_ptr as *const VALUE; + unsafe { + let object = value_ptr.read_unaligned(); + VALUE::from(iseq).write_barrier(object); + } + } +} + +/// Remove GC offsets that overlap with a given removed_range. +/// We do this when invalidation rewrites some code with a jump instruction +/// and GC offsets are corrupted by the rewrite, assuming no on-stack code +/// will step into the instruction with the GC offsets after invalidation. +pub fn remove_gc_offsets(mut version: IseqVersionRef, removed_range: &Range<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.retain(|&gc_offset| { + let offset_range = gc_offset..(gc_offset.add_bytes(SIZEOF_VALUE)); + !ranges_overlap(&offset_range, removed_range) + }); +} + +/// Return true if given `Range<CodePtr>` ranges overlap with each other +fn ranges_overlap<T>(left: &Range<T>, right: &Range<T>) -> bool where T: PartialOrd { + left.start < right.end && right.start < left.end +} + +/// GC callback for making all JIT code writable before updating references in bulk. +/// This avoids toggling W^X permissions per-page during GC compaction. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_mark_all_writable() { + if !ZJITState::has_instance() { + return; + } + ZJITState::get_code_block().mark_all_writable(); +} + +/// GC callback for making all JIT code executable after updating references in bulk. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_mark_all_executable() { + if !ZJITState::has_instance() { + return; + } + ZJITState::get_code_block().mark_all_executable(); +} + +/// Callback for marking GC objects inside [crate::invariants::Invariants]. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_mark() { + // Mark iseq pointers in all JITFrames. JITFrames that are currently on the + // stack are also marked via rb_execution_context_mark, but JITFrames not on + // the stack still need their iseqs kept alive because JIT code will reuse them. + if !ZJITState::has_instance() { + return; + } + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &*jit_frame }.mark(); + } +} diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs new file mode 100644 index 0000000000..6911129ad3 --- /dev/null +++ b/zjit/src/hir.rs @@ -0,0 +1,9358 @@ +//! High-level intermediary representation (IR) in static single-assignment (SSA) form. + +// We use the YARV bytecode constants which have a CRuby-style name +#![allow(non_upper_case_globals)] + +#![allow(clippy::if_same_then_else)] +#![allow(clippy::match_like_matches_macro)] +use crate::{ + backend::lir::C_ARG_OPNDS, + cast::IntoUsize, codegen::{local_idx_to_ep_offset, max_iseq_versions}, cruby::*, invariants::{self}, payload::{get_or_create_iseq_payload, IseqPayload}, options::{debug, get_option, DumpHIR}, state::ZJITState, json::Json, + state, +}; +use std::{ + cell::RefCell, collections::{HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, c_int, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter, + sync::atomic::Ordering, +}; +use crate::hir_type::{Type, types}; +use crate::hir_effect::{Effect, abstract_heaps, effects}; +use crate::bitset::BitSet; +use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::stats::Counter; +use SendFallbackReason::*; + +pub(crate) mod tests; +mod opt_tests; + +/// An index of an [`Insn`] in a [`Function`]. This is a popular +/// type since this effectively acts as a pointer to an [`Insn`]. +/// See also: [`Function::find`]. +#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Debug)] +pub struct InsnId(pub usize); + +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 + } +} + +impl std::fmt::Display for InsnId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "v{}", self.0) + } +} + +/// The index of a [`Block`], which effectively acts like a pointer. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "bb{}", self.0) + } +} + +type InsnSet = BitSet<InsnId>; +type BlockSet = BitSet<BlockId>; + +fn write_vec<T: std::fmt::Display>(f: &mut std::fmt::Formatter, objs: &Vec<T>) -> std::fmt::Result { + write!(f, "[")?; + let mut prefix = ""; + for obj in objs { + write!(f, "{prefix}{obj}")?; + prefix = ", "; + } + write!(f, "]") +} + +impl std::fmt::Display for VALUE { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl VALUE { + pub fn print(self, ptr_map: &PtrPrintMap) -> VALUEPrinter<'_> { + VALUEPrinter { inner: self, ptr_map } + } +} + +/// Print adaptor for [`VALUE`]. See [`PtrPrintMap`]. +pub struct VALUEPrinter<'a> { + inner: VALUE, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for VALUEPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + val if val.fixnum_p() => write!(f, "{}", val.as_fixnum()), + Qnil => write!(f, "nil"), + Qtrue => write!(f, "true"), + Qfalse => write!(f, "false"), + val => write!(f, "VALUE({:p})", self.ptr_map.map_ptr(val.as_ptr::<VALUE>())), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<InsnId>, +} + +impl std::fmt::Display for BranchEdge { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}(", self.target)?; + let mut prefix = ""; + for arg in &self.args { + write!(f, "{prefix}{arg}")?; + prefix = ", "; + } + write!(f, ")") + } +} + +/// Invalidation reasons +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Invariant { + /// Basic operation is redefined + BOPRedefined { + /// {klass}_REDEFINED_OP_FLAG + klass: RedefinitionFlag, + /// BOP_{bop} + bop: ruby_basic_operators, + }, + MethodRedefined { + /// The class object whose method we want to assume unchanged + klass: VALUE, + /// The method ID of the method we want to assume unchanged + method: ID, + /// The callable method entry that we want to track + cme: *const rb_callable_method_entry_t, + }, + /// A list of constant expression path segments that must have not been written to for the + /// following code to be valid. + StableConstantNames { + idlist: *const ID, + }, + /// TracePoint is not enabled. If TracePoint is enabled, this is invalidated. + NoTracePoint, + /// cfp->ep is not escaped to the heap on the ISEQ + NoEPEscape(IseqPtr), + /// There is one ractor running. If a non-root ractor gets spawned, this is invalidated. + SingleRactorMode, + /// Objects of this class have no singleton class. + /// When a singleton class is created for an object of this class, this is invalidated. + NoSingletonClass { + klass: VALUE, + }, + /// Only the root box is active, so we can safely read from the prime classext. + /// Invalidated if a non-root box duplicates any classext. + RootBoxOnly, +} + +impl Invariant { + pub fn print(self, ptr_map: &PtrPrintMap) -> InvariantPrinter<'_> { + InvariantPrinter { inner: self, ptr_map } + } +} + +impl Display for Invariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialObjectType { + VMCore = 1, + CBase = 2, + ConstBase = 3, +} + +impl From<u32> for SpecialObjectType { + fn from(value: u32) -> Self { + match value { + VM_SPECIAL_OBJECT_VMCORE => SpecialObjectType::VMCore, + VM_SPECIAL_OBJECT_CBASE => SpecialObjectType::CBase, + VM_SPECIAL_OBJECT_CONST_BASE => SpecialObjectType::ConstBase, + _ => panic!("Invalid special object type: {value}"), + } + } +} + +impl From<SpecialObjectType> for u64 { + fn from(special_type: SpecialObjectType) -> Self { + special_type as u64 + } +} + +impl std::fmt::Display for SpecialObjectType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SpecialObjectType::VMCore => write!(f, "VMCore"), + SpecialObjectType::CBase => write!(f, "CBase"), + SpecialObjectType::ConstBase => write!(f, "ConstBase"), + } + } +} + +/// Print adaptor for [`Invariant`]. See [`PtrPrintMap`]. +pub struct InvariantPrinter<'a> { + inner: Invariant, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for InvariantPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + Invariant::BOPRedefined { klass, bop } => { + write!(f, "BOPRedefined(")?; + match klass { + INTEGER_REDEFINED_OP_FLAG => write!(f, "INTEGER_REDEFINED_OP_FLAG")?, + STRING_REDEFINED_OP_FLAG => write!(f, "STRING_REDEFINED_OP_FLAG")?, + ARRAY_REDEFINED_OP_FLAG => write!(f, "ARRAY_REDEFINED_OP_FLAG")?, + HASH_REDEFINED_OP_FLAG => write!(f, "HASH_REDEFINED_OP_FLAG")?, + _ => write!(f, "{klass}")?, + } + write!(f, ", ")?; + match bop { + BOP_PLUS => write!(f, "BOP_PLUS")?, + BOP_MINUS => write!(f, "BOP_MINUS")?, + BOP_MULT => write!(f, "BOP_MULT")?, + BOP_DIV => write!(f, "BOP_DIV")?, + BOP_MOD => write!(f, "BOP_MOD")?, + BOP_EQ => write!(f, "BOP_EQ")?, + BOP_EQQ => write!(f, "BOP_EQQ")?, + BOP_LT => write!(f, "BOP_LT")?, + BOP_LE => write!(f, "BOP_LE")?, + BOP_LTLT => write!(f, "BOP_LTLT")?, + BOP_AREF => write!(f, "BOP_AREF")?, + BOP_ASET => write!(f, "BOP_ASET")?, + BOP_LENGTH => write!(f, "BOP_LENGTH")?, + BOP_SIZE => write!(f, "BOP_SIZE")?, + BOP_EMPTY_P => write!(f, "BOP_EMPTY_P")?, + BOP_NIL_P => write!(f, "BOP_NIL_P")?, + BOP_SUCC => write!(f, "BOP_SUCC")?, + BOP_GT => write!(f, "BOP_GT")?, + BOP_GE => write!(f, "BOP_GE")?, + BOP_NOT => write!(f, "BOP_NOT")?, + BOP_NEQ => write!(f, "BOP_NEQ")?, + BOP_MATCH => write!(f, "BOP_MATCH")?, + BOP_FREEZE => write!(f, "BOP_FREEZE")?, + BOP_UMINUS => write!(f, "BOP_UMINUS")?, + BOP_MAX => write!(f, "BOP_MAX")?, + BOP_MIN => write!(f, "BOP_MIN")?, + BOP_HASH => write!(f, "BOP_HASH")?, + BOP_CALL => write!(f, "BOP_CALL")?, + BOP_AND => write!(f, "BOP_AND")?, + BOP_OR => write!(f, "BOP_OR")?, + BOP_CMP => write!(f, "BOP_CMP")?, + BOP_DEFAULT => write!(f, "BOP_DEFAULT")?, + BOP_PACK => write!(f, "BOP_PACK")?, + BOP_INCLUDE_P => write!(f, "BOP_INCLUDE_P")?, + _ => write!(f, "{bop}")?, + } + write!(f, ")") + } + Invariant::MethodRedefined { klass, method, cme } => { + let class_name = get_class_name(klass); + write!(f, "MethodRedefined({class_name}@{:p}, {}@{:p}, cme:{:p})", + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>()), + method.contents_lossy(), + self.ptr_map.map_id(method.0), + self.ptr_map.map_ptr(cme) + ) + } + Invariant::StableConstantNames { idlist } => { + write!(f, "StableConstantNames({:p}, ", self.ptr_map.map_ptr(idlist))?; + let mut idx = 0; + let mut sep = ""; + loop { + let id = unsafe { *idlist.wrapping_add(idx) }; + if id.0 == 0 { + break; + } + write!(f, "{sep}{}", id.contents_lossy())?; + sep = "::"; + idx += 1; + } + write!(f, ")") + } + Invariant::NoTracePoint => write!(f, "NoTracePoint"), + Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)), + Invariant::SingleRactorMode => write!(f, "SingleRactorMode"), + Invariant::NoSingletonClass { klass } => { + let class_name = get_class_name(klass); + write!(f, "NoSingletonClass({}@{:p})", + class_name, + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>())) + } + Invariant::RootBoxOnly => write!(f, "RootBoxOnly"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Copy)] +pub enum Const { + Value(VALUE), + CBool(bool), + CInt8(i8), + CInt16(i16), + CInt32(i32), + CInt64(i64), + CUInt8(u8), + CUInt16(u16), + CUInt32(u32), + CAttrIndex(attr_index_t), + CShape(ShapeId), + CUInt64(u64), + CPtr(*const u8), + CDouble(f64), +} + +impl std::fmt::Display for Const { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl Const { + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { + ConstPrinter { inner: self, ptr_map } + } +} + +#[derive(Clone, Copy)] +pub enum RangeType { + Inclusive = 0, // include the end value + Exclusive = 1, // exclude the end value +} + +impl std::fmt::Display for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", match self { + RangeType::Inclusive => "NewRangeInclusive", + RangeType::Exclusive => "NewRangeExclusive", + }) + } +} + +impl std::fmt::Debug for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{self}") + } +} + +impl From<u32> for RangeType { + fn from(flag: u32) -> Self { + match flag { + 0 => RangeType::Inclusive, + 1 => RangeType::Exclusive, + _ => panic!("Invalid range flag: {flag}"), + } + } +} + +/// Special regex backref symbol types +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialBackrefSymbol { + LastMatch, // $& + PreMatch, // $` + PostMatch, // $' + LastGroup, // $+ +} + +impl TryFrom<u8> for SpecialBackrefSymbol { + type Error = String; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value as char { + '&' => Ok(SpecialBackrefSymbol::LastMatch), + '`' => Ok(SpecialBackrefSymbol::PreMatch), + '\'' => Ok(SpecialBackrefSymbol::PostMatch), + '+' => Ok(SpecialBackrefSymbol::LastGroup), + c => Err(format!("invalid backref symbol: '{c}'")), + } + } +} + +/// Print adaptor for [`Const`]. See [`PtrPrintMap`]. +pub struct ConstPrinter<'a> { + inner: &'a Const, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for ConstPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + Const::Value(val) => write!(f, "Value({})", val.print(self.ptr_map)), + // Since `&` coerces to a raw pointer, be careful to get `val` and not `&val` here. + &Const::CPtr(val) => write!(f, "CPtr({:p})", self.ptr_map.map_ptr(val)), + &Const::CShape(shape_id) => write!(f, "CShape({:p})", self.ptr_map.map_shape(shape_id)), + &Const::CUInt64(int) => { + // Print in hex if signed bit is set + if 0 != int & (1 << (u64::BITS - 1)) { + write!(f, "CUInt64(0x{int:x})") + } else { + write!(f, "CUInt64({int})") + } + } + _ => write!(f, "{:?}", self.inner), + } + } +} + +/// For output stability in tests, we assign each pointer with a stable +/// address the first time we see it. This mapping is off by default; +/// set [`PtrPrintMap::map_ptrs`] to switch it on. +/// +/// Because this is extra state external to any pointer being printed, a +/// printing adapter struct that wraps the pointer along with this map is +/// required to make use of this effectively. The [`std::fmt::Display`] +/// implementation on the adapter struct can then be reused to implement +/// `Display` on the inner type with a default [`PtrPrintMap`], which +/// does not perform any mapping. +pub struct PtrPrintMap { + inner: RefCell<PtrPrintMapInner>, + map_ptrs: bool, +} + +struct PtrPrintMapInner { + map: HashMap<*const c_void, *const c_void>, + next_ptr: *const c_void, +} + +impl PtrPrintMap { + /// Return a mapper that maps the pointer to itself. + pub fn identity() -> Self { + Self { + map_ptrs: false, + inner: RefCell::new(PtrPrintMapInner { + map: HashMap::default(), next_ptr: + ptr::without_provenance(0x1000) // Simulate 4 KiB zero page + }) + } + } +} + +struct Offset(i32); + +impl std::fmt::LowerHex for Offset { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let prefix = if f.alternate() { "0x" } else { "" }; + let bare_hex = format!("{:x}", self.0.abs()); + f.pad_integral(self.0 >= 0, prefix, &bare_hex) + } +} + +impl PtrPrintMap { + /// Map a pointer for printing + pub fn map_ptr<T>(&self, ptr: *const T) -> *const T { + // When testing, address stability is not a concern so print real address to enable code + // reuse + if !self.map_ptrs { + return ptr; + } + + use std::collections::hash_map::Entry::*; + let ptr = ptr.cast(); + let inner = &mut *self.inner.borrow_mut(); + match inner.map.entry(ptr) { + Occupied(entry) => entry.get().cast(), + Vacant(entry) => { + // Pick a fake address that is suitably aligns for T and remember it in the map + let mapped = inner.next_ptr.wrapping_add(inner.next_ptr.align_offset(align_of::<T>())); + entry.insert(mapped); + + // Bump for the next pointer + inner.next_ptr = mapped.wrapping_add(size_of::<T>()); + mapped.cast() + } + } + } + + /// Map a Ruby ID (index into intern table) for printing + fn map_id(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + /// Map an index into a Ruby object (e.g. for an ivar) for printing + fn map_index(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + fn map_offset(&self, id: i32) -> Offset { + Offset(self.map_ptr(id as *const c_void) as i32) + } + + /// Map shape ID into a pointer for printing + pub fn map_shape(&self, id: ShapeId) -> *const c_void { + self.map_ptr(id.0 as *const c_void) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum SideExitReason { + UnhandledNewarraySend(vm_opt_newarray_send_type), + UnhandledDuparraySend(u64), + UnknownSpecialVariable(u64), + UnhandledHIRThrow, + UnhandledHIRInvokeBuiltin, + UnhandledHIRUnknown(InsnId), + UnhandledYARVInsn(u32), + UnhandledCallType(CallType), + UnhandledBlockArg, + TooManyKeywordParameters, + FixnumAddOverflow, + FixnumSubOverflow, + FixnumMultOverflow, + FixnumLShiftOverflow, + GuardType(Type), + GuardShape(ShapeId), + ExpandArray, + GuardNotFrozen, + GuardNotShared, + GuardLess, + GuardGreaterEq, + GuardSuperMethodEntry, + PatchPoint(Invariant), + CalleeSideExit, + ObjToStringFallback, + Interrupt, + BlockParamProxyNotIseqOrIfunc, + BlockParamProxyNotNil, + BlockParamProxyFallbackMiss, + BlockParamProxyProfileNotCovered, + BlockParamWbRequired, + StackOverflow, + FixnumModByZero, + FixnumDivByZero, + BoxFixnumOverflow, + SplatKwNotNilOrHash, + SplatKwPolymorphic, + SplatKwNotProfiled, + DirectiveInduced, + SendWhileTracing, + NoProfileSend, + InvokeBlockNotIfunc, +} + +/// Controls how a side exit triggers recompilation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Recompile { + /// Profile receiver + arguments from the stack (for sends without profile data). + ProfileSend { argc: i32 }, + /// Profile self from the CFP (for shape guard failures). + ProfileSelf, +} + +#[derive(Debug, Clone, Copy)] +pub enum MethodType { + Iseq, + Cfunc, + Attrset, + Ivar, + Bmethod, + Zsuper, + Alias, + Undefined, + NotImplemented, + Optimized, + Missing, + Refined, + Null, +} + +impl From<u32> for MethodType { + fn from(value: u32) -> Self { + match value { + VM_METHOD_TYPE_ISEQ => MethodType::Iseq, + VM_METHOD_TYPE_CFUNC => MethodType::Cfunc, + VM_METHOD_TYPE_ATTRSET => MethodType::Attrset, + VM_METHOD_TYPE_IVAR => MethodType::Ivar, + VM_METHOD_TYPE_BMETHOD => MethodType::Bmethod, + VM_METHOD_TYPE_ZSUPER => MethodType::Zsuper, + VM_METHOD_TYPE_ALIAS => MethodType::Alias, + VM_METHOD_TYPE_UNDEF => MethodType::Undefined, + VM_METHOD_TYPE_NOTIMPLEMENTED => MethodType::NotImplemented, + VM_METHOD_TYPE_OPTIMIZED => MethodType::Optimized, + VM_METHOD_TYPE_MISSING => MethodType::Missing, + VM_METHOD_TYPE_REFINED => MethodType::Refined, + _ => unreachable!("unknown send_without_block def_type: {}", value), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum OptimizedMethodType { + Send, + Call, + BlockCall, + StructAref, + StructAset, +} + +impl From<u32> for OptimizedMethodType { + fn from(value: u32) -> Self { + match value { + OPTIMIZED_METHOD_TYPE_SEND => OptimizedMethodType::Send, + OPTIMIZED_METHOD_TYPE_CALL => OptimizedMethodType::Call, + OPTIMIZED_METHOD_TYPE_BLOCK_CALL => OptimizedMethodType::BlockCall, + OPTIMIZED_METHOD_TYPE_STRUCT_AREF => OptimizedMethodType::StructAref, + OPTIMIZED_METHOD_TYPE_STRUCT_ASET => OptimizedMethodType::StructAset, + _ => unreachable!("unknown send_without_block optimized method type: {}", value), + } + } +} + +impl std::fmt::Display for SideExitReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SideExitReason::UnhandledYARVInsn(opcode) => write!(f, "UnhandledYARVInsn({})", insn_name(*opcode as usize)), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MAX) => write!(f, "UnhandledNewarraySend(MAX)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MIN) => write!(f, "UnhandledNewarraySend(MIN)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_HASH) => write!(f, "UnhandledNewarraySend(HASH)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK) => write!(f, "UnhandledNewarraySend(PACK)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK_BUFFER) => write!(f, "UnhandledNewarraySend(PACK_BUFFER)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_INCLUDE_P) => write!(f, "UnhandledNewarraySend(INCLUDE_P)"), + SideExitReason::UnhandledDuparraySend(method_id) => write!(f, "UnhandledDuparraySend({method_id})"), + SideExitReason::GuardType(guard_type) => write!(f, "GuardType({guard_type})"), + SideExitReason::GuardNotShared => write!(f, "GuardNotShared"), + SideExitReason::PatchPoint(invariant) => write!(f, "PatchPoint({invariant})"), + _ => write!(f, "{self:?}"), + } + } +} + +/// Result of resolving the receiver type for method dispatch optimization. +/// Represents whether we know the receiver's class statically at compile-time, +/// have profiled type information, or know nothing about it. +pub enum ReceiverTypeResolution { + /// No profile information available for the receiver + NoProfile, + /// The receiver has a monomorphic profile (single type observed, guard needed) + Monomorphic { profiled_type: ProfiledType }, + /// The receiver is polymorphic (multiple types, none dominant) + Polymorphic, + /// The receiver has a skewed polymorphic profile (dominant type with some other types, guard needed) + SkewedPolymorphic { profiled_type: ProfiledType }, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic { profiled_type: ProfiledType }, + /// The receiver's class is statically known at JIT compile-time (no guard needed) + StaticallyKnown { class: VALUE }, +} + +/// Reason why a send-ish instruction cannot be optimized from a fallback instruction +#[derive(Debug, Clone, Copy)] +pub enum SendFallbackReason { + SendWithoutBlockPolymorphic, + SendWithoutBlockMegamorphic, + SendWithoutBlockNoProfiles, + SendWithoutBlockCfuncNotVariadic, + SendWithoutBlockCfuncArrayVariadic, + SendWithoutBlockNotOptimizedMethodType(MethodType), + SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType), + SendWithoutBlockNotOptimizedNeedPermission, + SendWithoutBlockBopRedefined, + SendWithoutBlockOperandsNotFixnum, + SendWithoutBlockPolymorphicFallback, + SendDirectKeywordMismatch, + SendDirectKeywordCountMismatch, + SendDirectMissingKeyword, + SendDirectTooManyKeywords, + SendPolymorphic, + SendMegamorphic, + SendNoProfiles, + SendCfuncVariadic, + SendCfuncArrayVariadic, + SendNotOptimizedMethodType(MethodType), + SendNotOptimizedNeedPermission, + CCallWithFrameTooManyArgs, + ObjToStringNotString, + TooManyArgsForLir, + /// The Proc object for a BMETHOD is not defined by an ISEQ. (See `enum rb_block_type`.) + BmethodNonIseqProc, + /// Caller supplies too few or too many arguments than what the callee's parameters expects. + ArgcParamMismatch, + /// The call has at least one feature on the caller or callee side that the optimizer does not + /// support. + ComplexArgPass, + /// Caller has keyword arguments but callee doesn't expect them; need to convert to hash. + UnexpectedKeywordArgs, + /// A singleton class has been seen for the receiver class, so we skip the optimization + /// to avoid an invalidation loop. + SingletonClassSeen, + /// The super call is passed a block that the optimizer does not support. + SuperCallWithBlock, + /// When the `super` is in a block, finding the running CME for guarding requires a loop. Not + /// supported for now. + SuperFromBlock, + /// The profiled super class cannot be found. + SuperClassNotFound, + /// The `super` call uses a complex argument pattern that the optimizer does not support. + SuperComplexArgsPass, + /// The cached target of a `super` call could not be found. + SuperTargetNotFound, + /// Attempted to specialize a `super` call that doesn't have profile data. + SuperNoProfiles, + /// Cannot optimize the `super` call due to the target method. + SuperNotOptimizedMethodType(MethodType), + /// The `super` call is polymorpic. + SuperPolymorphic, + /// The `super` target call uses a complex argument pattern that the optimizer does not support. + SuperTargetComplexArgsPass, + /// The `invokeblock` instruction is not yet optimized in `type_specialize`. + InvokeBlockNotSpecialized, + /// The `sendforward` instruction (argument forwarding `...`) is not yet optimized in + /// `type_specialize`. + SendForwardNotSpecialized, + /// The `invokesuperforward` instruction (super with forwarding `...`) is not yet optimized in + /// `type_specialize`. + InvokeSuperForwardNotSpecialized, + /// The single-ractor-mode assumption could not be made. + SingleRactorModeRequired, + /// Initial fallback reason for every instruction, which should be mutated to + /// a more actionable reason when an attempt to specialize the instruction fails. + Uncategorized(ruby_vminsn_type), +} + +impl Display for SendFallbackReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SendWithoutBlockPolymorphic => write!(f, "SendWithoutBlock: polymorphic call site"), + SendWithoutBlockMegamorphic => write!(f, "SendWithoutBlock: megamorphic call site"), + SendWithoutBlockNoProfiles => write!(f, "SendWithoutBlock: no profile data available"), + SendWithoutBlockCfuncNotVariadic => write!(f, "SendWithoutBlock: C function is not variadic"), + SendWithoutBlockCfuncArrayVariadic => write!(f, "SendWithoutBlock: C function expects array variadic"), + SendWithoutBlockNotOptimizedMethodType(method_type) => write!(f, "SendWithoutBlock: unsupported method type {:?}", method_type), + SendWithoutBlockNotOptimizedMethodTypeOptimized(opt_type) => write!(f, "SendWithoutBlock: unsupported optimized method type {:?}", opt_type), + SendWithoutBlockNotOptimizedNeedPermission => write!(f, "SendWithoutBlock: method private or protected and no FCALL"), + SendNotOptimizedNeedPermission => write!(f, "Send: method private or protected and no FCALL"), + SendWithoutBlockBopRedefined => write!(f, "SendWithoutBlock: basic operation was redefined"), + SendWithoutBlockOperandsNotFixnum => write!(f, "SendWithoutBlock: operands are not fixnums"), + SendWithoutBlockPolymorphicFallback => write!(f, "SendWithoutBlock: polymorphic fallback"), + SendDirectKeywordMismatch => write!(f, "SendDirect: keyword mismatch"), + SendDirectKeywordCountMismatch => write!(f, "SendDirect: keyword count mismatch"), + SendDirectMissingKeyword => write!(f, "SendDirect: missing keyword"), + SendDirectTooManyKeywords => write!(f, "SendDirect: too many keywords for fixnum bitmask"), + SendPolymorphic => write!(f, "Send: polymorphic call site"), + SendMegamorphic => write!(f, "Send: megamorphic call site"), + SendNoProfiles => write!(f, "Send: no profile data available"), + SendCfuncVariadic => write!(f, "Send: C function is variadic"), + SendCfuncArrayVariadic => write!(f, "Send: C function expects array variadic"), + SendNotOptimizedMethodType(method_type) => write!(f, "Send: unsupported method type {:?}", method_type), + CCallWithFrameTooManyArgs => write!(f, "CCallWithFrame: too many arguments"), + ObjToStringNotString => write!(f, "ObjToString: result is not a string"), + TooManyArgsForLir => write!(f, "Too many arguments for LIR"), + BmethodNonIseqProc => write!(f, "Bmethod: Proc object is not defined by an ISEQ"), + ArgcParamMismatch => write!(f, "Argument count does not match parameter count"), + ComplexArgPass => write!(f, "Complex argument passing"), + UnexpectedKeywordArgs => write!(f, "Unexpected Keyword Args"), + SingletonClassSeen => write!(f, "Singleton class previously created for receiver class"), + SuperFromBlock => write!(f, "super: call from within a block"), + SuperCallWithBlock => write!(f, "super: call made with a block"), + SuperClassNotFound => write!(f, "super: profiled class cannot be found"), + SuperComplexArgsPass => write!(f, "super: complex argument passing to `super` call"), + SuperNoProfiles => write!(f, "super: no profile data available"), + SuperNotOptimizedMethodType(method_type) => write!(f, "super: unsupported target method type {:?}", method_type), + SuperPolymorphic => write!(f, "super: polymorphic call site"), + SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), + SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + InvokeBlockNotSpecialized => write!(f, "InvokeBlock: not yet specialized"), + SendForwardNotSpecialized => write!(f, "SendForward: not yet specialized"), + InvokeSuperForwardNotSpecialized => write!(f, "InvokeSuperForward: not yet specialized"), + SingleRactorModeRequired => write!(f, "Single-ractor mode required"), + Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), + } + } +} + +/// How a block is passed to a send-like instruction. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum BlockHandler { + /// Literal block ISEQ (e.g. `foo { ... }`) + BlockIseq(IseqPtr), + /// Block arg passed via &proc (e.g. `foo(&block)`) + BlockArg, +} + +/// Identifier used by LoadField/StoreField/LoadArg for HIR dumps. Variants +/// without an associated value name internal VM fields that we used to intern +/// as CRuby IDs just to print them; the `Id` variant carries a real CRuby ID +/// (e.g. local variable, ivar, struct field name). +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FieldName { + VM_ENV_DATA_INDEX_ME_CREF, + VM_ENV_DATA_INDEX_SPECVAL, + VM_ENV_DATA_INDEX_FLAGS, + RBASIC_FLAGS, + shape_id, + as_heap, + fields_obj, + thread_ptr, + len, + SelfParam, + Id(ID), +} + +impl std::fmt::Display for FieldName { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use FieldName::*; + match self { + Id(id) if id_is_empty(*id) => f.write_str("<empty>"), + Id(id) => f.write_str(&id.contents_lossy()), + SelfParam => f.write_str("self"), + _ => write!(f, "{self:?}"), + } + } +} + +impl From<ID> for FieldName { + fn from(id: ID) -> Self { + FieldName::Id(id) + } +} + +/// An instruction in the SSA IR. The output of an instruction is referred to by the index of +/// the instruction ([`InsnId`]). SSA form enables this, and [`UnionFind`] ([`Function::find`]) +/// helps with editing. +#[derive(Debug, Clone)] +pub enum Insn { + Const { val: Const }, + /// SSA block parameter. Also used for function parameters in the function's entry block. + Param, + /// Load a function argument from the calling convention. + /// Used in JIT entry blocks. idx is the calling convention index, id is for display. + LoadArg { idx: u32, id: FieldName, val_type: Type }, + + /// Synthetic terminator for the entries superblock. Targets all entry blocks + /// so that CFG analyses see a single root. Not lowered to machine code. + Entries { targets: Vec<BlockId> }, + + StringCopy { val: InsnId, chilled: bool, state: InsnId }, + StringIntern { val: InsnId, state: InsnId }, + StringConcat { strings: Vec<InsnId>, state: InsnId }, + /// Call rb_str_getbyte with known-Fixnum index + StringGetbyte { string: InsnId, index: InsnId }, + StringSetbyteFixnum { string: InsnId, index: InsnId, value: InsnId }, + StringAppend { recv: InsnId, other: InsnId, state: InsnId }, + StringAppendCodepoint { recv: InsnId, other: InsnId, state: InsnId }, + StringEqual { left: InsnId, right: InsnId }, + + /// Combine count stack values into a regexp + ToRegexp { opt: usize, values: Vec<InsnId>, state: InsnId }, + + /// Put special object (VMCORE, CBASE, etc.) based on value_type + PutSpecialObject { value_type: SpecialObjectType, state: InsnId }, + + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. + ToArray { val: InsnId, state: InsnId }, + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. If we + /// called `to_a`, duplicate the returned array. + ToNewArray { val: InsnId, state: InsnId }, + NewArray { elements: Vec<InsnId>, state: InsnId }, + /// NewHash contains a vec of (key, value) pairs + NewHash { elements: Vec<InsnId>, state: InsnId }, + NewRange { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + NewRangeFixnum { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + ArrayDup { val: InsnId, state: InsnId }, + ArrayHash { elements: Vec<InsnId>, state: InsnId }, + ArrayMax { elements: Vec<InsnId>, state: InsnId }, + ArrayMin { elements: Vec<InsnId>, state: InsnId }, + ArrayInclude { elements: Vec<InsnId>, target: InsnId, state: InsnId }, + ArrayPackBuffer { elements: Vec<InsnId>, fmt: InsnId, buffer: Option<InsnId>, state: InsnId }, + DupArrayInclude { ary: VALUE, target: InsnId, state: InsnId }, + /// Extend `left` with the elements from `right`. `left` and `right` must both be `Array`. + ArrayExtend { left: InsnId, right: InsnId, state: InsnId }, + /// Push `val` onto `array`, where `array` is already `Array`. + ArrayPush { array: InsnId, val: InsnId, state: InsnId }, + ArrayAref { array: InsnId, index: InsnId }, + ArrayAset { array: InsnId, index: InsnId, val: InsnId }, + ArrayPop { array: InsnId, state: InsnId }, + /// Return the length of the array as a C `long` ([`types::CInt64`]) + ArrayLength { array: InsnId }, + /// Adjust potentially-negative index by the given length, returning the adjusted index. If + /// still negative, return a negative number, which indicates the index is still out-of-bounds. + AdjustBounds { index: InsnId, length: InsnId }, + + HashAref { hash: InsnId, key: InsnId, state: InsnId }, + HashAset { hash: InsnId, key: InsnId, val: InsnId, state: InsnId }, + HashDup { val: InsnId, state: InsnId }, + + /// Allocate an instance of the `val` object without calling `#initialize` on it. + /// This can: + /// * raise an exception if `val` is not a class + /// * run arbitrary code if `val` is a class with a custom allocator + ObjectAlloc { val: InsnId, state: InsnId }, + /// Allocate an instance of the `val` class without calling `#initialize` on it. + /// This requires that `class` has the default allocator (for example via `IsMethodCfunc`). + /// This won't raise or run arbitrary code because `class` has the default allocator. + ObjectAllocClass { class: VALUE, state: InsnId }, + + /// Check if the value is truthy and "return" a C boolean. In reality, we will likely fuse this + /// with IfTrue/IfFalse in the backend to generate jcc. + Test { val: InsnId }, + /// Return C `true` if `val` is `Qnil`, else `false`. + IsNil { val: InsnId }, + /// Return C `true` if `val`'s method on cd resolves to the cfunc. + IsMethodCfunc { val: InsnId, cd: *const rb_call_data, cfunc: *const u8, state: InsnId }, + /// Return C `true` if left == right + IsBitEqual { left: InsnId, right: InsnId }, + /// Return C `true` if left != right + IsBitNotEqual { left: InsnId, right: InsnId }, + /// Convert a C `bool` to a Ruby `Qtrue`/`Qfalse`. Same as `RBOOL` macro. + BoxBool { val: InsnId }, + /// Convert a C `long` to a Ruby `Fixnum`. Side exit on overflow. + BoxFixnum { val: InsnId, state: InsnId }, + UnboxFixnum { val: InsnId }, + FixnumAref { recv: InsnId, index: InsnId }, + // TODO(max): In iseq body types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. + // `lep_level` is the lexical distance from this insn's iseq up to its local_iseq, used only + // for the DEFINED_YIELD op_type to materialize the local EP inline. Zero for other op_types. + Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, lep_level: u32, state: InsnId }, + GetConstant { klass: InsnId, id: ID, allow_nil: InsnId, state: InsnId }, + GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, + /// Kernel#block_given? but without pushing a frame. Similar to [`Insn::Defined`] with + /// `DEFINED_YIELD` + IsBlockGiven { lep: InsnId }, + /// Test the bit at index of val, a Fixnum. + /// Return Qtrue if the bit is set, else Qfalse. + FixnumBitCheck { val: InsnId, index: u8 }, + /// Return Qtrue if `val` is an instance of `class`, else Qfalse. + /// Equivalent to `class_search_ancestor(CLASS_OF(val), class)`. + IsA { val: InsnId, class: InsnId }, + /// `case`/`when`/`rescue` match check for `pattern` against `target`. + CheckMatch { target: InsnId, pattern: InsnId, flag: u32, state: InsnId }, + + /// Get a global variable named `id` + GetGlobal { id: ID, state: InsnId }, + /// Set a global variable named `id` to `val` + SetGlobal { id: ID, val: InsnId, state: InsnId }, + + //NewObject? + /// Get an instance variable `id` from `self_val`, using the inline cache `ic` if present + GetIvar { self_val: InsnId, id: ID, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Set `self_val`'s instance variable `id` to `val`, using the inline cache `ic` if present + SetIvar { self_val: InsnId, id: ID, val: InsnId, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Check whether an instance variable exists on `self_val` + DefinedIvar { self_val: InsnId, id: ID, pushval: VALUE, state: InsnId }, + + /// Load cfp->pc + LoadPC, + /// Load EC + LoadEC, + /// Load SP + LoadSP, + /// Load cfp->self + LoadSelf, + LoadField { recv: InsnId, id: FieldName, offset: i32, return_type: Type }, + /// Write `val` at an offset of `recv`. + /// When writing a Ruby object to a Ruby object, one must use GuardNotFrozen (or equivalent) before and WriteBarrier after. + StoreField { recv: InsnId, id: FieldName, offset: i32, val: InsnId }, + WriteBarrier { recv: InsnId, val: InsnId }, + + /// Check whether VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set in the (already loaded) environment flags. + /// Returns CBool (0/1). + IsBlockParamModified { flags: InsnId }, + /// Get the block parameter as a Proc. + GetBlockParam { level: u32, ep_offset: u32, state: InsnId }, + /// Set a local variable in a higher scope or the heap + SetLocal { level: u32, ep_offset: u32, val: InsnId }, + GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, + GetSpecialNumber { nth: u64, state: InsnId }, + + /// Get a class variable `id` + GetClassVar { id: ID, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Set a class variable `id` to `val` + SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + + /// Get the EP at the given level from the current CFP. + GetEP { level: u32 }, + + /// Own a FrameState so that instructions can look up their dominating FrameState when + /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate + /// any code. + Snapshot { state: FrameState }, + + /// Unconditional jump + Jump(BranchEdge), + + /// Conditional branch + CondBranch { val: InsnId, if_true: BranchEdge, if_false: BranchEdge }, + + /// Call a C function without pushing a frame + /// `name` and `owner` are for printing purposes only + CCall { cfunc: *const u8, recv: InsnId, args: Vec<InsnId>, name: ID, owner: VALUE, return_type: Type, elidable: bool }, + + /// Call a C function that pushes a frame + CCallWithFrame { + cd: *const rb_call_data, // cd for falling back to Send + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + block: Option<BlockHandler>, + }, + + /// Call a variadic C function with signature: func(int argc, VALUE *argv, VALUE recv) + /// This handles frame setup, argv creation, and frame teardown all in one + CCallVariadic { + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + block: Option<BlockHandler>, + }, + + /// Un-optimized fallback implementation (dynamic dispatch) for send-ish instructions + /// Ignoring keyword arguments etc for now + Send { + recv: InsnId, + cd: *const rb_call_data, + block: Option<BlockHandler>, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + SendForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuper { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuperForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeBlock { + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + /// Optimized invokeblock for IFUNC block handlers. + /// Calls rb_vm_yield_with_cfunc directly instead of going through rb_vm_invokeblock. + InvokeBlockIfunc { + cd: *const rb_call_data, + block_handler: InsnId, + args: Vec<InsnId>, + state: InsnId, + }, + /// Call Proc#call optimized method type. + InvokeProc { + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + kw_splat: bool, + }, + + /// Optimized ISEQ call + SendDirect { + recv: InsnId, + cd: *const rb_call_data, + cme: *const rb_callable_method_entry_t, + iseq: IseqPtr, + args: Vec<InsnId>, + kw_bits: u32, + block: Option<BlockHandler>, + state: InsnId, + }, + + // Invoke a builtin function + InvokeBuiltin { + bf: rb_builtin_function, + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + leaf: bool, + return_type: Option<Type>, // None for unannotated builtins + }, + + /// Set up frame. Remember the address as the JIT entry for the insn_idx in `jit_entry_insns()[jit_entry_idx]`. + EntryPoint { jit_entry_idx: Option<usize> }, + /// Control flow instructions + Return { val: InsnId }, + /// Non-local control flow. See the throw YARV instruction + Throw { throw_state: u32, val: InsnId, state: InsnId }, + + /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, |, ^, << + FixnumAdd { left: InsnId, right: InsnId, state: InsnId }, + FixnumSub { left: InsnId, right: InsnId, state: InsnId }, + FixnumMult { left: InsnId, right: InsnId, state: InsnId }, + FixnumDiv { left: InsnId, right: InsnId, state: InsnId }, + FixnumMod { left: InsnId, right: InsnId, state: InsnId }, + FixnumEq { left: InsnId, right: InsnId }, + FixnumNeq { left: InsnId, right: InsnId }, + FixnumLt { left: InsnId, right: InsnId }, + FixnumLe { left: InsnId, right: InsnId }, + FixnumGt { left: InsnId, right: InsnId }, + FixnumGe { left: InsnId, right: InsnId }, + FixnumAnd { left: InsnId, right: InsnId }, + FixnumOr { left: InsnId, right: InsnId }, + FixnumXor { left: InsnId, right: InsnId }, + IntAnd { left: InsnId, right: InsnId }, + IntOr { left: InsnId, right: InsnId }, + FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, + FixnumRShift { left: InsnId, right: InsnId }, + + /// Float arithmetic: delegates to rb_float_plus/minus/mul/div with GC preparation + FloatAdd { recv: InsnId, other: InsnId, state: InsnId }, + FloatSub { recv: InsnId, other: InsnId, state: InsnId }, + FloatMul { recv: InsnId, other: InsnId, state: InsnId }, + FloatDiv { recv: InsnId, other: InsnId, state: InsnId }, + /// Float#to_i: truncate float to integer via rb_jit_flo_to_i + FloatToInt { recv: InsnId, state: InsnId }, + + // Distinct from `Send` with `mid:to_s` because does not have a patch point for String to_s being redefined + ObjToString { val: InsnId, cd: *const rb_call_data, state: InsnId }, + AnyToString { val: InsnId, str: InsnId, state: InsnId }, + + /// Refine the known type information of with additional type information. + /// Computes the intersection of the existing type and the new type. + RefineType { val: InsnId, new_type: Type }, + /// Return CBool[true] if val has type Type and CBool[false] otherwise. + HasType { val: InsnId, expected: Type }, + + /// Side-exit if val doesn't have the expected type. + GuardType { val: InsnId, guard_type: Type, state: InsnId }, + /// Side-exit if val is not the expected Const. + GuardBitEquals { val: InsnId, expected: Const, reason: SideExitReason, state: InsnId, recompile: Option<Recompile> }, + /// Side-exit if (val & mask) == 0 + GuardAnyBitSet { val: InsnId, mask: Const, mask_name: Option<ID>, reason: SideExitReason, state: InsnId }, + /// Side-exit if (val & mask) != 0 + GuardNoBitsSet { val: InsnId, mask: Const, mask_name: Option<ID>, reason: SideExitReason, state: InsnId }, + /// Side-exit if left is not greater than or equal to right (both operands are C long). + GuardGreaterEq { left: InsnId, right: InsnId, reason: SideExitReason, state: InsnId }, + /// Side-exit if left is not less than right (both operands are C long). + GuardLess { left: InsnId, right: InsnId, reason: SideExitReason, state: InsnId }, + + /// Generate no code (or padding if necessary) and insert a patch point + /// that can be rewritten to a side exit when the Invariant is broken. + PatchPoint { invariant: Invariant, state: InsnId }, + + /// Side-exit into the interpreter. + /// If recompile is not None, the side exit will profile and invalidate the ISEQ + /// so that it gets recompiled with the new profile data. + SideExit { state: InsnId, reason: SideExitReason, recompile: Option<Recompile> }, + + /// Increment a counter in ZJIT stats + IncrCounter(Counter), + + /// Increment a counter in ZJIT stats for the given counter pointer + IncrCounterPtr { counter_ptr: *mut u64 }, + + /// Equivalent of RUBY_VM_CHECK_INTS. Automatically inserted by the compiler before jumps and + /// return instructions. + CheckInterrupts { state: InsnId }, + + BreakPoint, + + /// Only use this instruction in tests where you need to end a block with + /// a terminator, but don't ever expect the code to be executed. This + /// instruction should never be generated from iseq_to_hir + Unreachable, +} + +/// Macro that enumerates all operands of an Insn, dispatching to caller-provided +/// `$visit_one` macro for a single InsnId field and `$visit_many` macro for a +/// slice/Vec of InsnIds. Used by both `for_each_operand` and `for_each_operand_mut`. +macro_rules! for_each_operand_impl { + ($self:expr, $visit_one:ident, $visit_many:ident) => { + match $self { + Insn::Const { .. } + | Insn::Param + | Insn::LoadArg { .. } + | Insn::Entries { .. } + | Insn::EntryPoint { .. } + | Insn::LoadPC + | Insn::LoadSP + | Insn::LoadEC + | Insn::GetEP { .. } + | Insn::LoadSelf + | Insn::BreakPoint | Insn::Unreachable + | Insn::IncrCounter(_) + | Insn::IncrCounterPtr { .. } => {} + + Insn::IsBlockGiven { lep } => { + $visit_one!(lep); + } + Insn::IsBlockParamModified { flags } => { + $visit_one!(flags); + } + Insn::CheckMatch { target, pattern, state, .. } => { + $visit_one!(target); + $visit_one!(pattern); + $visit_one!(state); + } + Insn::PatchPoint { state, .. } + | Insn::CheckInterrupts { state } + | Insn::PutSpecialObject { state, .. } + | Insn::GetBlockParam { state, .. } + | Insn::GetConstantPath { state, .. } => { + $visit_one!(state); + } + Insn::FixnumBitCheck { val, .. } => { + $visit_one!(val); + } + Insn::ArrayMax { elements, state, .. } + | Insn::ArrayMin { elements, state, .. } + | Insn::ArrayHash { elements, state, .. } + | Insn::NewHash { elements, state, .. } + | Insn::NewArray { elements, state, .. } => { + $visit_many!(elements); + $visit_one!(state); + } + Insn::ArrayInclude { elements, target, state, .. } => { + $visit_many!(elements); + $visit_one!(target); + $visit_one!(state); + } + Insn::ArrayPackBuffer { elements, fmt, buffer, state, .. } => { + $visit_many!(elements); + $visit_one!(fmt); + if let Some(buffer) = buffer { + $visit_one!(buffer); + } + $visit_one!(state); + } + Insn::DupArrayInclude { target, state, .. } => { + $visit_one!(target); + $visit_one!(state); + } + Insn::NewRange { low, high, state, .. } + | Insn::NewRangeFixnum { low, high, state, .. } => { + $visit_one!(low); + $visit_one!(high); + $visit_one!(state); + } + Insn::StringConcat { strings, state, .. } => { + $visit_many!(strings); + $visit_one!(state); + } + Insn::StringGetbyte { string, index } => { + $visit_one!(string); + $visit_one!(index); + } + Insn::StringSetbyteFixnum { string, index, value } => { + $visit_one!(string); + $visit_one!(index); + $visit_one!(value); + } + Insn::StringAppend { recv, other, state } + | Insn::StringAppendCodepoint { recv, other, state } => { + $visit_one!(recv); + $visit_one!(other); + $visit_one!(state); + } + Insn::StringEqual { left, right } => { + $visit_one!(left); + $visit_one!(right); + } + Insn::ToRegexp { values, state, .. } => { + $visit_many!(values); + $visit_one!(state); + } + Insn::RefineType { val, .. } + | Insn::HasType { val, .. } + | Insn::Return { val } + | Insn::Test { val } + | Insn::SetLocal { val, .. } + | Insn::BoxBool { val } + | Insn::IsNil { val } => { + $visit_one!(val); + } + Insn::SetGlobal { val, state, .. } + | Insn::Defined { v: val, state, .. } + | Insn::StringIntern { val, state } + | Insn::StringCopy { val, state, .. } + | Insn::ObjectAlloc { val, state } + | Insn::GuardType { val, state, .. } + | Insn::GuardBitEquals { val, state, .. } + | Insn::GuardAnyBitSet { val, state, .. } + | Insn::GuardNoBitsSet { val, state, .. } + | Insn::ToArray { val, state } + | Insn::IsMethodCfunc { val, state, .. } + | Insn::ToNewArray { val, state } + | Insn::BoxFixnum { val, state } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::GuardGreaterEq { left, right, state, .. } + | Insn::GuardLess { left, right, state, .. } => { + $visit_one!(left); + $visit_one!(right); + $visit_one!(state); + } + Insn::Snapshot { state } => { + $visit_many!(state.stack); + $visit_many!(state.locals); + } + Insn::FixnumAdd { left, right, state } + | Insn::FixnumSub { left, right, state } + | Insn::FixnumMult { left, right, state } + | Insn::FixnumDiv { left, right, state } + | Insn::FixnumMod { left, right, state } + | Insn::ArrayExtend { left, right, state } + | Insn::FixnumLShift { left, right, state } => { + $visit_one!(left); + $visit_one!(right); + $visit_one!(state); + } + Insn::FloatAdd { recv, other, state } + | Insn::FloatSub { recv, other, state } + | Insn::FloatMul { recv, other, state } + | Insn::FloatDiv { recv, other, state } => { + $visit_one!(recv); + $visit_one!(other); + $visit_one!(state); + } + Insn::FloatToInt { recv, state } => { + $visit_one!(recv); + $visit_one!(state); + } + Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::IntAnd { left, right } + | Insn::IntOr { left, right } + | Insn::FixnumRShift { left, right } + | Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + $visit_one!(left); + $visit_one!(right); + } + Insn::Jump(BranchEdge { args, .. }) => { + $visit_many!(args); + } + Insn::CondBranch { val, if_true: BranchEdge { args: true_args, .. }, if_false: BranchEdge { args: false_args, .. } } => { + $visit_one!(val); + $visit_many!(true_args); + $visit_many!(false_args); + } + Insn::ArrayDup { val, state } + | Insn::Throw { val, state, .. } + | Insn::HashDup { val, state } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::ArrayAref { array, index } => { + $visit_one!(array); + $visit_one!(index); + } + Insn::ArrayAset { array, index, val } => { + $visit_one!(array); + $visit_one!(index); + $visit_one!(val); + } + Insn::ArrayPop { array, state } => { + $visit_one!(array); + $visit_one!(state); + } + Insn::ArrayLength { array } => { + $visit_one!(array); + } + Insn::AdjustBounds { index, length } => { + $visit_one!(index); + $visit_one!(length); + } + Insn::HashAref { hash, key, state } => { + $visit_one!(hash); + $visit_one!(key); + $visit_one!(state); + } + Insn::HashAset { hash, key, val, state } => { + $visit_one!(hash); + $visit_one!(key); + $visit_one!(val); + $visit_one!(state); + } + Insn::Send { recv, args, state, .. } + | Insn::SendForward { recv, args, state, .. } + | Insn::CCallVariadic { recv, args, state, .. } + | Insn::CCallWithFrame { recv, args, state, .. } + | Insn::SendDirect { recv, args, state, .. } + | Insn::InvokeBuiltin { recv, args, state, .. } + | Insn::InvokeSuper { recv, args, state, .. } + | Insn::InvokeSuperForward { recv, args, state, .. } + | Insn::InvokeProc { recv, args, state, .. } => { + $visit_one!(recv); + $visit_many!(args); + $visit_one!(state); + } + Insn::InvokeBlock { args, state, .. } => { + $visit_many!(args); + $visit_one!(state); + } + Insn::InvokeBlockIfunc { block_handler, args, state, .. } => { + $visit_one!(block_handler); + $visit_many!(args); + $visit_one!(state); + } + Insn::CCall { recv, args, .. } => { + $visit_one!(recv); + $visit_many!(args); + } + Insn::GetIvar { self_val, state, .. } + | Insn::DefinedIvar { self_val, state, .. } => { + $visit_one!(self_val); + $visit_one!(state); + } + Insn::GetConstant { klass, allow_nil, state, .. } => { + $visit_one!(klass); + $visit_one!(allow_nil); + $visit_one!(state); + } + Insn::SetIvar { self_val, val, state, .. } => { + $visit_one!(self_val); + $visit_one!(val); + $visit_one!(state); + } + Insn::GetClassVar { state, .. } => { + $visit_one!(state); + } + Insn::SetClassVar { val, state, .. } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::ArrayPush { array, val, state } => { + $visit_one!(array); + $visit_one!(val); + $visit_one!(state); + } + Insn::ObjToString { val, state, .. } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::AnyToString { val, str, state, .. } => { + $visit_one!(val); + $visit_one!(str); + $visit_one!(state); + } + Insn::LoadField { recv, .. } => { + $visit_one!(recv); + } + Insn::StoreField { recv, val, .. } + | Insn::WriteBarrier { recv, val } => { + $visit_one!(recv); + $visit_one!(val); + } + Insn::GetGlobal { state, .. } + | Insn::GetSpecialSymbol { state, .. } + | Insn::GetSpecialNumber { state, .. } + | Insn::ObjectAllocClass { state, .. } + | Insn::SideExit { state, .. } => { + $visit_one!(state); + } + Insn::UnboxFixnum { val } => { + $visit_one!(val); + } + Insn::FixnumAref { recv, index } => { + $visit_one!(recv); + $visit_one!(index); + } + Insn::IsA { val, class } => { + $visit_one!(val); + $visit_one!(class); + } + } + }; +} + +impl Insn { + /// Not every instruction returns a value. Return true if the instruction does and false otherwise. + pub fn has_output(&self) -> bool { + match self { + Insn::Jump(_) + | Insn::Entries { .. } + | Insn::CondBranch { .. } | Insn::EntryPoint { .. } | Insn::Return { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. } + | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::BreakPoint | Insn::Unreachable + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } + | Insn::ArrayAset { .. } => false, + _ => true, + } + } + + /// Return true if the instruction ends a basic block and false otherwise. + pub fn is_terminator(&self) -> bool { + match self { + Insn::Unreachable | Insn::CondBranch { .. } | Insn::Jump(_) | Insn::Entries { .. } | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => true, + _ => false, + } + } + + /// Return true if the instruction is a jump (has successor blocks in the CFG). + pub fn is_jump(&self) -> bool { + match self { + Insn::CondBranch { .. } | Insn::Jump(_) | Insn::Entries { .. } => true, + _ => false, + } + } + + /// Call `f` on each operand (InsnId) of this instruction. + pub fn for_each_operand(&self, mut f: impl FnMut(InsnId)) { + macro_rules! visit_one { ($id:expr) => { f(*$id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); + } + + /// Call `f` on a mutable reference to each operand (InsnId) of this instruction. + pub fn for_each_operand_mut(&mut self, mut f: impl FnMut(&mut InsnId)) { + macro_rules! visit_one { ($id:expr) => { f($id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter_mut() { f(id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); + } + + /// Call `f` on each operand, short-circuiting on the first error. + pub fn try_for_each_operand<E>(&self, mut f: impl FnMut(InsnId) -> Result<(), E>) -> Result<(), E> { + macro_rules! visit_one { ($id:expr) => { f(*$id)? }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id)? } }; } + for_each_operand_impl!(self, visit_one, visit_many); + Ok(()) + } + + pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap, iseq: Option<IseqPtr>) -> InsnPrinter<'a> { + InsnPrinter { inner: self.clone(), ptr_map, iseq } + } + + // TODO(Jacob): Model SP. ie, all allocations modify stack size but using the effect for stack modification feels excessive + // TODO(Jacob): Add sideeffect failure bit + fn effects_of(&self) -> Effect { + const allocates: Effect = Effect::read_write(abstract_heaps::PC.union(abstract_heaps::Allocator), abstract_heaps::Allocator); + match &self { + Insn::Const { .. } => effects::Empty, + Insn::Param { .. } => effects::Empty, + Insn::LoadArg { .. } => effects::Empty, + Insn::StringCopy { .. } => allocates, + Insn::StringIntern { .. } => effects::Any, + Insn::StringConcat { .. } => effects::Any, + Insn::StringGetbyte { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StringSetbyteFixnum { .. } => effects::Any, + Insn::StringAppend { .. } => effects::Any, + Insn::StringAppendCodepoint { .. } => effects::Any, + Insn::StringEqual { .. } => Effect::write(abstract_heaps::Allocator), + Insn::ToRegexp { .. } => effects::Any, + Insn::PutSpecialObject { .. } => effects::Any, + Insn::ToArray { .. } => effects::Any, + Insn::ToNewArray { .. } => effects::Any, + Insn::NewArray { .. } => allocates, + Insn::NewHash { elements, .. } => { + // NewHash's operands may be hashed and compared for equality, which could have + // side-effects. Empty hashes are definitely elidable. + if elements.is_empty() { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::NewRange { .. } => effects::Any, + Insn::NewRangeFixnum { .. } => allocates, + Insn::ArrayDup { .. } => allocates, + Insn::ArrayHash { .. } => effects::Any, + Insn::ArrayMax { .. } => effects::Any, + Insn::ArrayMin { .. } => effects::Any, + Insn::ArrayInclude { .. } => effects::Any, + Insn::ArrayPackBuffer { .. } => effects::Any, + Insn::DupArrayInclude { .. } => effects::Any, + Insn::ArrayExtend { .. } => effects::Any, + Insn::ArrayPush { .. } => effects::Any, + Insn::ArrayAref { .. } => effects::Any, + Insn::ArrayAset { .. } => effects::Any, + Insn::ArrayPop { .. } => effects::Any, + Insn::ArrayLength { .. } => Effect::write(abstract_heaps::Empty), + Insn::AdjustBounds { .. } => effects::Empty, + Insn::HashAref { .. } => effects::Any, + Insn::HashAset { .. } => effects::Any, + Insn::HashDup { .. } => allocates, + Insn::ObjectAlloc { .. } => effects::Any, + Insn::ObjectAllocClass { .. } => allocates, + Insn::Test { .. } => effects::Empty, + Insn::IsNil { .. } => effects::Empty, + Insn::IsMethodCfunc { .. } => effects::Any, + Insn::IsBitEqual { .. } => effects::Empty, + Insn::IsBitNotEqual { .. } => effects::Empty, + Insn::BoxBool { .. } => effects::Empty, + Insn::BoxFixnum { .. } => effects::Empty, + Insn::UnboxFixnum { .. } => effects::Any, + Insn::FixnumAref { .. } => effects::Empty, + Insn::Defined { .. } => effects::Any, + Insn::GetConstant { .. } => effects::Any, + Insn::GetConstantPath { .. } => effects::Any, + Insn::IsBlockGiven { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::FixnumBitCheck { .. } => effects::Empty, + // IsA needs to read the class of the value and traverse the class hierarchy, which we model as reading from Memory. + Insn::IsA { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::GetGlobal { .. } => effects::Any, + Insn::SetGlobal { .. } => effects::Any, + Insn::GetIvar { .. } => effects::Any, + Insn::SetIvar { .. } => effects::Any, + Insn::DefinedIvar { .. } => effects::Any, + Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), + Insn::LoadEC { .. } => effects::Empty, + Insn::LoadSP { .. } => effects::Empty, + // GetEP reads from the current frame pointer (abstract_heaps::Frame) and also traverses previous frames too. + Insn::GetEP { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), + Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::StoreField { .. } => effects::Any, + // TODO: Refine CheckMatch effects by flag. + Insn::CheckMatch { .. } => effects::Any, + // WriteBarrier can write to object flags and mark bits in Allocator memory. + // This is why WriteBarrier writes to the "Memory" effect. We do not yet have a more granular specialization for flags + Insn::WriteBarrier { .. } => Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Allocator.union(abstract_heaps::Memory)), + Insn::SetLocal { .. } => effects::Any, + Insn::GetSpecialSymbol { .. } => effects::Any, + Insn::GetSpecialNumber { .. } => effects::Any, + Insn::GetClassVar { .. } => effects::Any, + Insn::SetClassVar { .. } => effects::Any, + Insn::IsBlockParamModified { .. } => effects::Empty, + Insn::GetBlockParam { .. } => effects::Any, + Insn::Snapshot { .. } => effects::Empty, + Insn::Jump(_) => effects::Any, + Insn::CondBranch { .. } => effects::Any, + Insn::CCall { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallWithFrame { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallVariadic { .. } => effects::Any, + Insn::Send { .. } => effects::Any, + Insn::SendForward { .. } => effects::Any, + Insn::InvokeSuper { .. } => effects::Any, + Insn::InvokeSuperForward { .. } => effects::Any, + Insn::InvokeBlock { .. } => effects::Any, + Insn::InvokeBlockIfunc { .. } => effects::Any, + Insn::SendDirect { .. } => effects::Any, + Insn::InvokeBuiltin { .. } => effects::Any, + Insn::EntryPoint { .. } => effects::Any, + Insn::Return { .. } => effects::Any, + Insn::Throw { .. } => effects::Any, + Insn::FixnumAdd { .. } => effects::Empty, + Insn::FixnumSub { .. } => effects::Empty, + Insn::FixnumMult { .. } => effects::Empty, + Insn::FixnumDiv { .. } => effects::Any, + Insn::FixnumMod { .. } => effects::Any, + Insn::FloatAdd { .. } => effects::Any, + Insn::FloatSub { .. } => effects::Any, + Insn::FloatMul { .. } => effects::Any, + Insn::FloatDiv { .. } => effects::Any, + Insn::FloatToInt { .. } => effects::Any, + Insn::FixnumEq { .. } => effects::Empty, + Insn::FixnumNeq { .. } => effects::Empty, + Insn::FixnumLt { .. } => effects::Empty, + Insn::FixnumLe { .. } => effects::Empty, + Insn::FixnumGt { .. } => effects::Empty, + Insn::FixnumGe { .. } => effects::Empty, + Insn::FixnumAnd { .. } => effects::Empty, + Insn::FixnumOr { .. } => effects::Empty, + Insn::FixnumXor { .. } => effects::Empty, + Insn::IntAnd { .. } => effects::Empty, + Insn::IntOr { .. } => effects::Empty, + Insn::FixnumLShift { .. } => effects::Empty, + Insn::FixnumRShift { .. } => effects::Empty, + Insn::ObjToString { .. } => effects::Any, + Insn::AnyToString { .. } => effects::Any, + Insn::GuardType { guard_type, .. } + => Effect::read_write( + if guard_type.is_subtype(types::Immediate) { abstract_heaps::Empty } else { abstract_heaps::Memory }, + abstract_heaps::Control + ), + Insn::GuardBitEquals { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardAnyBitSet { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardNoBitsSet { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardGreaterEq { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardLess { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::PatchPoint { .. } => Effect::read_write(abstract_heaps::PatchPoint, abstract_heaps::Control), + Insn::SideExit { .. } => effects::Any, + Insn::IncrCounter(_) => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Other), + Insn::IncrCounterPtr { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Other), + Insn::CheckInterrupts { .. } => Effect::read_write(abstract_heaps::InterruptFlag, abstract_heaps::Control), + Insn::InvokeProc { .. } => effects::Any, + Insn::RefineType { .. } => effects::Empty, + Insn::HasType { expected, .. } + => Effect::read_write( + if expected.is_subtype(types::Immediate) { abstract_heaps::Empty } else { abstract_heaps::Memory }, + abstract_heaps::Control + ), + Insn::Entries { .. } => effects::Any, + Insn::BreakPoint | Insn::Unreachable => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + } + } + + /// Return true if we can safely omit the instruction. This occurs when one of the following + /// conditions are met. + /// 1. The instruction does not write anything. + /// 2. The instruction only allocates and writes nothing else. + /// Calling the effects of our instruction `insn_effects`, we need: + /// `effects::Empty` to include `insn_effects.write` or `effects::Allocator` to include + /// `insn_effects.write`. + /// We can simplify this to `effects::Empty.union(effects::Allocator).includes(insn_effects.write)`. + /// But the union of `Allocator` and `Empty` is simply `Allocator`, so our entire function + /// collapses to `effects::Allocator.includes(insn_effects.write)`. + /// Note: These are restrictions on the `write` `EffectSet` only. Even instructions with + /// `read: effects::Any` could potentially be omitted. + fn is_elidable(&self) -> bool { + abstract_heaps::Allocator.includes(self.effects_of().write_bits()) + } +} + +/// Print adaptor for [`Insn`]. See [`PtrPrintMap`]. +pub struct InsnPrinter<'a> { + inner: Insn, + ptr_map: &'a PtrPrintMap, + iseq: Option<IseqPtr>, +} + +fn get_local_var_id(iseq: IseqPtr, level: u32, ep_offset: u32) -> ID { + let mut current_iseq = iseq; + for _ in 0..level { + current_iseq = unsafe { rb_get_iseq_body_parent_iseq(current_iseq) }; + } + let local_idx = ep_offset_to_local_idx(current_iseq, ep_offset); + unsafe { rb_zjit_local_id(current_iseq, local_idx.try_into().unwrap()) } +} + +/// Get the name of a local variable given iseq, level, and ep_offset. +/// Returns +/// - `":name"` if iseq is available and name is a real identifier, +/// - `"<empty>"` for anonymous locals. +/// - `None` if iseq is not available. +/// (When `Insn` is printed in a panic/debug message the `Display::fmt` method is called, which can't access an iseq.) +/// +/// This mimics local_var_name() from iseq.c. +fn get_local_var_name_for_printer(iseq: Option<IseqPtr>, level: u32, ep_offset: u32) -> Option<String> { + let id = get_local_var_id(iseq?, level, ep_offset); + + if id_is_empty(id) { + return Some(String::from("<empty>")); + } + + Some(format!(":{}", id.contents_lossy())) +} + + +fn id_is_empty(id: ID) -> bool { + id.0 == 0 || unsafe { rb_id2str(id) } == Qfalse +} + +/// Construct a qualified method name for display/debug output. +/// Returns strings like "Array#length" for instance methods or "Foo.bar" for singleton methods. +fn qualified_method_name(class: VALUE, method_id: ID) -> String { + let method_name = method_id.contents_lossy(); + // rb_zjit_singleton_class_p also checks if it's a class + if unsafe { rb_zjit_singleton_class_p(class) } { + let class_name = get_class_name(unsafe { rb_class_attached_object(class) }); + format!("{class_name}.{method_name}") + } else { + let class_name = get_class_name(class); + format!("{class_name}#{method_name}") + } +} + +static REGEXP_FLAGS: &[(u32, &str)] = &[ + (ONIG_OPTION_MULTILINE, "MULTILINE"), + (ONIG_OPTION_IGNORECASE, "IGNORECASE"), + (ONIG_OPTION_EXTEND, "EXTENDED"), + (ARG_ENCODING_FIXED, "FIXEDENCODING"), + (ARG_ENCODING_NONE, "NOENCODING"), +]; + +impl<'a> std::fmt::Display for InsnPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match &self.inner { + Insn::Const { val } => { write!(f, "Const {}", val.print(self.ptr_map)) } + Insn::Param => { write!(f, "Param") } + Insn::LoadArg { idx, id, .. } => { write!(f, "LoadArg :{id}@{idx}") } + Insn::Entries { targets } => { + write!(f, "Entries")?; + let mut prefix = " "; + for target in targets { + write!(f, "{prefix}{target}")?; + prefix = ", "; + } + Ok(()) + } + Insn::NewArray { elements, .. } => { + write!(f, "NewArray")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayAref { array, index, .. } => { + write!(f, "ArrayAref {array}, {index}") + } + Insn::ArrayAset { array, index, val, ..} => { + write!(f, "ArrayAset {array}, {index}, {val}") + } + Insn::ArrayPop { array, .. } => { + write!(f, "ArrayPop {array}") + } + Insn::ArrayLength { array } => { + write!(f, "ArrayLength {array}") + } + Insn::AdjustBounds { index, length } => { + write!(f, "AdjustBounds {index}, {length}") + } + Insn::NewHash { elements, .. } => { + write!(f, "NewHash")?; + let mut prefix = " "; + for chunk in elements.chunks(2) { + if let [key, value] = chunk { + write!(f, "{prefix}{key}: {value}")?; + prefix = ", "; + } + } + Ok(()) + } + Insn::NewRange { low, high, flag, .. } => { + write!(f, "NewRange {low} {flag} {high}") + } + Insn::NewRangeFixnum { low, high, flag, .. } => { + write!(f, "NewRangeFixnum {low} {flag} {high}") + } + Insn::ArrayMax { elements, .. } => { + write!(f, "ArrayMax")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayMin { elements, .. } => { + write!(f, "ArrayMin")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayHash { elements, .. } => { + write!(f, "ArrayHash")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayInclude { elements, target, .. } => { + write!(f, "ArrayInclude")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + write!(f, " | {target}") + } + Insn::ArrayPackBuffer { elements, fmt, buffer, .. } => { + write!(f, "ArrayPackBuffer ")?; + for element in elements { + write!(f, "{element}, ")?; + } + write!(f, "fmt: {fmt}")?; + if let Some(buffer) = buffer { + write!(f, ", buf: {buffer}")?; + } + Ok(()) + } + Insn::DupArrayInclude { ary, target, .. } => { + write!(f, "DupArrayInclude {} | {}", ary.print(self.ptr_map), target) + } + Insn::ArrayDup { val, .. } => { write!(f, "ArrayDup {val}") } + Insn::HashDup { val, .. } => { write!(f, "HashDup {val}") } + Insn::HashAref { hash, key, .. } => { write!(f, "HashAref {hash}, {key}")} + Insn::HashAset { hash, key, val, .. } => { write!(f, "HashAset {hash}, {key}, {val}")} + Insn::ObjectAlloc { val, .. } => { write!(f, "ObjectAlloc {val}") } + &Insn::ObjectAllocClass { class, .. } => { + let class_name = get_class_name(class); + write!(f, "ObjectAllocClass {class_name}:{}", class.print(self.ptr_map)) + } + Insn::StringCopy { val, .. } => { write!(f, "StringCopy {val}") } + Insn::StringConcat { strings, .. } => { + write!(f, "StringConcat")?; + let mut prefix = " "; + for string in strings { + write!(f, "{prefix}{string}")?; + prefix = ", "; + } + + Ok(()) + } + Insn::StringGetbyte { string, index, .. } => { + write!(f, "StringGetbyte {string}, {index}") + } + Insn::StringSetbyteFixnum { string, index, value, .. } => { + write!(f, "StringSetbyteFixnum {string}, {index}, {value}") + } + Insn::StringAppend { recv, other, .. } => { + write!(f, "StringAppend {recv}, {other}") + } + Insn::StringAppendCodepoint { recv, other, .. } => { + write!(f, "StringAppendCodepoint {recv}, {other}") + } + Insn::StringEqual { left, right } => { + write!(f, "StringEqual {left}, {right}") + } + Insn::ToRegexp { values, opt, .. } => { + write!(f, "ToRegexp")?; + let mut prefix = " "; + for value in values { + write!(f, "{prefix}{value}")?; + prefix = ", "; + } + + let opt = *opt as u32; + if opt != 0 { + write!(f, ", ")?; + let mut sep = ""; + for (flag, name) in REGEXP_FLAGS { + if opt & flag != 0 { + write!(f, "{sep}{name}")?; + sep = "|"; + } + } + } + + Ok(()) + } + Insn::Test { val } => { write!(f, "Test {val}") } + Insn::IsNil { val } => { write!(f, "IsNil {val}") } + Insn::IsMethodCfunc { val, cd, .. } => { write!(f, "IsMethodCFunc {val}, :{}", ruby_call_method_name(*cd)) } + Insn::IsBitEqual { left, right } => write!(f, "IsBitEqual {left}, {right}"), + Insn::IsBitNotEqual { left, right } => write!(f, "IsBitNotEqual {left}, {right}"), + Insn::BoxBool { val } => write!(f, "BoxBool {val}"), + Insn::BoxFixnum { val, .. } => write!(f, "BoxFixnum {val}"), + Insn::UnboxFixnum { val } => write!(f, "UnboxFixnum {val}"), + Insn::FixnumAref { recv, index } => write!(f, "FixnumAref {recv}, {index}"), + Insn::Jump(target) => { write!(f, "Jump {target}") } + Insn::CondBranch { val, if_true, if_false } => { write!(f, "CondBranch {val}, {if_true}, {if_false}") }, + Insn::SendDirect { recv, cme, iseq, args, block, .. } => { + let blockiseq = block.map(|bh| match bh { BlockHandler::BlockIseq(iseq) => iseq, BlockHandler::BlockArg => unreachable!() }); + let method_name = unsafe { (**cme).called_id }; + write!(f, "SendDirect {recv}, {:p}, :{} ({:?})", self.ptr_map.map_ptr(&blockiseq), method_name, self.ptr_map.map_ptr(iseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + Insn::Send { recv, cd, args, block, reason, .. } => { + // For tests, we want to check HIR snippets textually. Addresses change + // between runs, making tests fail. Instead, pick an arbitrary hex value to + // use as a "pointer" so we can check the rest of the HIR. + match *block { + Some(BlockHandler::BlockIseq(blockiseq)) => + write!(f, "Send {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?, + Some(BlockHandler::BlockArg) => + write!(f, "Send {recv}, &block, :{}", ruby_call_method_name(*cd))?, + None => + write!(f, "Send {recv}, :{}", ruby_call_method_name(*cd))?, + } + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::SendForward { recv, cd, args, blockiseq, reason, .. } => { + write!(f, "SendForward {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuper { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuper {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuperForward { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuperForward {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeBlock { args, reason, .. } => { + write!(f, "InvokeBlock")?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeBlockIfunc { block_handler, args, .. } => { + write!(f, "InvokeBlockIfunc {block_handler}")?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + Insn::InvokeProc { recv, args, kw_splat, .. } => { + write!(f, "InvokeProc {recv}")?; + for arg in args { + write!(f, ", {arg}")?; + } + if *kw_splat { + write!(f, ", kw_splat")?; + } + Ok(()) + } + Insn::InvokeBuiltin { bf, args, leaf, .. } => { + let bf_name = unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap(); + write!(f, "InvokeBuiltin{} {}", + if *leaf { " leaf" } else { "" }, + // e.g. Code that use `Primitive.cexpr!`. From BUILTIN_INLINE_PREFIX. + if bf_name.starts_with("_bi") { "<inline_expr>" } else { bf_name })?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + &Insn::EntryPoint { jit_entry_idx: Some(idx) } => write!(f, "EntryPoint JIT({idx})"), + &Insn::EntryPoint { jit_entry_idx: None } => write!(f, "EntryPoint interpreter"), + Insn::Return { val } => { write!(f, "Return {val}") } + Insn::FixnumAdd { left, right, .. } => { write!(f, "FixnumAdd {left}, {right}") }, + Insn::FixnumSub { left, right, .. } => { write!(f, "FixnumSub {left}, {right}") }, + Insn::FixnumMult { left, right, .. } => { write!(f, "FixnumMult {left}, {right}") }, + Insn::FixnumDiv { left, right, .. } => { write!(f, "FixnumDiv {left}, {right}") }, + Insn::FixnumMod { left, right, .. } => { write!(f, "FixnumMod {left}, {right}") }, + Insn::FloatAdd { recv, other, .. } => { write!(f, "FloatAdd {recv}, {other}") }, + Insn::FloatSub { recv, other, .. } => { write!(f, "FloatSub {recv}, {other}") }, + Insn::FloatMul { recv, other, .. } => { write!(f, "FloatMul {recv}, {other}") }, + Insn::FloatDiv { recv, other, .. } => { write!(f, "FloatDiv {recv}, {other}") }, + Insn::FloatToInt { recv, .. } => { write!(f, "FloatToInt {recv}") }, + Insn::FixnumEq { left, right, .. } => { write!(f, "FixnumEq {left}, {right}") }, + Insn::FixnumNeq { left, right, .. } => { write!(f, "FixnumNeq {left}, {right}") }, + Insn::FixnumLt { left, right, .. } => { write!(f, "FixnumLt {left}, {right}") }, + Insn::FixnumLe { left, right, .. } => { write!(f, "FixnumLe {left}, {right}") }, + Insn::FixnumGt { left, right, .. } => { write!(f, "FixnumGt {left}, {right}") }, + Insn::FixnumGe { left, right, .. } => { write!(f, "FixnumGe {left}, {right}") }, + Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, + Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, + Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::IntAnd { left, right } => { write!(f, "IntAnd {left}, {right}") }, + Insn::IntOr { left, right } => { write!(f, "IntOr {left}, {right}") }, + Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, + Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, + Insn::GuardType { val, guard_type, .. } => { write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map)) }, + Insn::RefineType { val, new_type, .. } => { write!(f, "RefineType {val}, {}", new_type.print(self.ptr_map)) }, + Insn::HasType { val, expected, .. } => { write!(f, "HasType {val}, {}", expected.print(self.ptr_map)) }, + Insn::GuardBitEquals { val, expected, recompile, .. } => { + write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map))?; + if recompile.is_some() { + write!(f, " recompile")?; + } + return Ok(()) + }, + Insn::GuardAnyBitSet { val, mask, mask_name: Some(name), .. } => { write!(f, "GuardAnyBitSet {val}, {name}={}", mask.print(self.ptr_map)) }, + Insn::GuardAnyBitSet { val, mask, .. } => { write!(f, "GuardAnyBitSet {val}, {}", mask.print(self.ptr_map)) }, + Insn::GuardNoBitsSet { val, mask, mask_name: Some(name), .. } => { write!(f, "GuardNoBitsSet {val}, {name}={}", mask.print(self.ptr_map)) }, + Insn::GuardNoBitsSet { val, mask, .. } => { write!(f, "GuardNoBitsSet {val}, {}", mask.print(self.ptr_map)) }, + Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), + Insn::GuardGreaterEq { left, right, .. } => write!(f, "GuardGreaterEq {left}, {right}"), + &Insn::GetBlockParam { level, ep_offset, .. } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset) + .map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetBlockParam {name}l{level}, EP@{ep_offset}") + }, + Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, + Insn::GetConstant { klass, id, allow_nil, .. } => { + write!(f, "GetConstant {klass}, :{}, {allow_nil}", id.contents_lossy()) + } + Insn::GetConstantPath { ic, .. } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, + Insn::IsBlockGiven { lep } => { write!(f, "IsBlockGiven {lep}") }, + Insn::FixnumBitCheck {val, index} => { write!(f, "FixnumBitCheck {val}, {index}") }, + Insn::CCall { cfunc, recv, args, name, owner, return_type: _, elidable: _ } => { + let display_name = if *owner == Qnil { name.contents_lossy().to_string() } else { qualified_method_name(*owner, *name) }; + write!(f, "CCall {recv}, :{}@{:p}", display_name, self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::CCallWithFrame { cfunc, recv, args, name, cme, block, .. } => { + write!(f, "CCallWithFrame {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + match block { + Some(BlockHandler::BlockIseq(blockiseq)) => + write!(f, ", block={:p}", self.ptr_map.map_ptr(blockiseq))?, + Some(BlockHandler::BlockArg) => + write!(f, ", block=&block")?, + None => {} + } + Ok(()) + }, + Insn::CCallVariadic { cfunc, recv, args, name, cme, .. } => { + write!(f, "CCallVariadic {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::IncrCounterPtr { .. } => write!(f, "IncrCounterPtr"), + Insn::Snapshot { state } => write!(f, "Snapshot {}", state.print(self.ptr_map)), + Insn::Defined { op_type, v, .. } => { + // op_type (enum defined_type) printing logic from iseq.c. + // Not sure why rb_iseq_defined_string() isn't exhaustive. + write!(f, "Defined ")?; + let op_type = *op_type as u32; + if op_type == DEFINED_FUNC { + write!(f, "func")?; + } else if op_type == DEFINED_REF { + write!(f, "ref")?; + } else if op_type == DEFINED_CONST_FROM { + write!(f, "constant-from")?; + } else { + write!(f, "{}", String::from_utf8_lossy(unsafe { rb_iseq_defined_string(op_type).as_rstring_byte_slice().unwrap() }))?; + }; + write!(f, ", {v}") + } + Insn::DefinedIvar { self_val, id, .. } => write!(f, "DefinedIvar {self_val}, :{}", id.contents_lossy()), + Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), + Insn::CheckMatch { target, pattern, flag, .. } => { + const TYPE_MASK: u32 = 0x03; + const ARRAY_FLAG: u32 = 0x04; + + let match_type = match *flag & TYPE_MASK { + VM_CHECKMATCH_TYPE_WHEN => "WHEN", + VM_CHECKMATCH_TYPE_CASE => "CASE", + VM_CHECKMATCH_TYPE_RESCUE => "RESCUE", + _ => return write!(f, "CheckMatch {target}, {pattern}, {flag}"), + }; + let flag = if *flag & ARRAY_FLAG != 0 { + format!("{match_type}|ARRAY") + } else { + match_type.to_string() + }; + write!(f, "CheckMatch {target}, {pattern}, {flag}") + } + Insn::LoadPC => write!(f, "LoadPC"), + Insn::LoadEC => write!(f, "LoadEC"), + Insn::LoadSP => write!(f, "LoadSP"), + &Insn::GetEP { level } => write!(f, "GetEP {level}"), + Insn::LoadSelf => write!(f, "LoadSelf"), + &Insn::LoadField { recv, id, offset, return_type: _ } => { + write!(f, "LoadField {recv}, :{id}@{:#x}", self.ptr_map.map_offset(offset)) + } + &Insn::StoreField { recv, id, offset, val } => write!(f, "StoreField {recv}, :{id}@{:#x}, {val}", self.ptr_map.map_offset(offset)), + &Insn::WriteBarrier { recv, val } => write!(f, "WriteBarrier {recv}, {val}"), + Insn::SetIvar { self_val, id, val, .. } => write!(f, "SetIvar {self_val}, :{}, {val}", id.contents_lossy()), + Insn::GetGlobal { id, .. } => write!(f, "GetGlobal :{}", id.contents_lossy()), + Insn::SetGlobal { id, val, .. } => write!(f, "SetGlobal :{}, {val}", id.contents_lossy()), + &Insn::IsBlockParamModified { flags } => { + write!(f, "IsBlockParamModified {flags}") + }, + &Insn::SetLocal { val, level, ep_offset } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "SetLocal {name}l{level}, EP@{ep_offset}, {val}") + }, + Insn::GetSpecialSymbol { symbol_type, .. } => write!(f, "GetSpecialSymbol {symbol_type:?}"), + Insn::GetSpecialNumber { nth, .. } => write!(f, "GetSpecialNumber {nth}"), + Insn::GetClassVar { id, .. } => write!(f, "GetClassVar :{}", id.contents_lossy()), + Insn::SetClassVar { id, val, .. } => write!(f, "SetClassVar :{}, {val}", id.contents_lossy()), + Insn::ToArray { val, .. } => write!(f, "ToArray {val}"), + Insn::ToNewArray { val, .. } => write!(f, "ToNewArray {val}"), + Insn::ArrayExtend { left, right, .. } => write!(f, "ArrayExtend {left}, {right}"), + Insn::ArrayPush { array, val, .. } => write!(f, "ArrayPush {array}, {val}"), + Insn::ObjToString { val, .. } => { write!(f, "ObjToString {val}") }, + Insn::StringIntern { val, .. } => { write!(f, "StringIntern {val}") }, + Insn::AnyToString { val, str, .. } => { write!(f, "AnyToString {val}, str: {str}") }, + Insn::SideExit { reason, recompile, .. } => { + if recompile.is_some() { + write!(f, "SideExit {reason} recompile") + } else { + write!(f, "SideExit {reason}") + } + } + Insn::PutSpecialObject { value_type, .. } => write!(f, "PutSpecialObject {value_type}"), + Insn::Throw { throw_state, val, .. } => { + write!(f, "Throw ")?; + match throw_state & VM_THROW_STATE_MASK { + RUBY_TAG_NONE => write!(f, "TAG_NONE"), + RUBY_TAG_RETURN => write!(f, "TAG_RETURN"), + RUBY_TAG_BREAK => write!(f, "TAG_BREAK"), + RUBY_TAG_NEXT => write!(f, "TAG_NEXT"), + RUBY_TAG_RETRY => write!(f, "TAG_RETRY"), + RUBY_TAG_REDO => write!(f, "TAG_REDO"), + RUBY_TAG_RAISE => write!(f, "TAG_RAISE"), + RUBY_TAG_THROW => write!(f, "TAG_THROW"), + RUBY_TAG_FATAL => write!(f, "TAG_FATAL"), + tag => write!(f, "{tag}") + }?; + if throw_state & VM_THROW_NO_ESCAPE_FLAG != 0 { + write!(f, "|NO_ESCAPE")?; + } + write!(f, ", {val}") + } + Insn::IncrCounter(counter) => write!(f, "IncrCounter {counter:?}"), + Insn::CheckInterrupts { .. } => write!(f, "CheckInterrupts"), + Insn::IsA { val, class } => write!(f, "IsA {val}, {class}"), + Insn::BreakPoint => write!(f, "BreakPoint"), + Insn::Unreachable => write!(f, "Unreachable"), + } + } +} + +impl std::fmt::Display for Insn { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity(), None).fmt(f) + } +} + +/// An extended basic block in a [`Function`]. +#[derive(Default, Debug)] +pub struct Block { + /// The index of the first YARV instruction for the Block in the ISEQ + pub insn_idx: u32, + params: Vec<InsnId>, + insns: Vec<InsnId>, +} + +impl Block { + /// Return an iterator over params + pub fn params(&self) -> Iter<'_, InsnId> { + self.params.iter() + } + + /// Return an iterator over insns + pub fn insns(&self) -> Iter<'_, InsnId> { + self.insns.iter() + } +} + +/// Pretty printer for [`Function`]. +pub struct FunctionPrinter<'a> { + fun: &'a Function, + display_snapshot_and_tp_patchpoints: bool, + ptr_map: PtrPrintMap, +} + +impl<'a> FunctionPrinter<'a> { + pub fn without_snapshot(fun: &'a Function) -> Self { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + Self { fun, display_snapshot_and_tp_patchpoints: false, ptr_map } + } + + pub fn with_snapshot(fun: &'a Function) -> FunctionPrinter<'a> { + let mut printer = Self::without_snapshot(fun); + printer.display_snapshot_and_tp_patchpoints = true; + printer + } +} + +/// Union-Find (Disjoint-Set) is a data structure for managing disjoint sets that has an interface +/// of two operations: +/// +/// * find (what set is this item part of?) +/// * union (join these two sets) +/// +/// Union-Find identifies sets by their *representative*, which is some chosen element of the set. +/// This is implemented by structuring each set as its own graph component with the representative +/// pointing at nothing. For example: +/// +/// * A -> B -> C +/// * D -> E +/// +/// This represents two sets `C` and `E`, with three and two members, respectively. In this +/// example, `find(A)=C`, `find(C)=C`, `find(D)=E`, and so on. +/// +/// To union sets, call `make_equal_to` on any set element. That is, `make_equal_to(A, D)` and +/// `make_equal_to(B, E)` have the same result: the two sets are joined into the same graph +/// component. After this operation, calling `find` on any element will return `E`. +/// +/// This is a useful data structure in compilers because it allows in-place rewriting without +/// linking/unlinking instructions and without replacing all uses. When calling `make_equal_to` on +/// any instruction, all of its uses now implicitly point to the replacement. +/// +/// This does mean that pattern matching and analysis of the instruction graph must be careful to +/// call `find` whenever it is inspecting an instruction (or its operands). If not, this may result +/// in missing optimizations. +#[derive(Debug)] +struct UnionFind<T: Copy + Into<usize>> { + forwarded: Vec<Option<T>>, +} + +impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { + fn new() -> UnionFind<T> { + UnionFind { forwarded: vec![] } + } + + /// Private. Return the internal representation of the forwarding pointer for a given element. + fn at(&self, idx: T) -> Option<T> { + self.forwarded.get(idx.into()).copied().flatten() + } + + /// Private. Set the internal representation of the forwarding pointer for the given element + /// `idx`. Extend the internal vector if necessary. + fn set(&mut self, idx: T, value: T) { + if idx.into() >= self.forwarded.len() { + self.forwarded.resize(idx.into()+1, None); + } + if idx != value { + self.forwarded[idx.into()] = Some(value); + } + } + + /// Find the set representative for `insn`. Perform path compression at the same time to speed + /// up further find operations. For example, before: + /// + /// `A -> B -> C` + /// + /// and after `find(A)`: + /// + /// ``` + /// A -> C + /// B ---^ + /// ``` + pub fn find(&mut self, insn: T) -> T { + let result = self.find_const(insn); + if result != insn { + // Path compression + self.set(insn, result); + } + result + } + + /// Find the set representative for `insn` without doing path compression. + fn find_const(&self, insn: T) -> T { + let mut result = insn; + loop { + match self.at(result) { + None => return result, + Some(insn) => { + assert!(result != insn, "cycle detected"); + result = insn; + } + } + } + } + + /// Union the two sets containing `insn` and `target` such that every element in `insn`s set is + /// now part of `target`'s. Neither argument must be the representative in its set. + pub fn make_equal_to(&mut self, insn: T, target: T) { + let found = self.find(insn); + self.set(found, target); + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ValidationError { + BlockHasNoTerminator(BlockId), + // The terminator and its actual position + TerminatorNotAtEnd(BlockId, InsnId, usize), + /// Expected length, actual length + MismatchedBlockArity(BlockId, usize, usize), + JumpTargetNotInRPO(BlockId), + // The offending instruction, and the operand + OperandNotDefined(BlockId, InsnId, InsnId), + /// The offending block and instruction + DuplicateInstruction(BlockId, InsnId), + /// The offending instruction, its operand, expected type string, actual type string + MismatchedOperandType(InsnId, InsnId, String, String), + MiscValidationError(InsnId, String), +} + +/// Check if we can do a direct send to the given iseq with the given args. +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, ci: *const rb_callinfo, send_insn: InsnId, args: &[InsnId], has_block: bool) -> bool { + let mut can_send = true; + let mut count_failure = |counter| { + can_send = false; + function.count(block, counter); + }; + let params = unsafe { iseq.params() }; + + let callee_has_block_param = 0 != params.flags.has_block(); + let caller_passes_block_arg = (unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) != 0; + + use Counter::*; + if 0 != params.flags.has_rest() { count_failure(complex_arg_pass_param_rest) } + if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } + if callee_has_block_param && caller_passes_block_arg + { count_failure(complex_arg_pass_param_block) } + if 0 != params.flags.has_kwrest() { count_failure(complex_arg_pass_param_kwrest) } + + // If the caller passes a block (literal or &block), we need to fall back to the + // interpreter for two cases it handles that we don't: + // 1. Methods with &nil reject blocks with ArgumentError + // 2. Methods that don't use blocks emit "unused block" warnings + let caller_passes_block = has_block || caller_passes_block_arg; + if caller_passes_block && 0 != params.flags.accepts_no_block() + { count_failure(complex_arg_pass_accepts_no_block) } + if caller_passes_block && 0 == params.flags.use_block() + { count_failure(complex_arg_pass_does_not_use_block) } + + if !can_send { + function.set_dynamic_send_reason(send_insn, ComplexArgPass); + return false; + } + + let lead_num = params.lead_num; + let opt_num = params.opt_num; + let post_num = params.post_num; + let keyword = params.keyword; + let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_positional = match args.len().checked_sub(caller_kw_count) { + Some(count) => count, + None => { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false; + } + }; + + let positional_ok = c_int::try_from(caller_positional) + .as_ref() + .map(|argc| (lead_num + post_num..=lead_num + opt_num + post_num).contains(argc)) + .unwrap_or(false); + let keyword_ok = c_int::try_from(caller_kw_count) + .as_ref() + .map(|argc| (kw_req_num..=kw_total_num).contains(argc)) + .unwrap_or(false); + if !positional_ok || !keyword_ok { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false + } + + // asm.ccall() doesn't support 6+ args. Compute the final argc after keyword setup: + // final_argc = caller's positional args + callee's total keywords (all kw slots are filled). + // Right now, the JIT entrypoint accepts the block as an param + // We may remove it, remove the block_arg addition to match + // See: https://github.com/ruby/ruby/pull/15911#discussion_r2710544982 + let block_arg = if 0 != params.flags.has_block() { 1 } else { 0 }; + let final_argc = caller_positional + kw_total_num as usize + block_arg; + if final_argc + 1 > C_ARG_OPNDS.len() { // +1 for self + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + // IseqCall stores num_optionals_passed and argc as u16 + if u16::try_from(args.len()).is_err() { + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + can_send +} + +/// Policy that controls how optimization passes generate code. +/// Determined at compile time based on the ISEQ's compilation history. +#[derive(Debug)] +struct CompilePolicy { + /// When true, optimization passes should avoid generating guards that + /// side-exit, and instead use fallback paths (e.g. C calls) on mismatch. + /// Set when this is the final version of an ISEQ after recompilation. + no_side_exits: bool, +} + +impl CompilePolicy { + fn new(iseq: *const rb_iseq_t) -> Self { + // When a previous version was invalidated and we've reached the version + // limit, avoid speculative optimizations that may side-exit. + let no_side_exits = if iseq.is_null() { + false + } else { + let payload = get_or_create_iseq_payload(iseq); + payload.versions.iter().any( + |v| unsafe { v.as_ref() }.is_invalidated() + ) && payload.versions.len() + 1 >= max_iseq_versions() + }; + Self { no_side_exits } + } +} + +/// A [`Function`], which is analogous to a Ruby ISeq, is a control-flow graph of [`Block`]s +/// containing instructions. +#[derive(Debug)] +pub struct Function { + // ISEQ this function refers to + iseq: *const rb_iseq_t, + /// Whether previously, a function for this ISEQ was invalidated due to + /// singleton class creation (violation of NoSingletonClass invariant). + was_invalidated_for_singleton_class_creation: bool, + /// Controls code generation strategy for optimization passes. + policy: CompilePolicy, + /// The types for the parameters of this function. They are copied to the type + /// of entry block params after infer_types() fills Empty to all insn_types. + param_types: Vec<Type>, + + insns: Vec<Insn>, + union_find: std::cell::RefCell<UnionFind<InsnId>>, + insn_types: Vec<Type>, + blocks: Vec<Block>, + /// Superblock that targets all entry blocks. The sole root for RPO/dominator computation. + pub entries_block: BlockId, + /// Entry block for the interpreter + entry_block: BlockId, + /// Entry block for JIT-to-JIT calls. Length will be `opt_num+1`, for callers + /// fulfilling `(0..=opt_num)` optional parameters. + jit_entry_blocks: Vec<BlockId>, + profiles: Option<ProfileOracle>, +} + +/// The kind of a value an ISEQ returns +enum IseqReturn { + Value(VALUE), + LocalVariable(u32), + Receiver, + // Builtin descriptor and return type (if known) + InvokeLeafBuiltin(rb_builtin_function, Option<Type>), +} + +unsafe extern "C" { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<InsnId>, ci_flags: u32) -> Option<IseqReturn> { + // Expect only two instructions and one possible operand + // NOTE: If an ISEQ has an optional keyword parameter with a default value that requires + // computation, the ISEQ will always have more than two instructions and won't be inlined. + + // Get the first two instructions + let first_insn = iseq_opcode_at_idx(iseq, 0); + let second_insn = iseq_opcode_at_idx(iseq, insn_len(first_insn as usize)); + + // Extract the return value if known + if second_insn != YARVINSN_leave { + return None; + } + match first_insn { + YARVINSN_getlocal_WC_0 => { + // Accept only cases where only positional arguments are used by both the callee and the caller. + // Keyword arguments may be specified by the callee or the caller but not used. + if captured_opnd.is_some() + // Equivalent to `VM_CALL_ARGS_SIMPLE - VM_CALL_KWARG - has_block_iseq` + || ci_flags & ( + VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING + ) != 0 + { + return None; + } + + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + + // Only inline if the local is a parameter (not a method-defined local) as we are indexing args. + let param_size = unsafe { rb_get_iseq_body_param_size(iseq) } as usize; + if local_idx >= param_size { + return None; + } + + if unsafe { rb_simple_iseq_p(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx.try_into().unwrap())); + } + + // TODO(max): Support only_kwparam case where the local_idx is a positional parameter + + None + } + YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), + YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), + YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), + YARVINSN_putobject_INT2FIX_1_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(1))), + // We don't support invokeblock for now. Such ISEQs are likely not used by blocks anyway. + YARVINSN_putself if captured_opnd.is_none() => Some(IseqReturn::Receiver), + YARVINSN_opt_invokebuiltin_delegate_leave => { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let argc = bf.argc as usize; + if argc != 0 { return None; } + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + if !leaf { return None; } + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + Some(IseqReturn::InvokeLeafBuiltin(bf, return_type)) + } + _ => None, + } +} + +impl Function { + fn new(iseq: *const rb_iseq_t) -> Function { + Function { + iseq, + was_invalidated_for_singleton_class_creation: false, + policy: CompilePolicy::new(iseq), + insns: vec![], + insn_types: vec![], + union_find: UnionFind::new().into(), + blocks: vec![Block::default(), Block::default()], + entries_block: BlockId(0), + entry_block: BlockId(1), + jit_entry_blocks: vec![], + param_types: vec![], + profiles: None, + } + } + + pub fn iseq(&self) -> *const rb_iseq_t { + self.iseq + } + + // Add an instruction to the function without adding it to any block + fn new_insn(&mut self, insn: Insn) -> InsnId { + let id = InsnId(self.insns.len()); + if insn.has_output() { + self.insn_types.push(types::Any); + } else { + self.insn_types.push(types::Empty); + } + self.insns.push(insn); + id + } + + // Add an instruction to an SSA block + pub fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { + let is_param = matches!(insn, Insn::Param); + let id = self.new_insn(insn); + if is_param { + self.blocks[block.0].params.push(id); + } else { + self.blocks[block.0].insns.push(id); + } + id + } + + // Add an instruction to an SSA block + fn push_insn_id(&mut self, block: BlockId, insn_id: InsnId) -> InsnId { + self.blocks[block.0].insns.push(insn_id); + insn_id + } + + /// Return the number of instructions + pub fn num_insns(&self) -> usize { + self.insns.len() + } + + /// Return a FrameState at the given instruction index. + pub fn frame_state(&self, insn_id: InsnId) -> FrameState { + match self.find(insn_id) { + Insn::Snapshot { state } => state, + insn => panic!("Unexpected non-Snapshot {insn} when looking up FrameState"), + } + } + + fn new_block(&mut self, insn_idx: u32) -> BlockId { + let id = BlockId(self.blocks.len()); + let block = Block { + insn_idx, + .. Block::default() + }; + self.blocks.push(block); + id + } + + fn remove_block(&mut self, block_id: BlockId) { + if BlockId(self.blocks.len() - 1) != block_id { + panic!("Can only remove the last block"); + } + self.blocks.pop(); + } + + fn successors(&self, block: BlockId) -> Vec<BlockId> { + let insns = &self.blocks[block.0].insns; + let last = self.find(*insns.last().unwrap()); + match last { + Insn::CondBranch { if_true, if_false, .. } => vec![if_true.target, if_false.target], + Insn::Jump(edge) => vec![edge.target], + Insn::Entries { targets } => targets, + Insn::Unreachable | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => vec![], + // Blocks that don't end with terminators are technically errors, + // every block in the CFG should end with a terminator. But we + // want to be able to iterate over poorly constructed CFG when + // debugging, so we'll return an empty vec. The validation + // routines check for terminators, so we should catch CFG errors there. + _ => vec![] + } + } + + /// Return a reference to the Block at the given index. + pub fn block(&self, block_id: BlockId) -> &Block { + &self.blocks[block_id.0] + } + + /// Return a reference to the entry block. + pub fn entry_block(&self) -> &Block { + &self.blocks[self.entry_block.0] + } + + /// Return the number of blocks + pub fn num_blocks(&self) -> usize { + self.blocks.len() + } + + pub fn assume_single_ractor_mode(&mut self, block: BlockId, state: InsnId) -> bool { + if unsafe { rb_jit_multi_ractor_p() } { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::SingleRactorMode, state }); + true + } + } + + /// Assume that only the root box is active, so we can safely read from the prime classext. + /// Returns true if safe to assume so and emits a PatchPoint. + pub fn assume_root_box(&mut self, block: BlockId, state: InsnId) -> bool { + if invariants::non_root_box_created() { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::RootBoxOnly, state }); + true + } + } + + /// Assume that objects of a given class will have no singleton class. + /// Returns true if safe to assume so and emits a PatchPoint. + /// Returns false if we've already seen a singleton class for this class, + /// to avoid an invalidation loop. + pub fn assume_no_singleton_classes(&mut self, block: BlockId, klass: VALUE, state: InsnId) -> bool { + if !klass.instance_can_have_singleton_class() { + // This class can never have a singleton class, so no patchpoint needed. + return true; + } + if klass.is_singleton_class() { + // When a value has a singleton class, its effective class can't change anymore. + // No patchpoint needed. + return true; + } + if self.was_invalidated_for_singleton_class_creation && invariants::has_singleton_class_of(klass) { + // A previous compilation of this ISEQ was invalidated for singleton class + // creation. Avoid repeating the invalidation. + return false; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state }); + true + } + + pub fn count(&mut self, block: BlockId, counter: Counter) { + if get_option!(stats) { + self.push_insn(block, Insn::IncrCounter(counter)); + } + } + + /// Return a copy of the instruction where the instruction and its operands have been read from + /// the union-find table (to find the current most-optimized version of this instruction). See + /// [`UnionFind`] for more. + /// + /// This is _the_ function for reading [`Insn`]. Use frequently. Example: + /// + /// ```rust + /// match func.find(insn_id) { + /// IfTrue { val, target } if func.is_truthy(val) => { + /// let jump = self.new_insn(Insn::Jump(target)); + /// func.make_equal_to(insn_id, jump); + /// } + /// _ => {} + /// } + /// ``` + pub fn find(&self, insn_id: InsnId) -> Insn { + macro_rules! find { + ( $x:expr ) => { + { + // TODO(max): Figure out why borrow_mut().find() causes `already borrowed: + // BorrowMutError` + self.union_find.borrow().find_const($x) + } + }; + } + let insn_id = find!(insn_id); + let mut result = self.insns[insn_id.0].clone(); + result.for_each_operand_mut(&mut |operand: &mut InsnId| { + *operand = find!(*operand); + }); + result + } + + /// Update DynamicSendReason for the instruction at insn_id + fn set_dynamic_send_reason(&mut self, insn_id: InsnId, dynamic_send_reason: SendFallbackReason) { + use Insn::*; + // Always set the reason: convert_no_profile_sends depends on it to identify + // sends that should be converted to side exits for exit-based recompilation. + match self.insns.get_mut(insn_id.0).unwrap() { + Send { reason, .. } + | SendForward { reason, .. } + | InvokeSuper { reason, .. } + | InvokeSuperForward { reason, .. } + | InvokeBlock { reason, .. } + => *reason = dynamic_send_reason, + _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) + } + } + + /// Replace `insn` with the new instruction `replacement`, which will get appended to `insns`. + fn make_equal_to(&mut self, insn: InsnId, replacement: InsnId) { + assert!(self.insns[insn.0].has_output(), + "Don't use make_equal_to for instruction with no output"); + assert!(self.insns[replacement.0].has_output(), + "Can't replace instruction that has output with instruction that has no output"); + // Don't push it to the block + self.union_find.borrow_mut().make_equal_to(insn, replacement); + } + + pub fn type_of(&self, insn: InsnId) -> Type { + assert!(self.insns[insn.0].has_output()); + self.insn_types[self.union_find.borrow_mut().find(insn).0] + } + + /// Check if the type of `insn` is a subtype of `ty`. + pub fn is_a(&self, insn: InsnId, ty: Type) -> bool { + self.type_of(insn).is_subtype(ty) + } + + fn infer_type(&self, insn: InsnId) -> Type { + assert!(self.insns[insn.0].has_output()); + match &self.insns[insn.0] { + Insn::Param => unimplemented!("params should not be present in block.insns"), + Insn::LoadArg { val_type, .. } => *val_type, + Insn::SetGlobal { .. } | Insn::Jump(_) | Insn::Entries { .. } | Insn::EntryPoint { .. } + | Insn::CondBranch { .. } | Insn::Return { .. } | Insn::Throw { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } + | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::BreakPoint | Insn::Unreachable + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => + panic!("Cannot infer type of instruction with no output: {}. See Insn::has_output().", self.insns[insn.0]), + Insn::Const { val: Const::Value(val) } => Type::from_value(*val), + Insn::Const { val: Const::CBool(val) } => Type::from_cbool(*val), + Insn::Const { val: Const::CInt8(val) } => Type::from_cint(types::CInt8, *val as i64), + Insn::Const { val: Const::CInt16(val) } => Type::from_cint(types::CInt16, *val as i64), + Insn::Const { val: Const::CInt32(val) } => Type::from_cint(types::CInt32, *val as i64), + Insn::Const { val: Const::CInt64(val) } => Type::from_cint(types::CInt64, *val), + Insn::Const { val: Const::CUInt8(val) } => Type::from_cint(types::CUInt8, *val as i64), + Insn::Const { val: Const::CUInt16(val) } => Type::from_cint(types::CUInt16, *val as i64), + Insn::Const { val: Const::CUInt32(val) } => Type::from_cint(types::CUInt32, *val as i64), + Insn::Const { val: Const::CAttrIndex(val) } => Type::from_cint(types::CAttrIndex, *val as i64), + Insn::Const { val: Const::CShape(val) } => Type::from_cint(types::CShape, val.0 as i64), + Insn::Const { val: Const::CUInt64(val) } => Type::from_cint(types::CUInt64, *val as i64), + Insn::Const { val: Const::CPtr(val) } => Type::from_cptr(*val), + Insn::Const { val: Const::CDouble(val) } => Type::from_double(*val), + Insn::Test { val } if self.type_of(*val).is_known_falsy() => Type::from_cbool(false), + Insn::Test { val } if self.type_of(*val).is_known_truthy() => Type::from_cbool(true), + Insn::Test { .. } => types::CBool, + Insn::IsNil { val } if self.is_a(*val, types::NilClass) => Type::from_cbool(true), + Insn::IsNil { val } if !self.type_of(*val).could_be(types::NilClass) => Type::from_cbool(false), + Insn::IsNil { .. } => types::CBool, + Insn::IsMethodCfunc { .. } => types::CBool, + Insn::IsBitEqual { .. } => types::CBool, + Insn::IsBitNotEqual { .. } => types::CBool, + Insn::BoxBool { .. } => types::BoolExact, + Insn::BoxFixnum { .. } => types::Fixnum, + Insn::UnboxFixnum { val } => self + .type_of(*val) + .fixnum_value() + .map_or(types::CInt64, |fixnum| Type::from_cint(types::CInt64, fixnum)), + Insn::FixnumAref { .. } => types::Fixnum, + Insn::StringCopy { .. } => types::StringExact, + Insn::StringIntern { .. } => types::Symbol, + Insn::StringConcat { .. } => types::StringExact, + Insn::StringGetbyte { .. } => types::Fixnum, + Insn::StringSetbyteFixnum { .. } => types::Fixnum, + Insn::StringAppend { .. } => types::StringExact, + Insn::StringAppendCodepoint { .. } => types::StringExact, + Insn::StringEqual { .. } => types::BoolExact, + Insn::ToRegexp { .. } => types::RegexpExact, + Insn::NewArray { .. } => types::ArrayExact, + Insn::ArrayDup { .. } => types::ArrayExact, + Insn::ArrayAref { .. } => types::BasicObject, + Insn::ArrayPop { .. } => types::BasicObject, + Insn::ArrayLength { .. } => types::CInt64, + Insn::AdjustBounds { .. } => types::CInt64, + Insn::HashAref { .. } => types::BasicObject, + Insn::NewHash { .. } => types::HashExact, + Insn::HashDup { .. } => types::HashExact, + Insn::NewRange { .. } => types::RangeExact, + Insn::NewRangeFixnum { .. } => types::RangeExact, + Insn::ObjectAlloc { .. } => types::HeapBasicObject, + Insn::ObjectAllocClass { class, .. } => Type::from_class(*class), + &Insn::CCallWithFrame { return_type, .. } => return_type, + Insn::CCall { return_type, .. } => *return_type, + &Insn::CCallVariadic { return_type, .. } => return_type, + Insn::CheckMatch { .. } => types::BasicObject, + Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), + Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), + Insn::HasType { .. } => types::CBool, + Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), + Insn::GuardAnyBitSet { val, .. } => self.type_of(*val), + Insn::GuardNoBitsSet { val, .. } => self.type_of(*val), + Insn::GuardLess { left, .. } => self.type_of(*left), + Insn::GuardGreaterEq { left, .. } => self.type_of(*left), + Insn::FixnumAdd { .. } => types::Fixnum, + Insn::FixnumSub { .. } => types::Fixnum, + Insn::FixnumMult { .. } => types::Fixnum, + Insn::FixnumDiv { .. } => types::Fixnum, + Insn::FixnumMod { .. } => types::Fixnum, + Insn::FloatAdd { .. } => types::Float, + Insn::FloatSub { .. } => types::Float, + Insn::FloatMul { .. } => types::Float, + Insn::FloatDiv { .. } => types::Float, + Insn::FloatToInt { .. } => types::Integer, + Insn::FixnumEq { .. } => types::BoolExact, + Insn::FixnumNeq { .. } => types::BoolExact, + Insn::FixnumLt { .. } => types::BoolExact, + Insn::FixnumLe { .. } => types::BoolExact, + Insn::FixnumGt { .. } => types::BoolExact, + Insn::FixnumGe { .. } => types::BoolExact, + Insn::FixnumAnd { .. } => types::Fixnum, + Insn::FixnumOr { .. } => types::Fixnum, + Insn::FixnumXor { .. } => types::Fixnum, + Insn::IntAnd { .. } => types::CInt64, + Insn::IntOr { left, .. } => self.type_of(*left).unspecialized(), + Insn::FixnumLShift { .. } => types::Fixnum, + Insn::FixnumRShift { .. } => types::Fixnum, + Insn::PutSpecialObject { .. } => types::BasicObject, + Insn::SendDirect { .. } => types::BasicObject, + Insn::Send { .. } => types::BasicObject, + Insn::SendForward { .. } => types::BasicObject, + Insn::InvokeSuper { .. } => types::BasicObject, + Insn::InvokeSuperForward { .. } => types::BasicObject, + Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeBlockIfunc { .. } => types::BasicObject, + Insn::InvokeProc { .. } => types::BasicObject, + Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), + Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::DefinedIvar { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::GetConstant { .. } => types::BasicObject, + Insn::GetConstantPath { .. } => types::BasicObject, + Insn::IsBlockGiven { .. } => types::BoolExact, + Insn::FixnumBitCheck { .. } => types::BoolExact, + Insn::ArrayMax { .. } => types::BasicObject, + Insn::ArrayMin { .. } => types::BasicObject, + Insn::ArrayInclude { .. } => types::BoolExact, + Insn::ArrayPackBuffer { .. } => types::String, + Insn::DupArrayInclude { .. } => types::BoolExact, + Insn::ArrayHash { .. } => types::Fixnum, + Insn::GetGlobal { .. } => types::BasicObject, + Insn::GetIvar { .. } => types::BasicObject, + Insn::LoadPC => types::CPtr, + Insn::LoadSP => types::CPtr, + Insn::LoadEC => types::CPtr, + Insn::GetEP { .. } => types::CPtr, + Insn::LoadSelf => types::BasicObject, + &Insn::LoadField { return_type, .. } => return_type, + Insn::GetSpecialSymbol { .. } => types::StringExact.union(types::NilClass), + Insn::GetSpecialNumber { .. } => types::StringExact.union(types::NilClass), + Insn::GetClassVar { .. } => types::BasicObject, + Insn::ToNewArray { .. } => types::ArrayExact, + Insn::ToArray { .. } => types::ArrayExact, + Insn::ObjToString { .. } => types::BasicObject, + Insn::AnyToString { .. } => types::String, + Insn::IsBlockParamModified { .. } => types::CBool, + Insn::GetBlockParam { .. } => types::BasicObject, + // The type of Snapshot doesn't really matter; it's never materialized. It's used only + // as a reference for FrameState, which we use to generate side-exit code. + Insn::Snapshot { .. } => types::Any, + Insn::IsA { .. } => types::BoolExact, + } + } + + /// Set self.param_types. They are copied to the param types of jit_entry_blocks. + fn set_param_types(&mut self) { + let iseq = self.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + self.param_types.push(types::BasicObject); // self + for local_idx in 0..param_size { + let param_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact // Rest parameters are always ArrayExact + } else { + types::BasicObject + }; + self.param_types.push(param_type); + } + } + + /// Copy self.param_types to the param types of jit_entry_blocks. + fn copy_param_types(&mut self) { + for jit_entry_block in self.jit_entry_blocks.iter() { + let entry_params = self.blocks[jit_entry_block.0].params.iter(); + let param_types = self.param_types.iter(); + assert!( + param_types.len() >= entry_params.len(), + "param types should be initialized before type inference", + ); + for (param, param_type) in std::iter::zip(entry_params, param_types) { + // We know that function parameters are BasicObject or some subclass + self.insn_types[param.0] = *param_type; + } + } + } + + fn infer_types(&mut self) { + // Reset all types + self.insn_types.fill(types::Empty); + + // Fill entry parameter types + self.copy_param_types(); + + // Assign `new_type` to `insn` if it differs from the recorded type. + // Returns `true` if a write actually happened, `false` if the type + // Macro instead of closure so the borrow checker sees individual field + // accesses rather than an `&mut self` borrow that conflicts with + // `&self.insns` held by an outer match. + macro_rules! set_type { + ($insn:expr, $new_type:expr) => {{ + let insn = $insn; + let new_type = $new_type; + let old_type = self.insn_types[self.union_find.borrow_mut().find(insn).0]; + if old_type.bit_equal(new_type) { + false + } else { + self.insn_types[insn.0] = new_type; + true + } + }}; + } + + let mut reachable = BlockSet::with_capacity(self.blocks.len()); + reachable.insert(self.entries_block); + + // Walk the graph, computing types until fixpoint + let rpo = self.reverse_post_order(); + loop { + let mut changed = false; + for &block in &rpo { + if !reachable.get(block) { continue; } + for i in 0..self.blocks[block.0].insns.len() { + let insn_id = self.blocks[block.0].insns[i]; + // Instructions without output, including branch instructions, can't be targets + // of make_equal_to, so we don't need find() here. + let insn_type = match &self.insns[insn_id.0] { + Insn::CondBranch { val, if_true, if_false } => { + assert!(!self.type_of(*val).bit_equal(types::Empty)); + if self.type_of(*val).could_be(Type::from_cbool(true)) { + reachable.insert(if_true.target); + // Snapshot arg types before any param updates so phi-style + // updates happen in parallel (the args of a self-loop may name + // params of `target` itself). + let arg_types: Vec<Type> = if_true.args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { + let param = self.blocks[if_true.target.0].params[idx]; + changed |= set_type!(param, self.type_of(param).union(arg_type)); + } + } + if self.type_of(*val).could_be(Type::from_cbool(false)) { + reachable.insert(if_false.target); + let arg_types: Vec<Type> = if_false.args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { + let param = self.blocks[if_false.target.0].params[idx]; + changed |= set_type!(param, self.type_of(param).union(arg_type)); + } + } + continue; + } + &Insn::Jump(BranchEdge { target, ref args }) => { + reachable.insert(target); + let arg_types: Vec<Type> = args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { + let param = self.blocks[target.0].params[idx]; + changed |= set_type!(param, self.type_of(param).union(arg_type)); + } + continue; + } + Insn::Entries { targets } => { + for &target in targets { + reachable.insert(target); + } + continue; + } + insn if insn.has_output() => self.infer_type(insn_id), + _ => continue, + }; + changed |= set_type!(insn_id, insn_type); + } + } + if !changed { + break; + } + } + } + + fn chase_insn(&self, insn: InsnId) -> InsnId { + let id = self.union_find.borrow().find_const(insn); + match self.insns[id.0] { + Insn::GuardType { val, .. } + | Insn::GuardBitEquals { val, .. } + | Insn::GuardAnyBitSet { val, .. } + | Insn::GuardNoBitsSet { val, .. } => self.chase_insn(val), + | Insn::RefineType { val, .. } => self.chase_insn(val), + _ => id, + } + } + + /// Return the profiled type of the HIR instruction at the given ISEQ instruction + /// index, if it is known to be monomorphic or skewed polymorphic. This historical type + /// record is not a guarantee and must be checked with a GuardType or similar. + fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: YarvInsnIdx) -> Option<ProfiledType> { + match self.resolve_receiver_type_from_profile(insn, iseq_insn_idx) { + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => Some(profiled_type), + _ => None, + } + } + + /// Prepare arguments for a direct send, handling keyword argument reordering and default synthesis. + /// Returns the (state, processed_args, kw_bits) to use for the SendDirect instruction, + /// or Err with the fallback reason if direct send isn't possible. + fn prepare_direct_send_args( + &mut self, + block: BlockId, + args: &[InsnId], + ci: *const rb_callinfo, + iseq: IseqPtr, + state: InsnId, + ) -> Result<(InsnId, Vec<InsnId>, u32), SendFallbackReason> { + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (processed_args, caller_argc, kw_bits) = self.setup_keyword_arguments(block, args, kwarg, iseq)?; + + // If args were reordered or synthesized, create a new snapshot with the updated stack + let send_state = if processed_args != args { + let new_state = self.frame_state(state).with_replaced_args(&processed_args, caller_argc); + self.push_insn(block, Insn::Snapshot { state: new_state }) + } else { + state + }; + + Ok((send_state, processed_args, kw_bits)) + } + + /// Reorder keyword arguments to match the callee's expected order, and synthesize + /// default values for any optional keywords not provided by the caller. + /// + /// The output always contains all of the callee's keyword arguments (required + optional), + /// so the returned vec may be larger than the input args. + /// + /// Returns Ok with (processed_args, caller_argc, kw_bits) if successful, or Err with the fallback reason if not. + /// - caller_argc: number of arguments the caller actually pushed (for stack calculations) + /// - kw_bits: bitmask indicating which optional keywords were NOT provided by the caller + /// (used by checkkeyword to determine if non-constant defaults need evaluation) + fn setup_keyword_arguments( + &mut self, + block: BlockId, + args: &[InsnId], + kwarg: *const rb_callinfo_kwarg, + iseq: IseqPtr, + ) -> Result<(Vec<InsnId>, usize, u32), SendFallbackReason> { + let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if callee_keyword.is_null() { + if !kwarg.is_null() { + // Caller is passing kwargs but callee doesn't expect them. + return Err(SendDirectKeywordMismatch); + } + // Neither caller nor callee have keywords - nothing to do + return Ok((args.to_vec(), args.len(), 0)); + } + + // kwarg may be null if caller passes no keywords but callee has optional keywords + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let callee_kw_count = unsafe { (*callee_keyword).num } as usize; + + // When there are 31+ keywords, CRuby uses a hash instead of a fixnum bitmask + // for kw_bits. Fall back to VM dispatch for this rare case. + if callee_kw_count >= VM_KW_SPECIFIED_BITS_MAX as usize { + return Err(SendDirectTooManyKeywords); + } + + let callee_kw_required = unsafe { (*callee_keyword).required_num } as usize; + let callee_kw_table = unsafe { (*callee_keyword).table }; + let default_values = unsafe { (*callee_keyword).default_values }; + + // Caller can't provide more keywords than callee expects (no **kwrest support yet). + if caller_kw_count > callee_kw_count { + return Err(SendDirectKeywordCountMismatch); + } + + // The keyword arguments are the last arguments in the args vector. + let kw_args_start = args.len() - caller_kw_count; + + // Build a mapping from caller keywords to their positions. + let mut caller_kw_order: Vec<ID> = Vec::with_capacity(caller_kw_count); + if !kwarg.is_null() { + for i in 0..caller_kw_count { + let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; + let id = unsafe { rb_sym2id(sym) }; + caller_kw_order.push(id); + } + } + + // Verify all caller keywords are expected by callee (no unknown keywords). + // Without **kwrest, unexpected keywords should raise ArgumentError at runtime. + for &caller_id in &caller_kw_order { + let mut found = false; + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + if caller_id == expected_id { + found = true; + break; + } + } + if !found { + // Caller is passing an unknown keyword - this will raise ArgumentError. + // Fall back to VM dispatch to handle the error. + return Err(SendDirectKeywordMismatch); + } + } + + // Reorder keyword arguments to match callee expectation. + // Track which optional keywords were not provided via kw_bits. + let mut kw_bits: u32 = 0; + let mut reordered_kw_args: Vec<InsnId> = Vec::with_capacity(callee_kw_count); + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + + // Find where this keyword is in the caller's order + let mut found = false; + for (j, &caller_id) in caller_kw_order.iter().enumerate() { + if caller_id == expected_id { + reordered_kw_args.push(args[kw_args_start + j]); + found = true; + break; + } + } + + if !found { + // Required keyword not provided by caller which will raise an ArgumentError. + if i < callee_kw_required { + return Err(SendDirectMissingKeyword); + } + + // Optional keyword not provided - use default value + let default_idx = i - callee_kw_required; + let default_value = unsafe { *default_values.add(default_idx) }; + + if default_value == Qundef { + // Non-constant default (e.g., `def foo(a: compute())`). + // Set the bit so checkkeyword knows to evaluate the default at runtime. + // Push Qnil as a placeholder; the callee's checkkeyword will detect this + // and branch to evaluate the default expression. + kw_bits |= 1 << default_idx; + let nil_insn = self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + reordered_kw_args.push(nil_insn); + } else { + // Constant default value - use it directly + let const_insn = self.push_insn(block, Insn::Const { val: Const::Value(default_value) }); + reordered_kw_args.push(const_insn); + } + } + } + + // Replace the keyword arguments with the reordered ones. + // Keep track of the original caller argc for stack calculations. + let caller_argc = args.len(); + let mut processed_args = args[..kw_args_start].to_vec(); + processed_args.extend(reordered_kw_args); + Ok((processed_args, caller_argc, kw_bits)) + } + + /// Resolve the receiver type for method dispatch optimization. + /// + /// Takes the receiver's Type, receiver HIR instruction, and ISEQ instruction index. + /// First checks if the receiver's class is statically known, otherwise consults profile data. + /// + /// Returns: + /// - `StaticallyKnown` if the receiver's exact class is known at compile-time + /// - Result of [`Self::resolve_receiver_type_from_profile`] if we need to check profile data + fn resolve_receiver_type(&self, recv: InsnId, recv_type: Type, insn_idx: YarvInsnIdx) -> ReceiverTypeResolution { + match self.resolve_receiver_type_from_profile(recv, insn_idx) { + ReceiverTypeResolution::NoProfile => { + // Use known type information as a fallback because it doesn't have shape + // information (and we can generally eliminate duplicate guards). + if let Some(class) = recv_type.runtime_exact_ruby_class() { + ReceiverTypeResolution::StaticallyKnown { class } + } else { + ReceiverTypeResolution::NoProfile + } + } + resolution => resolution, + } + } + + fn polymorphic_summary(&self, profiles: &ProfileOracle, recv: InsnId, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> { + let Some(entries) = profiles.types.get(&insn_idx) else { + return None; + }; + let recv = self.chase_insn(recv); + for (entry_insn, entry_type_summary) in entries { + if self.union_find.borrow().find_const(*entry_insn) == recv { + if entry_type_summary.is_polymorphic() || entry_type_summary.is_skewed_polymorphic() { + return Some(entry_type_summary.clone()); + } + return None; + } + } + None + } + + /// Resolve the receiver type for method dispatch optimization from profile data. + /// + /// Returns: + /// - `Monomorphic`/`SkewedPolymorphic` if we have usable profile data + /// - `Polymorphic` if the receiver has multiple types + /// - `Megamorphic`/`SkewedMegamorphic` if the receiver has too many types to optimize + /// (SkewedMegamorphic may be optimized in the future, but for now we don't) + /// - `NoProfile` if we have no type information + fn resolve_receiver_type_from_profile(&self, recv: InsnId, insn_idx: YarvInsnIdx) -> ReceiverTypeResolution { + let Some(profiles) = self.profiles.as_ref() else { + return ReceiverTypeResolution::NoProfile; + }; + let Some(entries) = profiles.types.get(&insn_idx) else { + return ReceiverTypeResolution::NoProfile; + }; + let recv = self.chase_insn(recv); + + for (entry_insn, entry_type_summary) in entries { + if self.chase_insn(*entry_insn) == recv { + if entry_type_summary.is_monomorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::Monomorphic { profiled_type }; + } else if entry_type_summary.is_skewed_polymorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedPolymorphic { profiled_type }; + } else if entry_type_summary.is_skewed_megamorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedMegamorphic { profiled_type }; + } else if entry_type_summary.is_polymorphic() { + return ReceiverTypeResolution::Polymorphic; + } else if entry_type_summary.is_megamorphic() { + return ReceiverTypeResolution::Megamorphic; + } + } + } + + ReceiverTypeResolution::NoProfile + } + + pub fn assume_expected_cfunc(&mut self, block: BlockId, class: VALUE, method_id: ID, cfunc: *mut c_void, state: InsnId) -> bool { + let cme = unsafe { rb_callable_method_entry(class, method_id) }; + if cme.is_null() { return false; } + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { return false; } + if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc { + return false; + } + self.gen_patch_points_for_optimized_ccall(block, class, method_id, cme, state); + if !self.assume_no_singleton_classes(block, class, state) { + return false; + } + true + } + + pub fn likely_a(&self, val: InsnId, ty: Type, state: InsnId) -> bool { + if self.type_of(val).is_subtype(ty) { + return true; + } + let frame_state = self.frame_state(state); + let iseq_insn_idx = frame_state.insn_idx; + let Some(profiled_type) = self.profiled_type_of_at(val, iseq_insn_idx) else { + return false; + }; + Type::from_profiled_type(profiled_type).is_subtype(ty) + } + + pub fn coerce_to(&mut self, block: BlockId, val: InsnId, guard_type: Type, state: InsnId) -> InsnId { + if self.is_a(val, guard_type) { return val; } + self.push_insn(block, Insn::GuardType { val, guard_type, state }) + } + + fn count_complex_call_features(&mut self, block: BlockId, ci_flags: c_uint) { + use Counter::*; + if 0 != ci_flags & VM_CALL_ARGS_SPLAT { self.count(block, complex_arg_pass_caller_splat); } + if 0 != ci_flags & VM_CALL_ARGS_BLOCKARG { self.count(block, complex_arg_pass_caller_blockarg); } + if 0 != ci_flags & VM_CALL_KWARG { self.count(block, complex_arg_pass_caller_kwarg); } + if 0 != ci_flags & VM_CALL_KW_SPLAT { self.count(block, complex_arg_pass_caller_kw_splat); } + if 0 != ci_flags & VM_CALL_TAILCALL { self.count(block, complex_arg_pass_caller_tailcall); } + if 0 != ci_flags & VM_CALL_SUPER { self.count(block, complex_arg_pass_caller_super); } + if 0 != ci_flags & VM_CALL_ZSUPER { self.count(block, complex_arg_pass_caller_zsuper); } + if 0 != ci_flags & VM_CALL_FORWARDING { self.count(block, complex_arg_pass_caller_forwarding); } + } + + fn rewrite_if_frozen(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, klass: u32, bop: u32, state: InsnId) { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + // If the basic operation is already redefined, we cannot optimize it. + self.set_dynamic_send_reason(orig_insn_id, SendWithoutBlockBopRedefined); + self.push_insn_id(block, orig_insn_id); + return; + } + let self_type = self.type_of(self_val); + if let Some(obj) = self_type.ruby_object() { + if obj.is_frozen() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state }); + self.make_equal_to(orig_insn_id, self_val); + return; + } + } + self.push_insn_id(block, orig_insn_id); + } + + pub fn try_inline_object_alloc(&mut self, block: BlockId, recv: InsnId, state: InsnId) -> Option<InsnId> { + let recv_type = self.type_of(recv); + if recv_type.is_subtype(types::Class) { + if let Some(class) = recv_type.ruby_object() { + // See class_get_alloc_func in object.c; if the class isn't initialized, is + // a singleton class, or has a custom allocator, ObjectAlloc might raise an + // exception or run arbitrary code. + if class_has_leaf_allocator(class) { + return Some(self.push_insn(block, Insn::ObjectAllocClass { class, state })); + } + } + } + None + } + + fn try_rewrite_freeze(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::ArrayExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::HashExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, HASH_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else { + self.push_insn_id(block, orig_insn_id); + } + } + + fn try_rewrite_uminus(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_UMINUS, state); + } else { + self.push_insn_id(block, orig_insn_id); + } + } + + pub fn load_rbasic_flags(&mut self, block: BlockId, recv: InsnId) -> InsnId { + // Technically this also includes the shape (_shape_id) because the (shape, flags) tuple is + // a (u32, u32) inside a u64 at RUBY_OFFSET_RBASIC_FLAGS (offset 0). It's fine to load the + // shape alongside the flags, but make sure not to *store* the shape accidentally by + // writing a u64. + self.push_insn(block, Insn::LoadField { recv, id: FieldName::RBASIC_FLAGS, offset: RUBY_OFFSET_RBASIC_FLAGS, return_type: types::CUInt64 }) + } + + fn load_ep_flags(&mut self, block: BlockId, ep: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { recv: ep, id: FieldName::VM_ENV_DATA_INDEX_FLAGS, offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), return_type: types::CUInt64 }) + } + + pub fn guard_not_frozen(&mut self, block: BlockId, recv: InsnId, state: InsnId) { + let flags = self.load_rbasic_flags(block, recv); + self.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(RUBY_FL_FREEZE as u64), mask_name: Some(ID!(RUBY_FL_FREEZE)), reason: SideExitReason::GuardNotFrozen, state }); + } + + pub fn guard_not_shared(&mut self, block: BlockId, recv: InsnId, state: InsnId) { + let flags = self.load_rbasic_flags(block, recv); + self.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(RUBY_ELTS_SHARED as u64), mask_name: Some(ID!(RUBY_ELTS_SHARED)), reason: SideExitReason::GuardNotShared, state }); + } + + fn get_local_from_ep( + &mut self, + block: BlockId, + ep: InsnId, + ep_offset: u32, + level: u32, + return_type: Type, + ) -> InsnId { + let local_id = get_local_var_id(self.iseq, level, ep_offset); + let ep_offset = i32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to i32")); + let offset = -(SIZEOF_VALUE_I32 * ep_offset); + + self.push_insn(block, Insn::LoadField { + recv: ep, + id: local_id.into(), + offset, + return_type, + }) + } + + fn get_local_from_sp( + &mut self, + block: BlockId, + sp: InsnId, + ep_offset: u32, + return_type: Type, + ) -> InsnId { + let local_id = get_local_var_id(self.iseq, 0, ep_offset); + let ep_offset = i32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to i32")); + let offset = -(SIZEOF_VALUE_I32 * (ep_offset + 1)); + + self.push_insn(block, Insn::LoadField { + recv: sp, + id: local_id.into(), + offset, + return_type, + }) + } + + /// Rewrite eligible Send opcodes into SendDirect + /// opcodes if we know the target ISEQ statically. This removes run-time method lookups and + /// opens the door for inlining. + /// Also try and inline constant caches, specialize object allocations, and more. + /// Calls to C functions are handled separately in optimize_c_calls. + fn type_specialize(&mut self) { + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::Send { recv, block: None, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.is_empty() => + self.try_rewrite_freeze(block, insn_id, recv, state), + Insn::Send { recv, block: None, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => + self.try_rewrite_uminus(block, insn_id, recv, state), + Insn::Send { mut recv, cd, state, block: send_block, args, .. } => { + let has_block = send_block.is_some(); + let frame_state = self.frame_state(state); + let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + let reason = if has_block { SendMegamorphic } else { SendWithoutBlockMegamorphic }; + self.set_dynamic_send_reason(insn_id, reason); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + let reason = if has_block { SendPolymorphic } else { SendWithoutBlockPolymorphic }; + self.set_dynamic_send_reason(insn_id, reason); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); + continue; + } + }; + let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + + let flags = unsafe { rb_vm_ci_flag(ci) }; + + let mid = unsafe { vm_ci_mid(ci) }; + // Do method lookup + let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; + if cme.is_null() { + let reason = if has_block { SendNotOptimizedMethodType(MethodType::Null) } else { SendWithoutBlockNotOptimizedMethodType(MethodType::Null) }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + // Load an overloaded cme if applicable. See vm_search_cc(). + // It allows you to use a faster ISEQ if possible. + cme = unsafe { rb_check_overloaded_cme(cme, ci) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + let reason = if has_block { SendNotOptimizedNeedPermission } else { SendWithoutBlockNotOptimizedNeedPermission }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + } + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + + // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendDirect`. + // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) handle their own argument constraints (e.g., kw_splat for Proc call). + if def_type != VM_METHOD_TYPE_OPTIMIZED && unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + + if def_type == VM_METHOD_TYPE_ISEQ { + // TODO(max): Allow non-iseq; cache cme + // Only specialize positional-positional calls + // TODO(max): Handle other kinds of parameter passing + let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice(), has_block) { + self.push_insn_id(block, insn_id); continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + + // Add PatchPoint for method redefinition + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + + // Add GuardType for profiled receiver + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let send_direct = self.push_insn(block, Insn::SendDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state, block: send_block }); + self.make_equal_to(insn_id, send_direct); + } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + // Target ISEQ bmethods. Can't handle for example, `define_method(:foo, &:foo)` + // which makes a `block_type_symbol` bmethod. + if proc_block.type_ != block_type_iseq { + self.set_dynamic_send_reason(insn_id, BmethodNonIseqProc); + self.push_insn_id(block, insn_id); continue; + } + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let iseq = unsafe { *capture.code.iseq.as_ref() }; + + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice(), has_block) { + self.push_insn_id(block, insn_id); continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + // Patch points: + // Check for "defined with an un-shareable Proc in a different Ractor" + if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { + // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let send_direct = self.push_insn(block, Insn::SendDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state, block: None }); + self.make_equal_to(insn_id, send_direct); + } else if !has_block && def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + let getivar = self.push_insn(block, Insn::GetIvar { self_val: recv, id, ic: std::ptr::null(), state }); + self.make_equal_to(insn_id, getivar); + } else if let (false, VM_METHOD_TYPE_ATTRSET, &[val]) = (has_block, def_type, args.as_slice()) { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state }); + self.make_equal_to(insn_id, val); + } else if !has_block && def_type == VM_METHOD_TYPE_OPTIMIZED { + let opt_type: OptimizedMethodType = unsafe { get_cme_def_body_optimized_type(cme) }.into(); + match (opt_type, args.as_slice()) { + (OptimizedMethodType::Call, _) => { + if flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KWARG) != 0 { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let kw_splat = flags & VM_CALL_KW_SPLAT != 0; + let invoke_proc = self.push_insn(block, Insn::InvokeProc { recv, args: args.clone(), state, kw_splat }); + self.make_equal_to(insn_id, invoke_proc); + } + (OptimizedMethodType::StructAref, &[]) | (OptimizedMethodType::StructAset, &[_]) => { + if unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + let index: i32 = unsafe { get_cme_def_body_optimized_index(cme) } + .try_into() + .unwrap(); + // We are going to use an encoding that takes a 4-byte immediate which + // limits the offset to INT32_MAX. + { + let native_index = (index as i64) * (SIZEOF_VALUE as i64); + if native_index > (i32::MAX as i64) { + self.set_dynamic_send_reason(insn_id, TooManyArgsForLir); + self.push_insn_id(block, insn_id); continue; + } + } + // Get the profiled type to check if the fields is embedded or heap allocated. + let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { + // No (monomorphic/skewed polymorphic) profile info + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + }; + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + // All structs from the same Struct class should have the same + // length. So if our recv is embedded all runtime + // structs of the same class should be as well, and the same is + // true of the converse. + // + // No need for a GuardShape. + if let OptimizedMethodType::StructAset = opt_type { + self.guard_not_frozen(block, recv, state); + } + + let (target, offset) = if is_embedded { + let offset = RUBY_OFFSET_RSTRUCT_AS_ARY + (SIZEOF_VALUE_I32 * index); + (recv, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv, id: FieldName::as_heap, offset: RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * index; + (as_heap, offset) + }; + + let replacement = if let (OptimizedMethodType::StructAset, &[val]) = (opt_type, args.as_slice()) { + self.push_insn(block, Insn::StoreField { recv: target, id: mid.into(), offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv, val }); + val + } else { // StructAref + self.push_insn(block, Insn::LoadField { recv: target, id: mid.into(), offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + }, + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType::from(opt_type))); + self.push_insn_id(block, insn_id); continue; + }, + }; + } else { + let reason = if has_block { SendNotOptimizedMethodType(MethodType::from(def_type)) } else { SendWithoutBlockNotOptimizedMethodType(MethodType::from(def_type)) }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + } + Insn::GetConstantPath { ic, state, .. } => { + let idlist: *const ID = unsafe { (*ic).segments }; + let ice = unsafe { (*ic).entry }; + if ice.is_null() { + self.push_insn_id(block, insn_id); continue; + } + let cref_sensitive = !unsafe { (*ice).ic_cref }.is_null(); + if cref_sensitive || !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + // Invalidate output code on any constant writes associated with constants + // referenced after the PatchPoint. + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::StableConstantNames { idlist }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(unsafe { (*ice).value }) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjToString { val, cd, state, .. } => { + if self.is_a(val, types::String) { + // behaves differently from `Send` with `mid:to_s` because ObjToString should not have a patch point for String to_s being redefined + self.make_equal_to(insn_id, val); continue; + } + + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(val, frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); continue + }; + + if recv_type.is_string() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_type.class() }, state }); + let guard = self.push_insn(block, Insn::GuardType { val, guard_type: types::String, state }); + // Infer type so AnyToString can fold off this + self.insn_types[guard.0] = self.infer_type(guard); + self.make_equal_to(insn_id, guard); + } else { + let recv = self.push_insn(block, Insn::GuardType { val, guard_type: Type::from_profiled_type(recv_type), state}); + let send_to_s = self.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], state, reason: ObjToStringNotString }); + self.make_equal_to(insn_id, send_to_s); + } + } + Insn::AnyToString { str, .. } => { + if self.is_a(str, types::String) { + self.make_equal_to(insn_id, str); + } else { + self.push_insn_id(block, insn_id); + } + } + Insn::IsMethodCfunc { val, cd, cfunc, state } if self.type_of(val).ruby_object_known() => { + let class = self.type_of(val).ruby_object().unwrap(); + let cme = unsafe { rb_zjit_vm_search_method(self.iseq.into(), cd as *mut rb_call_data, class) }; + let is_expected_cfunc = unsafe { rb_zjit_cme_is_cfunc(cme, cfunc as *const c_void) }; + let method = unsafe { rb_vm_ci_mid((*cd).ci) }; + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: class, method, cme }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::CBool(is_expected_cfunc) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjectAlloc { val, state } => { + if let Some(replacement) = self.try_inline_object_alloc(block, val, state) { + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } else { + self.push_insn_id(block, insn_id); + } + } + Insn::NewRange { low, high, flag, state } => { + let low_is_fix = self.is_a(low, types::Fixnum); + let high_is_fix = self.is_a(high, types::Fixnum); + + if low_is_fix || high_is_fix { + let low_fix = self.coerce_to(block, low, types::Fixnum, state); + let high_fix = self.coerce_to(block, high, types::Fixnum, state); + let replacement = self.push_insn(block, Insn::NewRangeFixnum { low: low_fix, high: high_fix, flag, state }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + } else { + self.push_insn_id(block, insn_id); + }; + } + Insn::InvokeSuper { recv, cd, blockiseq, args, state, .. } => { + // Helper to emit common guards for super call optimization. + fn emit_super_call_guards( + fun: &mut Function, + block: BlockId, + super_cme: *const rb_callable_method_entry_t, + current_cme: *const rb_callable_method_entry_t, + mid: ID, + state: InsnId, + ) { + fun.push_insn(block, Insn::PatchPoint { + invariant: Invariant::MethodRedefined { + klass: unsafe { (*super_cme).defined_class }, + method: mid, + cme: super_cme + }, + state + }); + + // Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + // Equivalent of GET_LEP() macro. + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + // Load ep[VM_ENV_DATA_INDEX_ME_CREF] + let method_entry = fun.push_insn(block, Insn::LoadField { recv: lep, id: FieldName::VM_ENV_DATA_INDEX_ME_CREF, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF, return_type: types::RubyValue }); + // Guard that it matches the expected CME + fun.push_insn(block, Insn::GuardBitEquals { val: method_entry, expected: Const::Value(current_cme.into()), reason: SideExitReason::GuardSuperMethodEntry, state, recompile: None }); + + let block_handler = fun.push_insn(block, Insn::LoadField { recv: lep, id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::RubyValue }); + fun.push_insn(block, Insn::GuardBitEquals { + val: block_handler, + expected: Const::Value(VALUE(VM_BLOCK_HANDLER_NONE as usize)), + reason: SideExitReason::UnhandledBlockArg, + state, + recompile: None, + }); + } + + // Don't handle calls with literal blocks (e.g., super { ... }) + if !blockiseq.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperCallWithBlock); + continue; + } + + let frame_state = self.frame_state(state); + + // Don't handle super in a block since that needs a loop to find the running CME. + if frame_state.iseq != unsafe { rb_get_iseq_body_local_iseq(frame_state.iseq) } { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperFromBlock); + continue; + } + + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert!(flags & VM_CALL_FCALL != 0); + + // Reject calls with complex argument handling. + if unspecializable_c_call_type(flags) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperComplexArgsPass); + continue; + } + + // Get the profiled CME from the current method. + let Some(profiles) = self.profiles.as_ref() else { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNoProfiles); + continue; + }; + + let Some(current_cme) = profiles.payload.profile.get_super_method_entry(frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); + + // The absence of the super CME could be due to a missing profile, but + // if we've made it this far the value would have been deleted, indicating + // that the call is at least polymorphic and possibly megamorphic. + self.set_dynamic_send_reason(insn_id, SuperPolymorphic); + continue; + }; + + // Get defined_class and method ID from the profiled CME. + let current_defined_class = unsafe { (*current_cme).defined_class }; + let mid = unsafe { get_def_original_id((*current_cme).def) }; + + // Compute superclass: RCLASS_SUPER(RCLASS_ORIGIN(defined_class)) + let superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; + if superclass.nil_p() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperClassNotFound); + continue; + } + + // Look up the super method. + let mut super_cme = unsafe { rb_callable_method_entry(superclass, mid) }; + if super_cme.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetNotFound); + continue; + } + + let mut def_type = unsafe { get_cme_def_type(super_cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + super_cme = unsafe { rb_aliased_callable_method_entry(super_cme) }; + def_type = unsafe { get_cme_def_type(super_cme) }; + } + + if def_type == VM_METHOD_TYPE_ISEQ { + // Check if the super method's parameters support direct send. + // If not, we can't do direct dispatch. + let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; + // TODO: pass Option<blockiseq> to can_direct_send when we start specializing `super { ... }`. + if !can_direct_send(self, block, super_iseq, ci, insn_id, args.as_slice(), false) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetComplexArgsPass); + continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, super_iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Use SendDirect with the super method's CME and ISEQ. + let send_direct = self.push_insn(block, Insn::SendDirect { + recv, + cd, + cme: super_cme, + iseq: super_iseq, + args: processed_args, + kw_bits, + state: send_state, + block: None, + }); + self.make_equal_to(insn_id, send_direct); + + } else if def_type == VM_METHOD_TYPE_CFUNC { + let cfunc = unsafe { get_cme_def_body_cfunc(super_cme) }; + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + + let props = ZJITState::get_method_annotations().get_cfunc_properties(super_cme); + if props.is_none() && get_option!(stats) { + self.count_not_annotated_cfunc(block, super_cme); + } + let props = props.unwrap_or_default(); + + match cfunc_argc { + // C function with fixed argument count. + 0.. => { + // Check argc matches + if args.len() != cfunc_argc as usize { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, ArgcParamMismatch); + continue; + } + + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + + // Use CCallWithFrame for the C function. + let name = unsafe { (*super_cme).called_id }; + let owner = unsafe { (*super_cme).owner }; + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + let ccall = if props.leaf && props.no_gc { + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }) + } else { + if get_option!(stats) { + self.count_not_inlined_cfunc(block, super_cme); + } + self.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc: cfunc_ptr, + recv, + args: args.clone(), + cme: super_cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + block: None, + }) + }; + self.make_equal_to(insn_id, ccall); + } + + // Variadic C function: func(int argc, VALUE *argv, VALUE recv) + -1 => { + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + + // Use CCallVariadic for the variadic C function. + let name = unsafe { (*super_cme).called_id }; + let owner = unsafe { (*super_cme).owner }; + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + let ccall = if props.leaf && props.no_gc { + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }) + } else { + if get_option!(stats) { + self.count_not_inlined_cfunc(block, super_cme); + } + self.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args: args.clone(), + cme: super_cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + block: None, + }) + }; + self.make_equal_to(insn_id, ccall); + } + + // Array-variadic: (self, args_ruby_array). + -2 => { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::Cfunc)); + continue; + } + _ => unreachable!("unknown cfunc argc: {}", cfunc_argc) + } + } else { + // Other method types (not ISEQ or CFUNC) + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::from(def_type))); + continue; + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn inline(&mut self) { + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + // We can inline SendDirect with blockiseq because we are prohibiting `yield` + // and `.call`, which would trigger autosplat. We only inline constants and + // variables and builtin calls. + Insn::SendDirect { recv, iseq, cd, args, state, .. } => { + let call_info = unsafe { (*cd).ci }; + let ci_flags = unsafe { vm_ci_flag(call_info) }; + // .send call is not currently supported for builtins + if ci_flags & VM_CALL_OPT_SEND != 0 { + self.push_insn_id(block, insn_id); continue; + } + let Some(value) = iseq_get_return_value(iseq, None, ci_flags) else { + self.push_insn_id(block, insn_id); continue; + }; + match value { + IseqReturn::LocalVariable(idx) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + self.make_equal_to(insn_id, args[idx as usize]); + } + IseqReturn::Value(value) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(value) }); + self.make_equal_to(insn_id, replacement); + } + IseqReturn::Receiver => { + self.count(block, Counter::inline_iseq_optimized_send_count); + self.make_equal_to(insn_id, recv); + } + IseqReturn::InvokeLeafBuiltin(bf, return_type) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + let replacement = self.push_insn(block, Insn::InvokeBuiltin { + bf, + recv, + args: vec![recv], + state, + leaf: true, + return_type, + }); + self.make_equal_to(insn_id, replacement); + } + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn load_shape(&mut self, block: BlockId, recv: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { + recv, + id: FieldName::shape_id, + offset: unsafe { rb_shape_id_offset() } as i32, + return_type: types::CShape + }) + } + + fn guard_shape(&mut self, block: BlockId, val: InsnId, expected: ShapeId, state: InsnId, recompile: Option<Recompile>) -> InsnId { + self.push_insn(block, Insn::GuardBitEquals { + val, + expected: Const::CShape(expected), + reason: SideExitReason::GuardShape(expected), + state, + recompile, + }) + } + + fn load_ivar_c_call(&mut self, block: BlockId, recv: InsnId, ivar_index: attr_index_t) -> InsnId { + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + let ivar_index_insn = self.push_insn(block, Insn::Const { val: Const::CAttrIndex(ivar_index) }); + self.push_insn(block, Insn::CCall { + cfunc: rb_ivar_get_at_no_ractor_check as *const u8, + recv, + args: vec![ivar_index_insn], + name: ID!(rb_ivar_get_at_no_ractor_check), + owner: Qnil, + return_type: types::BasicObject, + elidable: true }) + } + + fn load_ivar_heap(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let ptr = self.push_insn(block, Insn::LoadField { + recv, id: FieldName::as_heap, + offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, + return_type: types::CPtr, + }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + self.push_insn(block, Insn::LoadField { + recv: ptr, id: id.into(), offset, + return_type: types::BasicObject, + }) + } + + fn load_ivar_embedded(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + self.push_insn(block, Insn::LoadField { + recv, id: id.into(), offset, + return_type: types::BasicObject, + }) + } + + fn load_ivar_from_fields(&mut self, block: BlockId, recv: InsnId, is_embedded: bool, id: ID, ivar_index: attr_index_t) -> InsnId { + if is_embedded { + return self.load_ivar_embedded(block, recv, id, ivar_index); + } else { + return self.load_ivar_heap(block, recv, id, ivar_index); + } + } + + /// This puts a guard that establishes the preconditon for [Self::load_ivar] + fn load_ivar_guard_type(&mut self, block: BlockId, recv: InsnId, recv_type: ProfiledType, state: InsnId) -> InsnId { + if recv_type.flags().is_t_class() { + // Check class first since `Class < Module` + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::Class, state }) + } else if recv_type.flags().is_t_module() { + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::Module, state }) + } else if recv_type.flags().is_t_data() { + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::TData, state }) + } else { + // HeapBasicObject is wider than T_OBJECT, but shapes for T_OBJECTs are in a pool of + // its own and are guaranteed to be different from shapes of any other T_* types. So + // the shape check that follows already covers checking for T_OBJECT. + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::HeapBasicObject, state }) + } + } + + fn load_ivar(&mut self, block: BlockId, self_val: InsnId, recv_type: ProfiledType, id: ID, state: InsnId) -> InsnId { + // Too-complex shapes use hash tables; rb_shape_get_iv_index doesn't support them. + // Callers must filter these out before calling load_ivar. + assert!(!recv_type.shape().is_complex(), "load_ivar called with too-complex shape"); + let mut ivar_index: attr_index_t = 0; + if ! unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + return self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + } + if recv_type.flags().is_t_class() || recv_type.flags().is_t_module() { + // Class/module ivar: load from prime classext's fields_obj + if !self.assume_root_box(block, state) { + // Non-root box active: fall back to C call + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + return self.load_ivar_c_call(block, self_val, ivar_index); + } + // Root box only: load directly from prime classext + let fields_obj = self.push_insn(block, Insn::LoadField { + recv: self_val, id: FieldName::fields_obj, + offset: RCLASS_OFFSET_PRIME_FIELDS_OBJ as i32, + return_type: types::RubyValue, + }); + return self.load_ivar_from_fields(block, fields_obj, recv_type.flags().is_fields_embedded(), id, ivar_index); + } + if recv_type.flags().is_t_data() { + let fields_obj = self.push_insn(block, Insn::LoadField { + recv: self_val, id: FieldName::fields_obj, + offset: TDATA_OFFSET_FIELDS_OBJ as i32, + return_type: types::RubyValue, + }); + return self.load_ivar_from_fields(block, fields_obj, recv_type.flags().is_fields_embedded(), id, ivar_index); + } + if recv_type.flags().is_t_object() { + return self.load_ivar_from_fields(block, self_val, recv_type.flags().is_embedded(), id, ivar_index); + } + // Non-T_OBJECT, non-class/module, non-typed-data: fall back to C call + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + return self.load_ivar_c_call(block, self_val, ivar_index); + } + + fn optimize_getivar(&mut self) { + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::GetIvar { self_val, id, ic: _, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::getivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::getivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::getivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // On the final version, skip GetIvar shape specialization. + // iseq_to_hir already generates polymorphic branches with a + // GetIvar C call fallback for getinstancevariable, so we don't + // need to wrap it again here. + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state, Some(Recompile::ProfileSelf)); + let replacement = self.load_ivar(block, self_val, recv_type, id, state); + self.make_equal_to(insn_id, replacement); + } + Insn::DefinedIvar { self_val, id, pushval, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::definedivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::definedivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.count(block, Counter::definedivar_fallback_not_t_object); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::definedivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // TODO: Support polymorphic DefinedIvar shape-specialized paths. + // https://github.com/Shopify/ruby/issues/980 + // On the final version, keep the DefinedIvar fallback instead of another shape guard. + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state, Some(Recompile::ProfileSelf)); + let mut ivar_index: attr_index_t = 0; + let replacement = if unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + self.push_insn(block, Insn::Const { val: Const::Value(pushval) }) + } else { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::SetIvar { self_val, id, val, state, ic } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::setivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::setivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.count(block, Counter::setivar_fallback_not_t_object); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::setivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_frozen() { + // Can't set ivars on frozen objects + self.count(block, Counter::setivar_fallback_frozen); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // TODO: Support polymorphic SetIvar shape-specialized paths. + // https://github.com/Shopify/ruby/issues/927 + // On the final version, keep the SetIvar fallback instead of another shape guard. + self.push_insn_id(block, insn_id); continue; + } + let mut ivar_index: attr_index_t = 0; + let mut next_shape_id = recv_type.shape(); + if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // Current shape does not contain this ivar; do a shape transition. + let current_shape_id = recv_type.shape(); + let class = recv_type.class(); + // We're only looking at T_OBJECT so ignore all of the imemo stuff. + assert!(recv_type.flags().is_t_object()); + next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id.0, id, class) }); + // If the VM ran out of shapes, or this class generated too many leaf, + // it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table). + let new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id.0) }; + // TODO(max): Is it OK to bail out here after making a shape transition? + if new_shape_complex { + self.count(block, Counter::setivar_fallback_new_shape_complex); + self.push_insn_id(block, insn_id); continue; + } + let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) }; + assert!(ivar_result, "New shape must have the ivar index"); + let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) }; + let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) }; + // If the new shape has a different capacity, or is COMPLEX, we'll have to + // reallocate it. + let needs_extension = next_capacity != current_capacity; + if needs_extension { + self.count(block, Counter::setivar_fallback_new_shape_needs_extension); + self.push_insn_id(block, insn_id); continue; + } + // Fall through to emitting the ivar write + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + // TODO: attr_writer SetIvar has a null inline cache and may target a receiver + // operand other than CFP self. Support it with a reprofile strategy that + // profiles the receiver operand even after the send insn has finished profiling. + let recompile = if ic.is_null() { None } else { Some(Recompile::ProfileSelf) }; + self.guard_shape(block, shape, recv_type.shape(), state, recompile); + // Current shape contains this ivar + let (ivar_storage, offset) = if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + (self_val, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: FieldName::as_heap, offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + (as_heap, offset) + }; + self.push_insn(block, Insn::StoreField { recv: ivar_storage, id: id.into(), offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv: self_val, val }); + if next_shape_id != recv_type.shape() { + // Write the new shape ID + let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) }); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + self.push_insn(block, Insn::StoreField { recv: self_val, id: FieldName::shape_id, offset: shape_id_offset, val: shape_id }); + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn gen_patch_points_for_optimized_ccall(&mut self, block: BlockId, recv_class: VALUE, method_id: ID, cme: *const rb_callable_method_entry_struct, state: InsnId) { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme }, state }); + } + + /// Side exit back to the state after a block-backed send. + /// Using the pre-send snapshot would re-execute the send in the interpreter. + fn gen_post_send_no_ep_escape_patch_point(&mut self, block: BlockId, state: &FrameState, insn_idx: u32) { + let iseq = state.iseq; + let mut reload_state = state.clone(); + reload_state.insn_idx = insn_idx as usize; + reload_state.pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let reload_exit_id = self.push_insn(block, Insn::Snapshot { state: reload_state.without_locals() }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: reload_exit_id }); + } + + fn count_not_inlined_cfunc(&mut self, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_inlined_cfunc_counter_pointers = ZJITState::get_not_inlined_cfunc_counter_pointers(); + let counter_ptr = not_inlined_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + fn count_iseq_calls(&mut self, block: BlockId) { + let iseq_name = iseq_get_location(self.iseq, 0); + let access_counter_ptrs = crate::state::ZJITState::get_iseq_calls_count_pointers(); + let counter_ptr = access_counter_ptrs.entry(iseq_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + fn count_not_annotated_cfunc(&mut self, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_annotated_cfunc_counter_pointers = ZJITState::get_not_annotated_cfunc_counter_pointers(); + let counter_ptr = not_annotated_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + /// Optimize Send that land in a C method to a direct CCall without + /// runtime lookup. + fn optimize_c_calls(&mut self) { + if unsafe { rb_zjit_method_tracing_currently_enabled() } { + return; + } + + // Try to reduce a Send insn to a CCallWithFrame + fn reduce_send_to_ccall( + fun: &mut Function, + block: BlockId, + self_type: Type, + send: Insn, + send_insn_id: InsnId, + ) -> Result<(), ()> { + let Insn::Send { mut recv, cd, block: send_block, args, state, .. } = send else { + return Err(()); + }; + + let call_info = unsafe { (*cd).ci }; + let argc = unsafe { vm_ci_argc(call_info) }; + let method_id = unsafe { rb_vm_ci_mid(call_info) }; + + // If we have info about the class of the receiver + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type} => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), + }; + + // Do method lookup + let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); + return Err(()); + } + + // Filter for C methods + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + if def_type != VM_METHOD_TYPE_CFUNC { + return Err(()); + } + + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedNeedPermission); + return Err(()); + } + } + + // When seeing &block argument, fall back to dynamic dispatch for now + // TODO: Support block forwarding + if unspecializable_c_call_type(ci_flags) { + // Only count features NOT already counted in type_specialize. + if !unspecializable_call_type(ci_flags) { + fun.count_complex_call_features(block, ci_flags); + } + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + let blockiseq = match send_block { + Some(BlockHandler::BlockArg) => unreachable!("unsupported &block should have been filtered out"), + Some(BlockHandler::BlockIseq(blockiseq)) => Some(blockiseq), + None => None, + }; + + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; + // Find the `argc` (arity) of the C method, which describes the parameters it expects + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + let name = unsafe { (*cme).called_id }; + + // Look up annotations + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + fun.count_not_annotated_cfunc(block, cme); + } + let props = props.unwrap_or_default(); + let return_type = props.return_type; + let elidable = match blockiseq { + Some(_) => false, // Don't consider cfuncs with block arguments as elidable for now + None => props.elidable, + }; + + match cfunc_argc { + 0.. => { + // (self, arg0, arg1, ..., argc) form + // + // Bail on argc mismatch + if argc != cfunc_argc as u32 { + return Err(()); + } + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.count(block, Counter::inline_cfunc_optimized_send_count); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); + } + } + + // Emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); + } + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc: cfunc_ptr, + recv, + args, + cme, + name, + state, + return_type, + elidable, + block: blockiseq.map(BlockHandler::BlockIseq), + }); + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + // Variadic method + -1 => { + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.count(block, Counter::inline_cfunc_optimized_send_count); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); + } + } + + // No inlining; emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); + } + + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args, + cme, + name: method_id, + state, + return_type, + elidable, + block: blockiseq.map(BlockHandler::BlockIseq), + }); + + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + -2 => { + // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); + Err(()) + } + _ => unreachable!("unknown cfunc kind: argc={argc}") + } + } + + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + let send = self.find(insn_id); + match send { + send @ Insn::Send { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } + } + Insn::InvokeBuiltin { bf, recv, args, state, .. } => { + let props = ZJITState::get_method_annotations().get_builtin_properties(&bf).unwrap_or_default(); + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + } + _ => {} + } + self.push_insn_id(block, insn_id); + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + /// Convert `Send` instructions with no profile data into `SideExit` with recompile info. + /// This runs after strength reduction passes (type_specialize, inline, optimize_c_calls) so + /// that sends that can be optimized without profiling (e.g. known CFUNCs) are already handled. + /// The remaining no-profile sends are turned into side exits that trigger recompilation with + /// fresh profile data. + fn convert_no_profile_sends(&mut self) { + // On the final version, recompilation is not possible, so converting sends to + // SideExits would just add overhead (the exit fires every time without benefit). + // Keep them as Send fallbacks so the interpreter handles them directly. + let payload = get_or_create_iseq_payload(self.iseq); + if payload.versions.len() + 1 >= crate::codegen::max_iseq_versions() { + return; + } + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::Send { cd, state, reason: SendFallbackReason::SendWithoutBlockNoProfiles | SendFallbackReason::SendNoProfiles, .. } => { + let argc = unsafe { vm_ci_argc((*cd).ci) } as i32; + self.push_insn(block, Insn::SideExit { state, reason: SideExitReason::NoProfileSend, recompile: Some(Recompile::ProfileSend { argc }) }); + // SideExit is a terminator; don't add remaining instructions + break; + } + _ => { + self.push_insn_id(block, insn_id); + } + } + } + } + } + + fn optimize_load_store(&mut self) { + for block in self.reverse_post_order() { + let mut compile_time_heap: HashMap<(InsnId, i32), InsnId> = HashMap::new(); + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + let mut new_insns = vec![]; + for insn_id in old_insns { + let replacement_insn: InsnId = match self.find(insn_id) { + Insn::StoreField { recv, offset, val, .. } => { + let key = (self.chase_insn(recv), offset); + let heap_entry = compile_time_heap.get(&key).copied(); + // TODO(Jacob): Switch from actual to partial equality + if Some(val) == heap_entry { + // If the value is already stored, short circuit and don't add an instruction to the block + continue + } + // TODO(Jacob): Add TBAA to avoid removing so many entries + compile_time_heap.retain(|(_, off), _| *off != offset); + compile_time_heap.insert(key, val); + insn_id + }, + Insn::LoadField { recv, offset, .. } => { + let key = (self.chase_insn(recv), offset); + match compile_time_heap.entry(key) { + std::collections::hash_map::Entry::Occupied(entry) => { + // If the value is stored already, we should short circuit. + // However, we need to replace insn_id with its representative in the SSA union. + self.make_equal_to(insn_id, *entry.get()); + continue + } + std::collections::hash_map::Entry::Vacant(_) => { + // If the value has not been accessed, cache a copy to optimize future loads or stores. + compile_time_heap.insert(key, insn_id); + } + } + insn_id + } + Insn::WriteBarrier { .. } => { + // Currently, WriteBarrier write effects are Allocator and Memory when we'd really like them to be flags. + // We don't use LoadField for mark bits so we can ignore them for now. + // But flags does not exist in our effects abstract heap modeling and we don't want to add special casing to effects. + // This special casing in this pass here should be removed once we refine our effects system to provide greater granularity for WriteBarrier. + // TODO: use TBAA + let offset = RUBY_OFFSET_RBASIC_FLAGS; + compile_time_heap.retain(|(_, off), _| *off != offset); + insn_id + }, + insn => { + // If an instruction affects memory and we haven't modeled it, the compile_time_heap is invalidated + if insn.effects_of().includes(Effect::write(abstract_heaps::Memory)) { + compile_time_heap.clear(); + } + insn_id + } + }; + new_insns.push(replacement_insn); + } + self.blocks[block.0].insns = new_insns; + } + } + + /// Fold a binary operator on fixnums. + fn fold_fixnum_bop(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<i64>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .filter(|&n| n >= (RUBY_FIXNUM_MIN as i64) && n <= RUBY_FIXNUM_MAX as i64) + .map(|n| self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_isize(n as isize)) })) + .unwrap_or(insn_id) + } + + /// Fold a binary predicate on fixnums. + fn fold_fixnum_pred(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<bool>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .map(|b| if b { Qtrue } else { Qfalse }) + .map(|b| self.new_insn(Insn::Const { val: Const::Value(b) })) + .unwrap_or(insn_id) + } + + /// Block-local canonicalize: rewrite each operand through union-find and a + /// per-block map of the most recent `Guard*` for that value. Forwards + /// guarded values into branch-edge args (so `infer_types` narrows merge-block + /// parameters and `fold_constants` drops redundant CFG-join guards) and + /// ordinary in-block uses. + /// + /// `Guard*` substitutions are unconditional within a block: a guard's + /// side-exit semantics guarantee the substituted value type holds for every + /// downstream use in the same block. + /// + /// `RefineType` is intentionally skipped: its narrowing is only valid on one + /// branch arm, which would require dropping refine-derived rewrites at each + /// `IfTrue`/`IfFalse`. Cross-arm refine forwarding is left for a follow-up + /// dominator-scoped pass. + /// + /// Inspired by Cranelift's aegraph canonicalize step + /// (<https://cfallin.org/blog/2026/04/09/aegraph/>). + fn canonicalize(&mut self) { + let mut rewrite_map: HashMap<InsnId, InsnId> = HashMap::new(); + for block in self.reverse_post_order() { + rewrite_map.clear(); + for i in 0..self.blocks[block.0].insns.len() { + let insn_id = self.blocks[block.0].insns[i]; + let canonical_id = self.union_find.borrow().find_const(insn_id); + + let union_find = &self.union_find; + self.insns[canonical_id.0].for_each_operand_mut(|operand| { + let canon = union_find.borrow().find_const(*operand); + *operand = rewrite_map.get(&canon).copied().unwrap_or(canon); + }); + + // For the binary guards only `left` is registered because their infer_type is + // type_of(left). + match &self.insns[canonical_id.0] { + Insn::GuardType { val: src, .. } + | Insn::GuardBitEquals { val: src, .. } + | Insn::GuardAnyBitSet { val: src, .. } + | Insn::GuardNoBitsSet { val: src, .. } + | Insn::GuardGreaterEq { left: src, .. } + | Insn::GuardLess { left: src, .. } => { + rewrite_map.insert(*src, canonical_id); + } + _ => {} + } + } + } + + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + /// Use type information left by `infer_types` to fold away operations that can be evaluated at compile-time. + /// + /// It can fold fixnum math, truthiness tests, and branches with constant conditionals. + fn fold_constants(&mut self) { + // TODO(max): Determine if it's worth it for us to reflow types after each branch + // simplification. This means that we can have nice cascading optimizations if what used to + // be a union of two different basic block arguments now has a single value. + // + // This would require 1) fixpointing, 2) worklist, or 3) (slightly less powerful) calling a + // function-level infer_types after each pruned branch. + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + let mut new_insns = vec![]; + for insn_id in old_insns { + let replacement_id = match self.find(insn_id) { + Insn::GuardType { val, guard_type, .. } if self.is_a(val, guard_type) => { + self.make_equal_to(insn_id, val); + // Don't bother re-inferring the type of val; we already know it. + continue; + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::BasicObject) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const VALUE; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::CShape) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const u32; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::CShape(ShapeId(val)) }) + } + _ => insn_id, + } + } + Insn::ArrayLength { array } => { + match self.type_of(array).ruby_object() { + Some(array_obj) if array_obj.is_frozen() => { + let length = unsafe { rb_jit_array_len(array_obj) }; + self.new_insn(Insn::Const { val: Const::CInt64(length) }) + } + _ => insn_id, + } + } + Insn::UnboxFixnum { val } => { + let recv_type = self.type_of(val); + match recv_type.fixnum_value() { + Some(val) => self.new_insn(Insn::Const { val: Const::CInt64(val) }), + _ => insn_id, + } + }, + Insn::GuardGreaterEq { left, right, state, reason } => { + let left_num = self.type_of(left).cint64_value(); + let right_num = self.type_of(right).cint64_value(); + match (left_num, right_num) { + (Some(l), Some(r)) if l >= r => { + self.make_equal_to(insn_id, left); + continue + }, + (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason, recompile: None }), + _ => insn_id, + } + }, + Insn::GuardLess { left, right, state, reason } => { + let left_num = self.type_of(left).cint64_value(); + let right_num = self.type_of(right).cint64_value(); + match (left_num, right_num) { + (Some(l), Some(r)) if l < r => { + self.make_equal_to(insn_id, left); + continue + }, + (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason, recompile: None }), + _ => insn_id, + } + }, + Insn::GuardBitEquals { val, expected, .. } => { + let recv_type = self.type_of(val); + if recv_type.has_value(expected) { + continue; + } else { + insn_id + } + } + Insn::AnyToString { str, .. } if self.is_a(str, types::String) => { + self.make_equal_to(insn_id, str); + // Don't bother re-inferring the type of str; we already know it. + continue; + } + Insn::IsA { val, class } => 'is_a: { + let class_type = self.type_of(class); + if !class_type.is_subtype(types::Class) { + break 'is_a insn_id; + } + let Some(class_value) = class_type.ruby_object() else { + break 'is_a insn_id; + }; + let val_type = self.type_of(val); + let the_class = Type::from_class_inexact(class_value); + if val_type.is_subtype(the_class) { + self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) + } else if !val_type.could_be(the_class) { + self.new_insn(Insn::Const { val: Const::Value(Qfalse) }) + } else { + insn_id + } + } + Insn::StringEqual { left, right } => { + let left = self.chase_insn(left); + let right = self.chase_insn(right); + // If both operands resolve to the same SSA value, + // String#== is guaranteed to be true. + if left == right { + self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) + } else { + let left_type = self.type_of(left); + let right_type = self.type_of(right); + match (left_type.ruby_object(), right_type.ruby_object()) { + (Some(left_obj), Some(right_obj)) + if left_obj.is_frozen() && right_obj.is_frozen() => + { + // For known frozen objects, evaluate String#== at compile time. + let val = unsafe { rb_yarv_str_eql_internal(left_obj, right_obj) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + } + Insn::FixnumAdd { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_add(r), + _ => None, + }) + } + Insn::FixnumSub { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_sub(r), + _ => None, + }) + } + Insn::FixnumMult { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_mul(r), + (Some(0), _) | (_, Some(0)) => Some(0), + _ => None, + }) + } + Insn::FixnumDiv { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) if l == (RUBY_FIXNUM_MIN as i64) && r == -1 => None, // Avoid Fixnum overflow + (Some(_l), Some(r)) if r == 0 => None, // Avoid Divide by zero. + (Some(l), Some(r)) => { + let l_obj = VALUE::fixnum_from_isize(l as isize); + let r_obj = VALUE::fixnum_from_isize(r as isize); + Some(unsafe { rb_jit_fix_div_fix(l_obj, r_obj) }.as_fixnum()) + }, + _ => None, + }) + } + Insn::FixnumMod { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) if r != 0 => { + let l_obj = VALUE::fixnum_from_isize(l as isize); + let r_obj = VALUE::fixnum_from_isize(r as isize); + Some(unsafe { rb_jit_fix_mod_fix(l_obj, r_obj) }.as_fixnum()) + }, + _ => None, + }) + } + Insn::FixnumXor { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l ^ r), + _ => None, + }) + } + Insn::FixnumAnd { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l & r), + _ => None, + }) + } + Insn::FixnumOr { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l | r), + _ => None, + }) + } + Insn::FixnumEq { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l == r), + _ => None, + }) + } + Insn::FixnumNeq { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l != r), + _ => None, + }) + } + Insn::FixnumLt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l < r), + _ => None, + }) + } + Insn::FixnumLe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l <= r), + _ => None, + }) + } + Insn::FixnumGt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l > r), + _ => None, + }) + } + Insn::FixnumGe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l >= r), + _ => None, + }) + } + Insn::ArrayAref { array, index } + if self.type_of(array).ruby_object_known() + && self.type_of(index).is_subtype(types::CInt64) => { + let array_obj = self.type_of(array).ruby_object().unwrap(); + match (array_obj.is_frozen(), self.type_of(index).cint64_value()) { + (true, Some(index)) => { + let val = unsafe { rb_yarv_ary_entry_internal(array_obj, index) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::AdjustBounds { index, .. } => { + // If index is known nonnegative, then we don't need to adjust bounds. + if self.type_of(index).known_nonnegative() { + self.make_equal_to(insn_id, index); + // Don't bother re-inferring the type of index; we already know it. + continue; + } else { + insn_id + } + } + Insn::Test { val } if self.type_of(val).is_known_falsy() => { + self.new_insn(Insn::Const { val: Const::CBool(false) }) + } + Insn::Test { val } if self.type_of(val).is_known_truthy() => { + self.new_insn(Insn::Const { val: Const::CBool(true) }) + } + Insn::Test { val: test_val } => { + if let Insn::BoxBool { val: bool_val } = self.find(test_val) { + self.make_equal_to(insn_id, bool_val); + continue; + } else { + insn_id + } + } + Insn::CondBranch { val, if_true, .. } if self.is_a(val, Type::from_cbool(true)) => { + self.new_insn(Insn::Jump(if_true)) + } + Insn::CondBranch { val, if_false, .. } if self.is_a(val, Type::from_cbool(false)) => { + self.new_insn(Insn::Jump(if_false)) + } + _ => insn_id, + }; + // If we're adding a new instruction, mark the two equivalent in the union-find and + // do an incremental flow typing of the new instruction. + if insn_id != replacement_id && self.insns[replacement_id.0].has_output() { + self.make_equal_to(insn_id, replacement_id); + self.insn_types[replacement_id.0] = self.infer_type(replacement_id); + } + new_insns.push(replacement_id); + // If we've just folded an IfTrue into a Jump, for example, don't bother copying + // over unreachable instructions afterward. + if self.insns[replacement_id.0].is_terminator() { + break; + } + } + self.blocks[block.0].insns = new_insns; + } + } + + /// Remove instructions that do not have side effects and are not referenced by any other + /// instruction. + fn eliminate_dead_code(&mut self) { + let rpo = self.reverse_post_order(); + let mut worklist = VecDeque::new(); + // Find all of the instructions that have side effects, are control instructions, or are + // otherwise necessary to keep around + for block_id in &rpo { + for insn_id in &self.blocks[block_id.0].insns { + if !&self.insns[insn_id.0].is_elidable() { + worklist.push_back(*insn_id); + } + } + } + let mut necessary = InsnSet::with_capacity(self.insns.len()); + // Now recursively traverse their data dependencies and mark those as necessary + while let Some(insn_id) = worklist.pop_front() { + if necessary.get(insn_id) { continue; } + necessary.insert(insn_id); + let insn_id = self.union_find.borrow().find_const(insn_id); + self.insns[insn_id.0].for_each_operand(|operand| { + worklist.push_back(self.union_find.borrow().find_const(operand)); + }); + } + // Now remove all unnecessary instructions + for block_id in &rpo { + self.blocks[block_id.0].insns.retain(|&insn_id| necessary.get(insn_id)); + } + } + + fn absorb_dst_block(&mut self, num_in_edges: &[u32], block: BlockId) -> bool { + let Some(terminator_id) = self.blocks[block.0].insns.last() + else { return false }; + let Insn::Jump(BranchEdge { target, args }) = self.find(*terminator_id) + else { return false }; + if target == block { + // Can't absorb self + return false; + } + if num_in_edges[target.0] != 1 { + // Can't absorb block if it's the target of more than one branch + return false; + } + // Link up params with block args + let params = std::mem::take(&mut self.blocks[target.0].params); + assert_eq!(args.len(), params.len()); + for (arg, param) in args.iter().zip(params) { + self.make_equal_to(param, *arg); + } + // Remove branch instruction + self.blocks[block.0].insns.pop(); + // Move target instructions into block + let target_insns = std::mem::take(&mut self.blocks[target.0].insns); + self.blocks[block.0].insns.extend(target_insns); + true + } + + /// Clean up linked lists of blocks A -> B -> C into A (with B's and C's instructions). + fn clean_cfg(&mut self) { + // num_in_edges is invariant throughout cleaning the CFG: + // * we don't allocate new blocks + // * blocks that get absorbed are not in RPO anymore + // * blocks pointed to by blocks that get absorbed retain the same number of in-edges + let mut num_in_edges = vec![0; self.blocks.len()]; + for block in self.reverse_post_order() { + for target in self.successors(block) { + num_in_edges[target.0] += 1; + } + } + let mut changed = false; + loop { + let mut iter_changed = false; + for block in self.reverse_post_order() { + // Ignore transient empty blocks + if self.blocks[block.0].insns.is_empty() { continue; } + loop { + let absorbed = self.absorb_dst_block(&num_in_edges, block); + if !absorbed { break; } + iter_changed = true; + } + } + if !iter_changed { break; } + changed = true; + } + if changed { + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + } + + /// Remove duplicate PatchPoint instructions within each basic block. + /// Two PatchPoints are redundant if they assert the same Invariant and no + /// intervening instruction could invalidate it (i.e., writes to PatchPoint). + fn remove_redundant_patch_points(&mut self) { + for block_id in self.reverse_post_order() { + let mut seen = HashSet::new(); + let insns = std::mem::take(&mut self.blocks[block_id.0].insns); + let mut new_insns = Vec::with_capacity(insns.len()); + for insn_id in insns { + let insn = self.find(insn_id); + if let Insn::PatchPoint { invariant, .. } = insn { + if !seen.insert(invariant) { + continue; + } + } else if insn.effects_of().write_bits().overlaps(abstract_heaps::PatchPoint) { + seen.clear(); + } + new_insns.push(insn_id); + } + self.blocks[block_id.0].insns = new_insns; + } + } + + /// Remove duplicate CheckInterrupts instructions within each basic block. + /// Only the first CheckInterrupts in a block is needed unless an intervening + /// instruction writes to InterruptFlag (e.g. a call), which resets tracking. + fn remove_duplicate_check_interrupts(&mut self) { + for block_id in self.reverse_post_order() { + let mut seen = false; + let insns = std::mem::take(&mut self.blocks[block_id.0].insns); + let mut new_insns = Vec::with_capacity(insns.len()); + for insn_id in insns { + let insn = &self.insns[insn_id.0]; + if matches!(insn, Insn::CheckInterrupts { .. }) { + if seen { continue; } + seen = true; + } else if insn.effects_of().write_bits().overlaps(abstract_heaps::InterruptFlag) { + seen = false; + } + new_insns.push(insn_id); + } + self.blocks[block_id.0].insns = new_insns; + } + } + + /// Return a list that has entry_block and then jit_entry_blocks + fn entry_blocks(&self) -> Vec<BlockId> { + let mut entry_blocks = self.jit_entry_blocks.clone(); + entry_blocks.insert(0, self.entry_block); + entry_blocks + } + + pub fn is_entry_block(&self, block_id: BlockId) -> bool { + self.entry_block == block_id || self.jit_entry_blocks.contains(&block_id) + } + + /// Populate the entries superblock with an Entries instruction targeting all entry blocks. + /// Must be called after all entry blocks have been created. + fn seal_entries(&mut self) { + let targets = self.entry_blocks(); + self.push_insn(self.entries_block, Insn::Entries { targets }); + } + + /// Return a traversal of the `Function`'s `BlockId`s in reverse post-order. + pub fn reverse_post_order(&self) -> Vec<BlockId> { + let mut result = self.po_from(self.entries_block); + result.reverse(); + result + } + + fn po_from(&self, start: BlockId) -> Vec<BlockId> { + #[derive(PartialEq)] + enum Action { + VisitEdges, + VisitSelf, + } + let mut result = vec![]; + let mut seen = BlockSet::with_capacity(self.blocks.len()); + let mut stack = vec![(start, Action::VisitEdges)]; + while let Some((block, action)) = stack.pop() { + if action == Action::VisitSelf { + result.push(block); + continue; + } + if !seen.insert(block) { continue; } + stack.push((block, Action::VisitSelf)); + for target in self.successors(block) { + stack.push((target, Action::VisitEdges)); + } + } + result + } + + fn assert_validates(&self) { + if let Err(err) = self.validate() { + eprintln!("Function failed validation."); + eprintln!("Err: {err:?}"); + eprintln!("{}", FunctionPrinter::with_snapshot(self)); + panic!("Aborting..."); + } + } + + /// Helper function to make an Iongraph JSON "instruction". + /// `uses`, `memInputs` and `attributes` are left empty for now, but may be populated + /// in the future. + fn make_iongraph_instr(id: InsnId, inputs: Vec<Json>, opcode: &str, ty: &str) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("opcode", opcode) + .insert("attributes", Json::empty_array()) + .insert("inputs", Json::Array(inputs)) + .insert("uses", Json::empty_array()) + .insert("memInputs", Json::empty_array()) + .insert("type", ty) + .build() + } + + /// Helper function to make an Iongraph JSON "block". + fn make_iongraph_block(id: BlockId, predecessors: Vec<BlockId>, successors: Vec<BlockId>, instructions: Vec<Json>, attributes: Vec<&str>, loop_depth: u32) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("loopDepth", loop_depth) + .insert("attributes", Json::array(attributes)) + .insert("predecessors", Json::array(predecessors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("successors", Json::array(successors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("instructions", Json::array(instructions)) + .build() + } + + /// Helper function to make an Iongraph JSON "function". + /// Note that `lir` is unpopulated right now as ZJIT doesn't use its functionality. + fn make_iongraph_function(pass_name: &str, hir_blocks: Vec<Json>) -> Json { + Json::object() + .insert("name", pass_name) + .insert("mir", Json::object() + .insert("blocks", Json::array(hir_blocks)) + .build() + ) + .insert("lir", Json::object() + .insert("blocks", Json::empty_array()) + .build() + ) + .build() + } + + /// Generate an iongraph JSON pass representation for this function. + pub fn to_iongraph_pass(&self, pass_name: &str) -> Json { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + + let mut hir_blocks = Vec::new(); + let cfi = ControlFlowInfo::new(self); + let dominators = Dominators::new(self); + let loop_info = LoopInfo::new(&cfi, &dominators); + + // Push each block from the iteration in reverse post order to `hir_blocks`. + for block_id in self.reverse_post_order() { + // Create the block with instructions. + let block = &self.blocks[block_id.0]; + let predecessors = cfi.predecessors(block_id).collect(); + let successors = cfi.successors(block_id).collect(); + let mut instructions = Vec::new(); + + // Process all instructions (parameters and body instructions). + // Parameters are currently guaranteed to be Parameter instructions, but in the future + // they might be refined to other instruction kinds by the optimizer. + for insn_id in block.params.iter().chain(block.insns.iter()) { + let insn_id = self.union_find.borrow().find_const(*insn_id); + let insn = self.find(insn_id); + + // Snapshots are not serialized, so skip them. + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + + // Instructions with no output or an empty type should have an empty type field. + let type_str = if insn.has_output() { + let insn_type = self.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + String::new() + } else { + insn_type.print(&ptr_map).to_string() + } + } else { + String::new() + }; + + + let opcode = insn.print(&ptr_map, Some(self.iseq)).to_string(); + + // Collect inputs for a given instruction. + let mut inputs = Vec::new(); + insn.for_each_operand(|id| inputs.push(id.0.into())); + let inputs: Vec<Json> = inputs; + + instructions.push( + Self::make_iongraph_instr( + insn_id, + inputs, + &opcode, + &type_str + ) + ); + } + + let mut attributes = vec![]; + if loop_info.is_back_edge_source(block_id) { + attributes.push("backedge"); + } + if loop_info.is_loop_header(block_id) { + attributes.push("loopheader"); + } + let loop_depth = loop_info.loop_depth(block_id); + + hir_blocks.push(Self::make_iongraph_block( + block_id, + predecessors, + successors, + instructions, + attributes, + loop_depth, + )); + } + + Self::make_iongraph_function(pass_name, hir_blocks) + } + + /// Run all the optimization passes we have. + pub fn optimize(&mut self) { + let mut passes: Vec<Json> = Vec::new(); + let should_dump = get_option!(dump_hir_iongraph); + + macro_rules! counter_for { + // Bucket all strength reduction together + (type_specialize) => { Counter::compile_hir_strength_reduce_time_ns }; + (inline) => { Counter::compile_hir_strength_reduce_time_ns }; + (optimize_getivar) => { Counter::compile_hir_strength_reduce_time_ns }; + (optimize_c_calls) => { Counter::compile_hir_strength_reduce_time_ns }; + (convert_no_profile_sends) => { Counter::compile_hir_strength_reduce_time_ns }; + // End strength reduction bucket + (optimize_load_store) => { Counter::compile_hir_optimize_load_store_time_ns }; + (canonicalize) => { Counter::compile_hir_canonicalize_time_ns }; + (fold_constants) => { Counter::compile_hir_fold_constants_time_ns }; + (clean_cfg) => { Counter::compile_hir_clean_cfg_time_ns }; + (remove_redundant_patch_points) => { Counter::compile_hir_remove_redundant_patch_points_time_ns }; + (remove_duplicate_check_interrupts) => { Counter::compile_hir_remove_duplicate_check_interrupts_time_ns }; + (eliminate_dead_code) => { Counter::compile_hir_eliminate_dead_code_time_ns }; + ($name:ident) => { unimplemented!("Counter for pass {}", stringify!($name)) }; + } + + macro_rules! run_pass { + ($name:ident) => { + let counter = counter_for!($name); + crate::stats::trace_compile_phase(stringify!($name), || + crate::stats::with_time_stat(counter, || self.$name()) + ); + #[cfg(debug_assertions)] crate::stats::trace_compile_phase("validate", || self.assert_validates()); + if should_dump { + passes.push( + self.to_iongraph_pass(stringify!($name)) + ); + } + } + } + + if should_dump { + passes.push(self.to_iongraph_pass("unoptimized")); + } + + // Function is assumed to have types inferred already + run_pass!(type_specialize); + run_pass!(inline); + run_pass!(optimize_getivar); + run_pass!(optimize_c_calls); + run_pass!(convert_no_profile_sends); + run_pass!(optimize_load_store); + run_pass!(canonicalize); + run_pass!(fold_constants); + run_pass!(clean_cfg); + run_pass!(remove_redundant_patch_points); + run_pass!(remove_duplicate_check_interrupts); + run_pass!(eliminate_dead_code); + + if should_dump { + let iseq_name = iseq_get_location(self.iseq, 0); + self.dump_iongraph(&iseq_name, passes); + } + } + + /// Dump HIR passed to codegen if specified by options. + pub fn dump_hir(&self) { + // Dump HIR after optimization + match get_option!(dump_hir_opt) { + Some(DumpHIR::WithoutSnapshot) => println!("Optimized HIR:\n{}", FunctionPrinter::without_snapshot(self)), + Some(DumpHIR::All) => println!("Optimized HIR:\n{}", FunctionPrinter::with_snapshot(self)), + Some(DumpHIR::Debug) => println!("Optimized HIR:\n{:#?}", &self), + None => {}, + } + } + + pub fn dump_iongraph(&self, function_name: &str, passes: Vec<Json>) { + fn sanitize_for_filename(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == '-' { + c + } else { + '_' + } + }) + .collect() + } + + use std::io::Write; + let dir = format!("/tmp/zjit-iongraph-{}", std::process::id()); + std::fs::create_dir_all(&dir).expect("Unable to create directory."); + let sanitized = sanitize_for_filename(function_name); + let path = format!("{dir}/func_{sanitized}.json"); + let mut file = std::fs::File::create(path).unwrap(); + let json = Json::object() + .insert("name", function_name) + .insert("passes", passes) + .build(); + writeln!(file, "{json}").unwrap(); + } + + /// Validates the following: + /// 1. Basic block jump args match parameter arity. + /// 2. Every terminator must be in the last position. + /// 3. Every block must have a terminator. + fn validate_block_terminators_and_jumps(&self) -> Result<(), ValidationError> { + let check_edge = |block_id: BlockId, edge: &BranchEdge| -> Result<(), ValidationError> { + let target_len = self.blocks[edge.target.0].params.len(); + let args_len = edge.args.len(); + if target_len != args_len { + return Err(ValidationError::MismatchedBlockArity(block_id, target_len, args_len)); + } + Ok(()) + }; + + for block_id in self.reverse_post_order() { + let insns = &self.blocks[block_id.0].insns; + for (idx, insn_id) in insns.iter().enumerate() { + let insn = self.find(*insn_id); + // Validate arity for all branch edges + match &insn { + Insn::Jump(edge) => { + check_edge(block_id, edge)?; + } + Insn::CondBranch { if_true, if_false, .. } => { + check_edge(block_id, if_true)?; + check_edge(block_id, if_false)?; + } + _ => {} + } + + if insn.is_terminator() { + // Blow up if we have a terminator that isn't at the end + // of the block. + if idx != insns.len() - 1 { + return Err(ValidationError::TerminatorNotAtEnd(block_id, *insn_id, idx)) + } + } + // If the last instruction isn't a terminator, return an error + if idx == insns.len() - 1 { + if !insn.is_terminator() { + return Err(ValidationError::BlockHasNoTerminator(block_id)); + } + } + } + } + Ok(()) + } + + // This performs a dataflow def-analysis over the entire CFG to detect any + // possibly undefined instruction operands. + fn validate_definite_assignment(&self) -> Result<(), ValidationError> { + // Map of block ID -> InsnSet + // Initialize with all missing values at first, to catch if a jump target points to a + // missing location. + let mut assigned_in = vec![None; self.num_blocks()]; + let rpo = self.reverse_post_order(); + // Begin with every block having every variable defined, except for entries_block, which + // starts with nothing defined. + for &block in &rpo { + if block == self.entries_block { + assigned_in[block.0] = Some(InsnSet::with_capacity(self.insns.len())); + } else { + let mut all_ones = InsnSet::with_capacity(self.insns.len()); + all_ones.insert_all(); + assigned_in[block.0] = Some(all_ones); + } + } + let mut worklist = VecDeque::with_capacity(self.num_blocks()); + worklist.push_back(self.entries_block); + while let Some(block) = worklist.pop_front() { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + let insn = self.find(insn_id); + let mut propagate = |target: BlockId| -> Result<(), ValidationError> { + let Some(block_in) = assigned_in[target.0].as_mut() else { + return Err(ValidationError::JumpTargetNotInRPO(target)); + }; + if block_in.intersect_with(&assigned) { + worklist.push_back(target); + } + Ok(()) + }; + match insn { + Insn::Jump(edge) => propagate(edge.target)?, + Insn::CondBranch { if_true, if_false, .. } => { + propagate(if_true.target)?; + propagate(if_false.target)?; + } + Insn::Entries { ref targets } => { + for &target in targets { + propagate(target)?; + } + } + insn if insn.has_output() => { + assigned.insert(insn_id); + } + _ => {} + } + } + } + // Check that each instruction's operands are assigned + for &block in &rpo { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + self.insns[insn_id.0].try_for_each_operand(|operand| { + let operand = self.union_find.borrow().find_const(operand); + if !assigned.get(operand) { + return Err(ValidationError::OperandNotDefined(block, insn_id, operand)); + } + Ok(()) + })?; + if self.insns[insn_id.0].has_output() { + assigned.insert(insn_id); + } + } + } + Ok(()) + } + + /// Checks that each instruction('s representative) appears only once in the CFG. + fn validate_insn_uniqueness(&self) -> Result<(), ValidationError> { + let mut seen = InsnSet::with_capacity(self.insns.len()); + for block_id in self.reverse_post_order() { + for &insn_id in &self.blocks[block_id.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + if !seen.insert(insn_id) { + return Err(ValidationError::DuplicateInstruction(block_id, insn_id)); + } + } + } + Ok(()) + } + + fn assert_subtype(&self, user: InsnId, operand: InsnId, expected: Type) -> Result<(), ValidationError> { + let actual = self.type_of(operand); + if !actual.is_subtype(expected) { + return Err(ValidationError::MismatchedOperandType(user, operand, format!("{expected}"), format!("{actual}"))); + } + Ok(()) + } + + fn validate_insn_type(&self, insn_id: InsnId) -> Result<(), ValidationError> { + let insn_id = self.union_find.borrow().find_const(insn_id); + let insn = self.find(insn_id); + match insn { + // Instructions with no InsnId operands (except state) or nothing to assert + Insn::Const { .. } + | Insn::Param + | Insn::LoadArg { .. } + | Insn::PutSpecialObject { .. } + | Insn::LoadField { .. } + | Insn::GetConstantPath { .. } + | Insn::IsBlockGiven { .. } + | Insn::GetGlobal { .. } + | Insn::LoadPC + | Insn::LoadSP + | Insn::LoadEC + | Insn::GetEP { .. } + | Insn::BreakPoint | Insn::Unreachable + | Insn::LoadSelf + | Insn::Snapshot { .. } + | Insn::Jump { .. } + | Insn::Entries { .. } + | Insn::EntryPoint { .. } + | Insn::PatchPoint { .. } + | Insn::SideExit { .. } + | Insn::IncrCounter { .. } + | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } + | Insn::GetClassVar { .. } + | Insn::GetSpecialNumber { .. } + | Insn::GetSpecialSymbol { .. } + | Insn::GetBlockParam { .. } + | Insn::StoreField { .. } => { + Ok(()) + } + // Instructions with 1 Ruby object operand + Insn::Test { val } + | Insn::IsNil { val } + | Insn::IsMethodCfunc { val, .. } + | Insn::SetGlobal { val, .. } + | Insn::SetLocal { val, .. } + | Insn::SetClassVar { val, .. } + | Insn::Return { val } + | Insn::Throw { val, .. } + | Insn::ObjToString { val, .. } + | Insn::GuardType { val, .. } + | Insn::ToArray { val, .. } + | Insn::ToNewArray { val, .. } + | Insn::Defined { v: val, .. } + | Insn::ObjectAlloc { val, .. } + | Insn::DupArrayInclude { target: val, .. } + | Insn::GetIvar { self_val: val, .. } + | Insn::CCall { recv: val, .. } + | Insn::FixnumBitCheck { val, .. } // TODO (https://github.com/Shopify/ruby/issues/859) this should check Fixnum, but then test_checkkeyword_tests_fixnum_bit fails + | Insn::DefinedIvar { self_val: val, .. } => { + self.assert_subtype(insn_id, val, types::BasicObject) + } + // Instructions with 2 Ruby object operands + Insn::SetIvar { self_val: left, val: right, .. } + | Insn::NewRange { low: left, high: right, .. } + | Insn::AnyToString { val: left, str: right, .. } + | Insn::CheckMatch { target: left, pattern: right, .. } + | Insn::WriteBarrier { recv: left, val: right } => { + self.assert_subtype(insn_id, left, types::BasicObject)?; + self.assert_subtype(insn_id, right, types::BasicObject) + } + Insn::GetConstant { klass, allow_nil, .. } => { + self.assert_subtype(insn_id, klass, types::BasicObject)?; + self.assert_subtype(insn_id, allow_nil, types::BoolExact) + } + // Instructions with recv and a Vec of Ruby objects + Insn::SendDirect { recv, ref args, .. } + | Insn::Send { recv, ref args, .. } + | Insn::SendForward { recv, ref args, .. } + | Insn::InvokeSuper { recv, ref args, .. } + | Insn::InvokeSuperForward { recv, ref args, .. } + | Insn::CCallWithFrame { recv, ref args, .. } + | Insn::CCallVariadic { recv, ref args, .. } + | Insn::InvokeBuiltin { recv, ref args, .. } + | Insn::InvokeProc { recv, ref args, .. } + | Insn::ArrayInclude { target: recv, elements: ref args, .. } => { + self.assert_subtype(insn_id, recv, types::BasicObject)?; + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::ArrayPackBuffer { ref elements, fmt, buffer, .. } => { + self.assert_subtype(insn_id, fmt, types::BasicObject)?; + if let Some(buffer) = buffer { + self.assert_subtype(insn_id, buffer, types::BasicObject)?; + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + // Instructions with a Vec of Ruby objects + Insn::InvokeBlock { ref args, .. } + | Insn::InvokeBlockIfunc { ref args, .. } + | Insn::NewArray { elements: ref args, .. } + | Insn::ArrayHash { elements: ref args, .. } + | Insn::ArrayMin { elements: ref args, .. } + | Insn::ArrayMax { elements: ref args, .. } => { + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::NewHash { ref elements, .. } => { + if elements.len() % 2 != 0 { + return Err(ValidationError::MiscValidationError(insn_id, "NewHash elements length is not even".to_string())); + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + Insn::StringConcat { ref strings, .. } + | Insn::ToRegexp { values: ref strings, .. } => { + for &string in strings { + self.assert_subtype(insn_id, string, types::String)?; + } + Ok(()) + } + // Instructions with String operands + Insn::StringCopy { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringIntern { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringAppend { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::String) + } + Insn::StringAppendCodepoint { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::Fixnum) + } + Insn::StringEqual { left, right } => { + self.assert_subtype(insn_id, left, types::String)?; + self.assert_subtype(insn_id, right, types::String) + } + // Instructions with Array operands + Insn::ArrayDup { val, .. } => self.assert_subtype(insn_id, val, types::ArrayExact), + Insn::ArrayExtend { left, right, .. } => { + // TODO(max): Do left and right need to be ArrayExact? + self.assert_subtype(insn_id, left, types::Array)?; + self.assert_subtype(insn_id, right, types::Array) + } + Insn::ArrayPush { array, .. } + | Insn::ArrayPop { array, .. } + | Insn::ArrayLength { array, .. } => { + self.assert_subtype(insn_id, array, types::Array) + } + Insn::ArrayAref { array, index } => { + self.assert_subtype(insn_id, array, types::Array)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::ArrayAset { array, index, .. } => { + self.assert_subtype(insn_id, array, types::ArrayExact)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::AdjustBounds { index, length } => { + self.assert_subtype(insn_id, index, types::CInt64)?; + self.assert_subtype(insn_id, length, types::CInt64) + } + // Instructions with Hash operands + Insn::HashAref { hash, .. } + | Insn::HashAset { hash, .. } => self.assert_subtype(insn_id, hash, types::HashExact), + Insn::HashDup { val, .. } => self.assert_subtype(insn_id, val, types::HashExact), + // Other + Insn::ObjectAllocClass { class, .. } => { + if !class_has_leaf_allocator(class) { + return Err(ValidationError::MiscValidationError(insn_id, "ObjectAllocClass must have leaf allocator".to_string())); + } + Ok(()) + } + Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + if self.is_a(left, types::CInt) && self.is_a(right, types::CInt) { + // TODO(max): Check that int sizes match + Ok(()) + } else if self.is_a(left, types::CPtr) && self.is_a(right, types::CPtr) { + Ok(()) + } else if self.is_a(left, types::RubyValue) && self.is_a(right, types::RubyValue) { + Ok(()) + } else { + Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())) + } + } + Insn::IntAnd { left, right } + | Insn::IntOr { left, right } => { + // TODO: Expand this to other matching C integer sizes when we need them. + let left_type = self.type_of(left); + if left_type.is_subtype(types::CInt64) { + self.assert_subtype(insn_id, right, types::CInt64) + } else if left_type.is_subtype(types::CUInt64) { + self.assert_subtype(insn_id, right, types::CUInt64) + } else { + let all_ints = types::CInt64.union(types::CUInt64); + self.assert_subtype(insn_id, left, all_ints)?; + self.assert_subtype(insn_id, right, all_ints) + } + } + Insn::BoxBool { val } + | Insn::CondBranch { val, .. } => { + self.assert_subtype(insn_id, val, types::CBool) + } + Insn::BoxFixnum { val, .. } => self.assert_subtype(insn_id, val, types::CInt64), + Insn::UnboxFixnum { val } => { + self.assert_subtype(insn_id, val, types::Fixnum) + } + Insn::FixnumAref { recv, index } => { + self.assert_subtype(insn_id, recv, types::Fixnum)?; + self.assert_subtype(insn_id, index, types::Fixnum) + } + Insn::FixnumAdd { left, right, .. } + | Insn::FixnumSub { left, right, .. } + | Insn::FixnumMult { left, right, .. } + | Insn::FixnumDiv { left, right, .. } + | Insn::FixnumMod { left, right, .. } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::NewRangeFixnum { low: left, high: right, .. } + => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum) + } + Insn::FloatAdd { recv, other, .. } + | Insn::FloatSub { recv, other, .. } + | Insn::FloatMul { recv, other, .. } + | Insn::FloatDiv { recv, other, .. } + => { + self.assert_subtype(insn_id, recv, types::Flonum)?; + // other can be Flonum or Fixnum (rb_float_plus etc. handle both) + self.assert_subtype(insn_id, other, types::Flonum.union(types::Fixnum)) + } + Insn::FloatToInt { recv, .. } => { + self.assert_subtype(insn_id, recv, types::Flonum) + } + Insn::FixnumLShift { left, right, .. } + | Insn::FixnumRShift { left, right, .. } => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum)?; + let Some(obj) = self.type_of(right).fixnum_value() else { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<a compile-time constant>".into(), "<unknown>".into())); + }; + if obj < 0 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<positive>".into(), format!("{obj}"))); + } + if obj > 63 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<less than 64>".into(), format!("{obj}"))); + } + Ok(()) + } + Insn::GuardBitEquals { val, expected, .. } => { + match expected { + Const::Value(_) => self.assert_subtype(insn_id, val, types::RubyValue), + Const::CInt8(_) => self.assert_subtype(insn_id, val, types::CInt8), + Const::CInt16(_) => self.assert_subtype(insn_id, val, types::CInt16), + Const::CInt32(_) => self.assert_subtype(insn_id, val, types::CInt32), + Const::CInt64(_) => self.assert_subtype(insn_id, val, types::CInt64), + Const::CUInt8(_) => self.assert_subtype(insn_id, val, types::CUInt8), + Const::CUInt16(_) => self.assert_subtype(insn_id, val, types::CUInt16), + Const::CUInt32(_) => self.assert_subtype(insn_id, val, types::CUInt32), + Const::CAttrIndex(_) => self.assert_subtype(insn_id, val, types::CAttrIndex), + Const::CShape(_) => self.assert_subtype(insn_id, val, types::CShape), + Const::CUInt64(_) => self.assert_subtype(insn_id, val, types::CUInt64), + Const::CBool(_) => self.assert_subtype(insn_id, val, types::CBool), + Const::CDouble(_) => self.assert_subtype(insn_id, val, types::CDouble), + Const::CPtr(_) => self.assert_subtype(insn_id, val, types::CPtr), + } + } + Insn::GuardAnyBitSet { val, mask, .. } + | Insn::GuardNoBitsSet { val, mask, .. } => { + match mask { + Const::CUInt8(_) | Const::CUInt16(_) | Const::CUInt32(_) | Const::CUInt64(_) + if self.is_a(val, types::CInt) || self.is_a(val, types::RubyValue) => { + Ok(()) + } + _ => { + Err(ValidationError::MiscValidationError(insn_id, "GuardAnyBitSet/GuardNoBitsSet can only compare RubyValue/CUInt or CInt/CUInt".to_string())) + } + } + } + Insn::GuardLess { left, right, .. } + | Insn::GuardGreaterEq { left, right, .. } => { + self.assert_subtype(insn_id, left, types::CInt64)?; + self.assert_subtype(insn_id, right, types::CInt64) + }, + Insn::StringGetbyte { string, index } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::CInt64) + }, + Insn::StringSetbyteFixnum { string, index, value } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::Fixnum)?; + self.assert_subtype(insn_id, value, types::Fixnum) + } + Insn::IsA { val, class } => { + self.assert_subtype(insn_id, val, types::BasicObject)?; + self.assert_subtype(insn_id, class, types::Class) + } + Insn::RefineType { .. } => Ok(()), + Insn::HasType { val, .. } => self.assert_subtype(insn_id, val, types::BasicObject), + Insn::IsBlockParamModified { flags } => self.assert_subtype(insn_id, flags, types::CUInt64), + } + } + + /// Check that insn types match the expected types for each instruction. + fn validate_types(&self) -> Result<(), ValidationError> { + for block_id in self.reverse_post_order() { + for &insn_id in &self.blocks[block_id.0].insns { + self.validate_insn_type(insn_id)?; + } + } + Ok(()) + } + + /// Run all validation passes we have. + pub fn validate(&self) -> Result<(), ValidationError> { + self.validate_block_terminators_and_jumps()?; + self.validate_definite_assignment()?; + self.validate_insn_uniqueness()?; + self.validate_types()?; + Ok(()) + } +} + +impl<'a> std::fmt::Display for FunctionPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let fun = &self.fun; + // In tests, there may not be an iseq to get location from. + let iseq_name = if fun.iseq.is_null() { + String::from("<manual>") + } else { + iseq_get_location(fun.iseq, 0) + }; + + // In tests, strip the line number for builtin ISEQs to make tests stable across line changes + let iseq_name = if cfg!(test) && iseq_name.contains("@<internal:") { + iseq_name[..iseq_name.rfind(':').unwrap()].to_string() + } else { + iseq_name + }; + writeln!(f, "fn {iseq_name}:")?; + for block_id in fun.reverse_post_order() { + if !self.display_snapshot_and_tp_patchpoints && block_id == fun.entries_block { + // Unless we're doing --zjit-dump-hir=all, skip the entries superblock -- it's an + // internal CFG artifact + continue; + } + write!(f, "{block_id}(")?; + if !fun.blocks[block_id.0].params.is_empty() { + let mut sep = ""; + for param in &fun.blocks[block_id.0].params { + write!(f, "{sep}{param}")?; + let insn_type = fun.type_of(*param); + if !insn_type.is_subtype(types::Empty) { + write!(f, ":{}", insn_type.print(&self.ptr_map))?; + } + sep = ", "; + } + } + writeln!(f, "):")?; + for insn_id in &fun.blocks[block_id.0].insns { + let insn = fun.find(*insn_id); + if !self.display_snapshot_and_tp_patchpoints && + matches!(insn, Insn::Snapshot {..} | Insn::PatchPoint { invariant: Invariant::NoTracePoint, .. }) { + continue; + } + write!(f, " ")?; + if insn.has_output() { + let insn_type = fun.type_of(*insn_id); + if insn_type.is_subtype(types::Empty) { + write!(f, "{insn_id} = ")?; + } else { + write!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; + } + } + writeln!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FrameState { + pub iseq: IseqPtr, + insn_idx: YarvInsnIdx, + // Ruby bytecode instruction pointer + pub pc: *const VALUE, + + stack: Vec<InsnId>, + locals: Vec<InsnId>, +} + +impl FrameState { + /// Get the YARV instruction index for the current instruction + pub fn insn_idx(&self) -> YarvInsnIdx { + self.insn_idx + } + + /// Return itself without locals. Useful for side-exiting without spilling locals. + fn without_locals(&self) -> Self { + let mut state = self.clone(); + state.locals.clear(); + state + } + + /// Return itself without stack. Used by leaf calls with GC to reset SP to the base pointer. + pub fn without_stack(&self) -> Self { + let mut state = self.clone(); + state.stack.clear(); + state + } + + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. + /// `original_argc` is the number of args originally on the stack (before processing). + fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { + let mut state = self.clone(); + let args_start = state.stack.len() - original_argc; + state.stack.truncate(args_start); + state.stack.extend_from_slice(new_args); + state + } + + fn replace(&mut self, old: InsnId, new: InsnId) { + for slot in &mut self.stack { + if *slot == old { + *slot = new; + } + } + for slot in &mut self.locals { + if *slot == old { + *slot = new; + } + } + } +} + +/// Print adaptor for [`FrameState`]. See [`PtrPrintMap`]. +pub struct FrameStatePrinter<'a> { + inner: &'a FrameState, + ptr_map: &'a PtrPrintMap, +} + +/// Compute the index of a local variable from its slot index +fn ep_offset_to_local_idx(iseq: IseqPtr, ep_offset: u32) -> usize { + // Layout illustration + // This is an array of VALUE + // | VM_ENV_DATA_SIZE | + // v v + // low addr <+-------+-------+-------+-------+------------------+ + // |local 0|local 1| ... |local n| .... | + // +-------+-------+-------+-------+------------------+ + // ^ ^ ^ ^ + // +-------+---local_table_size----+ cfp->ep--+ + // | | + // +------------------ep_offset---------------+ + // + // See usages of local_var_name() from iseq.c for similar calculation. + + // Equivalent of iseq->body->local_table_size + let local_table_size: i32 = unsafe { get_iseq_body_local_table_size(iseq) } + .try_into() + .unwrap(); + let op = (ep_offset - VM_ENV_DATA_SIZE) as i32; + let local_idx = local_table_size - op - 1; + assert!(local_idx >= 0 && local_idx < local_table_size); + local_idx.try_into().unwrap() +} + +impl FrameState { + fn new(iseq: IseqPtr) -> FrameState { + FrameState { iseq, pc: std::ptr::null::<VALUE>(), insn_idx: 0, stack: vec![], locals: vec![] } + } + + /// Get the number of stack operands + pub fn stack_size(&self) -> usize { + self.stack.len() + } + + /// Iterate over all stack slots + pub fn stack(&self) -> Iter<'_, InsnId> { + self.stack.iter() + } + + /// Iterate over all local variables + pub fn locals(&self) -> Iter<'_, InsnId> { + self.locals.iter() + } + + /// Push a stack operand + fn stack_push(&mut self, opnd: InsnId) { + self.stack.push(opnd); + } + + /// Pop a stack operand + fn stack_pop(&mut self) -> Result<InsnId, ParseError> { + self.stack.pop().ok_or_else(|| ParseError::StackUnderflow(self.clone())) + } + + fn stack_pop_n(&mut self, count: usize) -> Result<Vec<InsnId>, ParseError> { + // Check if we have enough values on the stack + let stack_len = self.stack.len(); + if stack_len < count { + return Err(ParseError::StackUnderflow(self.clone())); + } + + Ok(self.stack.split_off(stack_len - count)) + } + + /// Get a stack-top operand + fn stack_top(&self) -> Result<InsnId, ParseError> { + self.stack.last().ok_or_else(|| ParseError::StackUnderflow(self.clone())).copied() + } + + /// Set a stack operand at idx + fn stack_setn(&mut self, idx: usize, opnd: InsnId) { + let idx = self.stack.len() - idx - 1; + self.stack[idx] = opnd; + } + + /// Get a stack operand at idx + fn stack_topn(&self, idx: usize) -> Result<InsnId, ParseError> { + let idx = self.stack.len() - idx - 1; + self.stack.get(idx).ok_or_else(|| ParseError::StackUnderflow(self.clone())).copied() + } + + fn setlocal(&mut self, ep_offset: u32, opnd: InsnId) { + let idx = ep_offset_to_local_idx(self.iseq, ep_offset); + self.locals[idx] = opnd; + } + + fn getlocal(&mut self, ep_offset: u32) -> InsnId { + let idx = ep_offset_to_local_idx(self.iseq, ep_offset); + self.locals[idx] + } + + fn as_args(&self, self_param: InsnId) -> Vec<InsnId> { + // We're currently passing around the self parameter as a basic block + // argument because the register allocator uses a fixed register based + // on the basic block argument index, which would cause a conflict if + // we reuse an argument from another basic block. + // TODO: Modify the register allocator to allow reusing an argument + // of another basic block. + let mut args = vec![self_param]; + args.extend(self.locals.iter().chain(self.stack.iter()).copied()); + args + } + + /// Get the opcode for the current instruction + pub fn get_opcode(&self) -> i32 { + unsafe { rb_iseq_opcode_at_pc(self.iseq, self.pc) } + } + + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> FrameStatePrinter<'a> { + FrameStatePrinter { inner: self, ptr_map } + } +} + +impl Display for FrameStatePrinter<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let inner = self.inner; + write!(f, "FrameState {{ pc: {:?}, stack: ", self.ptr_map.map_ptr(inner.pc))?; + write_vec(f, &inner.stack)?; + write!(f, ", locals: [")?; + for (idx, local) in inner.locals.iter().enumerate() { + let name: ID = unsafe { rb_zjit_local_id(inner.iseq, idx.try_into().unwrap()) }; + let name = name.contents_lossy(); + if idx > 0 { write!(f, ", ")?; } + write!(f, "{name}={local}")?; + } + write!(f, "] }}") + } +} + +/// Get YARV instruction argument +fn get_arg(pc: *const VALUE, arg_idx: isize) -> VALUE { + unsafe { *(pc.offset(arg_idx + 1)) } +} + +/// Compute YARV instruction index at relative offset +fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { + ((idx as isize) + (offset as isize)) as u32 +} + +struct BytecodeInfo { + jump_targets: Vec<u32>, +} + +fn compute_bytecode_info(iseq: *const rb_iseq_t, opt_table: &[u32]) -> BytecodeInfo { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + let mut jump_targets: HashSet<u32> = opt_table.iter().copied().collect(); + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + insn_idx += insn_len(opcode as usize); + match opcode { + YARVINSN_branchunless | YARVINSN_jump | YARVINSN_branchif | YARVINSN_branchnil + | YARVINSN_branchunless_without_ints | YARVINSN_jump_without_ints | YARVINSN_branchif_without_ints | YARVINSN_branchnil_without_ints => { + let offset = get_arg(pc, 0).as_i64(); + jump_targets.insert(insn_idx_at_offset(insn_idx, offset)); + } + YARVINSN_opt_new => { + let offset = get_arg(pc, 1).as_i64(); + jump_targets.insert(insn_idx_at_offset(insn_idx, offset)); + } + YARVINSN_leave | YARVINSN_opt_invokebuiltin_delegate_leave => { + if insn_idx < iseq_size { + jump_targets.insert(insn_idx); + } + } + _ => {} + } + } + let mut result = jump_targets.into_iter().collect::<Vec<_>>(); + result.sort(); + BytecodeInfo { jump_targets: result } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum CallType { + Splat, + Kwarg, + Tailcall, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ParseError { + StackUnderflow(FrameState), + MalformedIseq(u32), // insn_idx into iseq_encoded + Validation(ValidationError), + NotAllowed, + DirectiveInduced, +} + +/// Return the number of locals in the current ISEQ (includes parameters) +fn num_locals(iseq: *const rb_iseq_t) -> usize { + (unsafe { get_iseq_body_local_table_size(iseq) }).to_usize() +} + +/// If we can't handle the type of send (yet), bail out. +fn unhandled_call_type(flags: u32) -> Result<(), CallType> { + if (flags & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } + Ok(()) +} + +/// If a given call to a c func uses overly complex arguments, then we won't specialize. +fn unspecializable_c_call_type(flags: u32) -> bool { + ((flags & VM_CALL_KWARG) != 0) || + unspecializable_call_type(flags) +} + +/// If a given call uses overly complex arguments, then we won't specialize. +fn unspecializable_call_type(flags: u32) -> bool { + ((flags & VM_CALL_ARGS_SPLAT) != 0) || + ((flags & VM_CALL_KW_SPLAT) != 0) || + ((flags & VM_CALL_ARGS_BLOCKARG) != 0) || + ((flags & VM_CALL_FORWARDING) != 0) +} + +/// We have IseqPayload, which keeps track of HIR Types in the interpreter, but this is not useful +/// or correct to query from inside the optimizer. Instead, ProfileOracle provides an API to look +/// up profiled type information by HIR InsnId at a given ISEQ instruction. +#[derive(Debug)] +struct ProfileOracle { + payload: &'static IseqPayload, + /// types is a map from ISEQ instruction indices -> profiled type information at that ISEQ + /// instruction index. At a given ISEQ instruction, the interpreter has profiled the stack + /// operands to a given ISEQ instruction, and this list of pairs of (InsnId, Type) map that + /// profiling information into HIR instructions. + types: HashMap<YarvInsnIdx, Vec<(InsnId, TypeDistributionSummary)>>, +} + +impl ProfileOracle { + fn new(payload: &'static IseqPayload) -> Self { + Self { payload, types: Default::default() } + } + + /// Map the interpreter-recorded types of the stack onto the HIR operands on our compile-time virtual stack. + fn profile_stack(&mut self, state: &FrameState) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + // operand_types is always going to be <= stack size (otherwise it would have an underflow + // at run-time) so use that to drive iteration. + for (idx, insn_type_distribution) in operand_types.iter().rev().enumerate() { + let insn = state.stack_topn(idx).expect("Unexpected stack underflow in profiling"); + entry.push((insn, TypeDistributionSummary::new(insn_type_distribution))) + } + } + + /// Map the interpreter-recorded types of self onto the HIR self + fn profile_self(&mut self, state: &FrameState, self_param: InsnId) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + if operand_types.is_empty() { + return; + } + let self_type_distribution = &operand_types[0]; + entry.push((self_param, TypeDistributionSummary::new(self_type_distribution))) + } +} + +fn invalidates_locals(opcode: u32, operands: *const VALUE) -> bool { + match opcode { + // Control-flow is non-leaf in the interpreter because it can execute arbitrary code on + // interrupt. But in the JIT, we side-exit if there is a pending interrupt. + YARVINSN_jump + | YARVINSN_branchunless + | YARVINSN_branchif + | YARVINSN_branchnil + | YARVINSN_jump_without_ints + | YARVINSN_branchunless_without_ints + | YARVINSN_branchif_without_ints + | YARVINSN_branchnil_without_ints + | YARVINSN_leave => false, + // TODO(max): Read the invokebuiltin target from operands and determine if it's leaf + _ => unsafe { !rb_zjit_insn_leaf(opcode as i32, operands) } + } +} + +/// The index of the self parameter in the HIR function +pub const SELF_PARAM_IDX: usize = 0; + +/// Compile ISEQ into High-level IR +pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { + if !ZJITState::can_compile_iseq(iseq) { + return Err(ParseError::NotAllowed); + } + let payload = get_or_create_iseq_payload(iseq); + let mut profiles = ProfileOracle::new(payload); + let mut fun = Function::new(iseq); + fun.was_invalidated_for_singleton_class_creation = payload.was_invalidated_for_singleton_class_creation; + + // Compute a map of PC->Block by finding jump targets + let jit_entry_insns = unsafe { iseq.params() }.opt_table_slice().iter().copied().map(VALUE::as_u32).collect::<Vec<_>>(); + let BytecodeInfo { jump_targets } = compute_bytecode_info(iseq, &jit_entry_insns); + + // Make all empty basic blocks. The ordering of the BBs matters for getting fallthrough jumps + // in good places, but it's not necessary for correctness. TODO: Higher quality scheduling during lowering. + let mut insn_idx_to_block = HashMap::new(); + // Make blocks for optionals first, and put them right next to their JIT entrypoint + for insn_idx in jit_entry_insns.iter().copied() { + let jit_entry_block = fun.new_block(insn_idx); + fun.jit_entry_blocks.push(jit_entry_block); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Make blocks for the rest of the jump targets + for insn_idx in jump_targets { + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Done, drop `mut`. + let insn_idx_to_block = insn_idx_to_block; + + // Compile an entry_block for the interpreter + compile_entry_block(&mut fun, jit_entry_insns.as_slice(), &insn_idx_to_block); + + // Compile all JIT-to-JIT entry blocks + for (jit_entry_idx, insn_idx) in jit_entry_insns.iter().enumerate() { + let target_block = insn_idx_to_block.get(insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + compile_jit_entry_block(&mut fun, jit_entry_idx, target_block); + } + + // Check if the EP is escaped for the ISEQ from the beginning. We give up + // optimizing locals in that case because they're shared with other frames. + let ep_starts_escaped = iseq_escapes_ep(iseq); + // Check if the EP has been escaped at some point in the ISEQ. If it has, then we assume that + // its EP is shared with other frames. + let ep_has_been_escaped = crate::invariants::iseq_escapes_ep(iseq); + let ep_escaped = ep_starts_escaped || ep_has_been_escaped; + + // Iteratively fill out basic blocks using a queue. + // TODO(max): Basic block arguments at edges + let mut queue = VecDeque::new(); + for &insn_idx in jit_entry_insns.iter() { + queue.push_back((FrameState::new(iseq), insn_idx_to_block[&insn_idx], /*insn_idx=*/insn_idx, /*local_inval=*/false)); + } + + // Keep compiling blocks until the queue becomes empty + let mut visited = HashSet::new(); + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + while let Some((incoming_state, mut block, mut insn_idx, mut local_inval)) = queue.pop_front() { + // Compile each block only once + if visited.contains(&block) { continue; } + visited.insert(block); + + // Load basic block params first + let mut self_param = fun.push_insn(block, Insn::Param); + let mut state = { + let mut result = FrameState::new(iseq); + let local_size = if jit_entry_insns.contains(&insn_idx) { num_locals(iseq) } else { incoming_state.locals.len() }; + for _ in 0..local_size { + result.locals.push(fun.push_insn(block, Insn::Param)); + } + for _ in incoming_state.stack { + result.stack.push(fun.push_insn(block, Insn::Param)); + } + result + }; + + // Start the block off with a Snapshot so that if we need to insert a new Guard later on + // and we don't have a Snapshot handy, we can just iterate backward (at the earliest, to + // the beginning of the block). + fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + while insn_idx < iseq_size { + state.insn_idx = insn_idx as usize; + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + state.pc = pc; + let exit_state = state.clone(); + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + + // If TracePoint has been enabled after we have collected profiles, we'll see + // trace_getinstancevariable in the ISEQ. We have to treat it like getinstancevariable + // for profiling purposes: there is no operand on the stack to look up; we have + // profiled cfp->self. + if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_setinstancevariable || opcode == YARVINSN_trace_setinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_definedivar || opcode == YARVINSN_trace_definedivar { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some(operand_types) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + if let [self_type_distribution] = &operand_types[..] { + let summary = TypeDistributionSummary::new(&self_type_distribution); + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 } { + fun.count(block, Counter::invokeblock_handler_monomorphic_iseq); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 } { + fun.count(block, Counter::invokeblock_handler_monomorphic_ifunc); + } else { + fun.count(block, Counter::invokeblock_handler_monomorphic_other); + } + } else if summary.is_skewed_polymorphic() || summary.is_polymorphic() { + fun.count(block, Counter::invokeblock_handler_polymorphic); + } else if summary.is_skewed_megamorphic() || summary.is_megamorphic() { + fun.count(block, Counter::invokeblock_handler_megamorphic); + } else { + fun.count(block, Counter::invokeblock_handler_no_profiles); + } + } else { + fun.count(block, Counter::invokeblock_handler_no_profiles); + } + } + } + } else if opcode == YARVINSN_getblockparamproxy || opcode == YARVINSN_trace_getblockparamproxy { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some([block_handler_distribution]) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + let summary = TypeDistributionSummary::new(block_handler_distribution); + + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1} { + fun.count(block, Counter::getblockparamproxy_handler_iseq); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1} { + fun.count(block, Counter::getblockparamproxy_handler_ifunc); + } + else if obj.nil_p() { + fun.count(block, Counter::getblockparamproxy_handler_nil); + } + else if obj.symbol_p() { + fun.count(block, Counter::getblockparamproxy_handler_symbol); + } else if unsafe { rb_obj_is_proc(obj).test() } { + fun.count(block, Counter::getblockparamproxy_handler_proc); + } + } else if summary.is_polymorphic() || summary.is_skewed_polymorphic() { + fun.count(block, Counter::getblockparamproxy_handler_polymorphic); + } else if summary.is_megamorphic() || summary.is_skewed_megamorphic() { + fun.count(block, Counter::getblockparamproxy_handler_megamorphic); + } + } else { + fun.count(block, Counter::getblockparamproxy_handler_no_profiles); + } + } + } + else { + profiles.profile_stack(&exit_state); + } + + // Flag a future getlocal/setlocal to add a patch point if this instruction is not leaf. + if invalidates_locals(opcode, unsafe { pc.offset(1) }) { + local_inval = true; + } + + // We add NoTracePoint patch points before every instruction that could be affected by TracePoint. + // This ensures that if TracePoint is enabled, we can exit the generated code as fast as possible. + unsafe extern "C" { + fn rb_iseq_event_flags(iseq: IseqPtr, pos: usize) -> rb_event_flag_t; + } + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.clone() }); + if unsafe { rb_iseq_event_flags(iseq, insn_idx as usize) } != 0 { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state: exit_id }); + } + + // Increment zjit_insn_count for each YARV instruction if --zjit-stats is enabled. + if get_option!(stats) { + fun.push_insn(block, Insn::IncrCounter(Counter::zjit_insn_count)); + } + // Move to the next instruction to compile + insn_idx += insn_len(opcode as usize); + + match opcode { + YARVINSN_nop => {}, + YARVINSN_putnil => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); }, + YARVINSN_putobject => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) })); }, + YARVINSN_putspecialobject => { + let value_type = SpecialObjectType::from(get_arg(pc, 0).as_u32()); + let insn = if value_type == SpecialObjectType::VMCore { + Insn::Const { val: Const::Value(unsafe { rb_mRubyVMFrozenCore }) } + } else { + Insn::PutSpecialObject { value_type, state: exit_id } + }; + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_dupstring => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: false, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_dupchilledstring => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: true, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_putself => { state.stack_push(self_param); } + YARVINSN_intern => { + let val = state.stack_pop()?; + let insn_id = fun.push_insn(block, Insn::StringIntern { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_concatstrings => { + let count = get_arg(pc, 0).as_u32(); + let strings = state.stack_pop_n(count as usize)?; + let insn_id = fun.push_insn(block, Insn::StringConcat { strings, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_toregexp => { + // First arg contains the options (multiline, extended, ignorecase) used to create the regexp + let opt = get_arg(pc, 0).as_usize(); + let count = get_arg(pc, 1).as_usize(); + let values = state.stack_pop_n(count)?; + let insn_id = fun.push_insn(block, Insn::ToRegexp { opt, values, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_newarray => { + let count = get_arg(pc, 0).as_usize(); + let elements = state.stack_pop_n(count)?; + state.stack_push(fun.push_insn(block, Insn::NewArray { elements, state: exit_id })); + } + YARVINSN_opt_newarray_send => { + let count = get_arg(pc, 0).as_usize(); + let method = get_arg(pc, 1).as_u32(); + let (bop, insn) = match method { + VM_OPT_NEWARRAY_SEND_MAX => { + let elements = state.stack_pop_n(count)?; + (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_MIN => { + let elements = state.stack_pop_n(count)?; + (BOP_MIN, Insn::ArrayMin { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_HASH => { + let elements = state.stack_pop_n(count)?; + (BOP_HASH, Insn::ArrayHash { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_INCLUDE_P => { + let target = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_INCLUDE_P, Insn::ArrayInclude { elements, target, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK => { + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer: None, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => { + let buffer = state.stack_pop()?; + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 2)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer: Some(buffer), state: exit_id }) + } + _ => { + // Unknown opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method), recompile: None }); + break; // End the block + } + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None }); + break; // End the block + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_duparray => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::ArrayDup { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_opt_duparray_send => { + let ary = get_arg(pc, 0); + let method_id = get_arg(pc, 1).as_u64(); + let argc = get_arg(pc, 2).as_usize(); + if argc != 1 { + break; + } + let target = state.stack_pop()?; + let bop = match method_id { + x if x == ID!(include_p).0 => BOP_INCLUDE_P, + _ => { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id), recompile: None }); + break; + }, + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None }); + break; + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + let insn_id = fun.push_insn(block, Insn::DupArrayInclude { ary, target, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_newhash => { + let count = get_arg(pc, 0).as_usize(); + assert!(count % 2 == 0, "newhash count should be even"); + let mut elements = vec![]; + for _ in 0..(count/2) { + let value = state.stack_pop()?; + let key = state.stack_pop()?; + elements.push(value); + elements.push(key); + } + elements.reverse(); + state.stack_push(fun.push_insn(block, Insn::NewHash { elements, state: exit_id })); + } + YARVINSN_duphash => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::HashDup { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_splatarray => { + let flag = get_arg(pc, 0); + let result_must_be_mutable = flag.test(); + let val = state.stack_pop()?; + let obj = if result_must_be_mutable { + fun.push_insn(block, Insn::ToNewArray { val, state: exit_id }) + } else { + fun.push_insn(block, Insn::ToArray { val, state: exit_id }) + }; + state.stack_push(obj); + } + YARVINSN_splatkw => { + let block_val = state.stack_pop()?; + let hash = state.stack_pop()?; + // Get profiled type of hash (operand index 0) + let summary = profiles.payload.profile.get_operand_types(exit_state.insn_idx) + .and_then(|types| types.first()) + .map(|dist| TypeDistributionSummary::new(dist)); + let Some(summary) = summary else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotProfiled, recompile: None }); + break; // End the block + }; + if !summary.is_monomorphic() { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwPolymorphic, recompile: None }); + break; // End the block + } + let ty = Type::from_profiled_type(summary.bucket(0)); + let obj = if ty.is_subtype(types::NilClass) { + fun.push_insn(block, Insn::GuardType { val: hash, guard_type: types::NilClass, state: exit_id }) + } else if ty.is_subtype(types::HashExact) { + fun.push_insn(block, Insn::GuardType { val: hash, guard_type: types::HashExact, state: exit_id }) + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotNilOrHash, recompile: None }); + break; // End the block + }; + state.stack_push(obj); + state.stack_push(block_val); + } + YARVINSN_concattoarray => { + let right = state.stack_pop()?; + let left = state.stack_pop()?; + let right_array = fun.push_insn(block, Insn::ToArray { val: right, state: exit_id }); + fun.push_insn(block, Insn::ArrayExtend { left, right: right_array, state: exit_id }); + state.stack_push(left); + } + YARVINSN_pushtoarray => { + let count = get_arg(pc, 0).as_usize(); + let vals = state.stack_pop_n(count)?; + let array = state.stack_pop()?; + fun.guard_not_frozen(block, array, exit_id); + for val in vals.into_iter() { + fun.push_insn(block, Insn::ArrayPush { array, val, state: exit_id }); + } + state.stack_push(array); + } + YARVINSN_putobject_INT2FIX_0_ => { + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(0)) })); + } + YARVINSN_putobject_INT2FIX_1_ => { + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(1)) })); + } + YARVINSN_defined => { + // (rb_num_t op_type, VALUE obj, VALUE pushval) + let op_type = get_arg(pc, 0).as_usize(); + let obj = get_arg(pc, 1); + let pushval = get_arg(pc, 2); + let v = state.stack_pop()?; + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let insn = if op_type == DEFINED_YIELD as usize && unsafe { rb_get_iseq_body_type(local_iseq) } != ISEQ_TYPE_METHOD { + // `yield` goes to the block handler stowed in the "local" iseq which is + // the current iseq or a parent. Only the "method" iseq type can be passed a + // block handler. (e.g. `yield` in the top level script is a syntax error.) + // + // Similar to gen_is_block_given + Insn::Const { val: Const::Value(Qnil) } + } else { + // For DEFINED_YIELD, codegen materializes the local EP inline (similar to + // gen_is_block_given) to check for a block handler. Precompute the lexical + // distance from this iseq up to local_iseq so codegen does not have to + // walk the parent chain. Any DEFINED_YIELD reaching this branch has a + // method local_iseq by construction -- the above branch has already + // diverted the non-method case to Qnil. + let lep_level = if op_type == DEFINED_YIELD as usize { + get_lvar_level(iseq) + } else { + 0 + }; + Insn::Defined { op_type, obj, pushval, v, lep_level, state: exit_id } + }; + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_definedivar => { + // (ID id, IVC ic, VALUE pushval) + let id = ID(get_arg(pc, 0).as_u64()); + let pushval = get_arg(pc, 2); + state.stack_push(fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id })); + } + YARVINSN_checkkeyword => { + // When a keyword is unspecified past index 32, a hash will be used instead. + // This can only happen in iseqs taking more than 32 keywords. + // In this case, we side exit to the interpreter. + if unsafe {(*rb_get_iseq_body_param_keyword(iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap()} { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters, recompile: None }); + break; + } + let ep_offset = get_arg(pc, 0).as_u32(); + let index = get_arg(pc, 1).as_u64(); + let index: u8 = index.try_into().map_err(|_| ParseError::MalformedIseq(insn_idx))?; + // Use FrameState to get kw_bits when possible, just like getlocal_WC_0. + let val = if !local_inval { + state.getlocal(ep_offset) + } else if ep_escaped { + let ep = fun.push_insn(block, Insn::GetEP { level: 0 }); + fun.get_local_from_ep(block, ep, ep_offset, 0, types::BasicObject) + } else { + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + state.getlocal(ep_offset) + }; + state.stack_push(fun.push_insn(block, Insn::FixnumBitCheck { val, index })); + } + YARVINSN_checkmatch => { + let flag = get_arg(pc, 0).as_u32(); + let pattern = state.stack_pop()?; + let target = state.stack_pop()?; + let result = fun.push_insn(block, Insn::CheckMatch { target, pattern, flag, state: exit_id }); + state.stack_push(result); + } + YARVINSN_getconstant => { + let id = ID(get_arg(pc, 0).as_u64()); + let allow_nil = state.stack_pop()?; + let klass = state.stack_pop()?; + let result = fun.push_insn(block, Insn::GetConstant { klass, id, allow_nil, state: exit_id }); + state.stack_push(result); + } + YARVINSN_opt_getconstant_path => { + let ic = get_arg(pc, 0).as_ptr(); + let get_const_path = fun.push_insn(block, Insn::GetConstantPath { ic, state: exit_id }); + state.stack_push(get_const_path); + + // Check for `::RubyVM::ZJIT` for directives + unsafe { + let mut current_segment = (*ic).segments; + let mut segments = [ID(0); 4 /* expected segment length */]; + for segment in segments.iter_mut() { + *segment = current_segment.read(); + if *segment == ID(0) { + break; + } + current_segment = current_segment.add(1); + } + if [ID!(NULL), ID!(RubyVM), ID!(ZJIT), ID(0)] == segments { + debug_assert_ne!(ID!(NULL), ID(0)); + let ruby_vm_mod = rb_const_lookup(rb_cObject, ID!(RubyVM)); + if !ruby_vm_mod.is_null() && (*ruby_vm_mod).value == rb_cRubyVM { + let zjit_module = VALUE(state::ZJIT_MODULE.load(Ordering::Relaxed)); + let lookedup_module = rb_const_lookup(rb_cRubyVM, ID!(ZJIT)); + if !lookedup_module.is_null() && (*lookedup_module).value == zjit_module { + fun.insn_types[get_const_path.0] = Type::from_value(zjit_module); + } + } + } + } + } + YARVINSN_branchunless | YARVINSN_branchunless_without_ints => { + if opcode == YARVINSN_branchunless { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::Test { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + let mut iffalse_state = state.clone(); + iffalse_state.replace(val, nil_false); + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target, args: iffalse_state.as_args(self_param) } + }); + + block = fall_through; + + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + state.replace(val, not_nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_branchif | YARVINSN_branchif_without_ints => { + if opcode == YARVINSN_branchif { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::Test { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, not_nil_false); + + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target, args: iftrue_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + + block = fall_through; + + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + state.replace(val, nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_branchnil | YARVINSN_branchnil_without_ints => { + if opcode == YARVINSN_branchnil { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::IsNil { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let nil = fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, nil); + + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target, args: iftrue_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + + block = fall_through; + let new_type = types::NotNil; + let not_nil = fun.push_insn(block, Insn::RefineType { val, new_type }); + state.replace(val, not_nil); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_opt_case_dispatch => { + // TODO: Some keys are visible at compile time, so in the future we can + // compile jump targets for certain cases + // Pop the key from the stack and fallback to the === branches for now + state.stack_pop()?; + } + YARVINSN_opt_new => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let dst = get_arg(pc, 1).as_i64(); + + // Check if #new resolves to rb_class_new_instance_pass_kw. + // TODO: Guard on a profiled class and add a patch point for #new redefinition + let argc = crate::profile::num_arguments_on_stack(cd); + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert_eq!(flags & VM_CALL_ARGS_BLOCKARG, 0); + let val = state.stack_topn(argc)?; + let test_id = fun.push_insn(block, Insn::IsMethodCfunc { val, cd, cfunc: rb_class_new_instance_pass_kw as *const u8, state: exit_id }); + + // Jump to the fallback block if it's not the expected function. + // Skip CheckInterrupts since the #new call will do it very soon anyway. + let target_idx = insn_idx_at_offset(insn_idx, dst); + let target = insn_idx_to_block[&target_idx]; + let fall_through = fun.new_block(insn_idx); + fun.push_insn(block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target, args: state.as_args(self_param) } + }); + block = fall_through; + queue.push_back((state.clone(), target, target_idx, local_inval)); + + // Move on to the fast path + let insn_id = fun.push_insn(block, Insn::ObjectAlloc { val, state: exit_id }); + state.stack_setn(argc, insn_id); + state.stack_setn(argc + 1, insn_id); + } + YARVINSN_jump | YARVINSN_jump_without_ints => { + let offset = get_arg(pc, 0).as_i64(); + if opcode == YARVINSN_jump { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let _branch_id = fun.push_insn(block, Insn::Jump( + BranchEdge { target, args: state.as_args(self_param) } + )); + queue.push_back((state.clone(), target, target_idx, local_inval)); + break; // Don't enqueue the next block as a successor + } + YARVINSN_getlocal_WC_0 => { + let ep_offset = get_arg(pc, 0).as_u32(); + if !local_inval { + // The FrameState is the source of truth for locals until invalidated. + // In case of JIT-to-JIT send locals might never end up in EP memory. + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else if ep_escaped { + // Read the local using EP + let ep = fun.push_insn(block, Insn::GetEP { level: 0 }); + let val = fun.get_local_from_ep(block, ep, ep_offset, 0, types::BasicObject); + state.setlocal(ep_offset, val); // remember the result to spill on side-exits + state.stack_push(val); + } else { + assert!(local_inval); // if check above + // There has been some non-leaf call since JIT entry or the last patch point, + // so add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + + // Read the local from FrameState + let val = state.getlocal(ep_offset); + state.stack_push(val); + } + } + YARVINSN_setlocal_WC_0 => { + let ep_offset = get_arg(pc, 0).as_u32(); + let val = state.stack_pop()?; + if ep_escaped { + // Write the local using EP + fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 }); + } else if local_inval { + // If there has been any non-leaf call since JIT entry or the last patch point, + // add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + } + // Write the local into FrameState + state.setlocal(ep_offset, val); + } + YARVINSN_getlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + let ep = fun.push_insn(block, Insn::GetEP { level: 1 }); + state.stack_push(fun.get_local_from_ep(block, ep, ep_offset, 1, types::BasicObject)); + } + YARVINSN_setlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level: 1 }); + } + YARVINSN_getlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + if level == 0 && !local_inval { + // Same optimization as getlocal_WC_0: use FrameState + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else { + let ep = fun.push_insn(block, Insn::GetEP { level }); + let val = fun.get_local_from_ep(block, ep, ep_offset, level, types::BasicObject); + if level == 0 { + state.setlocal(ep_offset, val); + } + state.stack_push(val); + } + } + YARVINSN_setlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level }); + } + YARVINSN_setblockparam => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetLocal { val, ep_offset, level }); + if level == 0 { + state.setlocal(ep_offset, val); + } + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.push_insn(block, Insn::LoadField { + recv: ep, + id: FieldName::VM_ENV_DATA_INDEX_FLAGS, + offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), + return_type: types::CInt64, + }); + let modified_flag = fun.push_insn(block, Insn::Const { + val: Const::CInt64(VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()), + }); + let modified = fun.push_insn(block, Insn::IntOr { left: flags, right: modified_flag }); + fun.push_insn(block, Insn::StoreField { + recv: ep, + id: FieldName::VM_ENV_DATA_INDEX_FLAGS, + offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), + val: modified, + }); + } + YARVINSN_getblockparamproxy => { + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + enum ProfiledBlockHandlerFamily { + Nil, + IseqOrIfunc, + } + impl ProfiledBlockHandlerFamily { + fn from_profiled_type(profiled_type: ProfiledType) -> Option<Self> { + let obj = profiled_type.class(); + if obj.nil_p() { + Some(Self::Nil) + } else if unsafe { + rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 + || rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 + } { + Some(Self::IseqOrIfunc) + } else { + None + } + } + } + + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let branch_insn_idx = exit_state.insn_idx as u32; + + // `getblockparamproxy` has two semantic paths: + // - modified: return the already-materialized block local from EP + // - unmodified: inspect the block handler and produce proxy/nil + let modified_block = fun.new_block(branch_insn_idx); + let unmodified_block = fun.new_block(branch_insn_idx); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_result = fun.push_insn(join_block, Insn::Param); + let join_local = if level == 0 { Some(fun.push_insn(join_block, Insn::Param)) } else { None }; + + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); + + fun.push_insn(block, Insn::CondBranch { + val: is_modified, + if_true: BranchEdge { target: modified_block, args: vec![] }, + if_false: BranchEdge { target: unmodified_block, args: vec![] } + }); + + // Push modified block: load the block local via EP. + let modified_val = fun.get_local_from_ep(modified_block, ep, ep_offset, level, types::BasicObject); + let mut modified_args = vec![modified_val]; + if level == 0 { modified_args.push(modified_val); } + fun.push_insn(modified_block, Insn::Jump(BranchEdge { target: join_block, args: modified_args })); + + // Push unmodified block: inspect the current block handler to + // decide whether this path returns `nil` or `BlockParamProxy`. + let block_handler = fun.push_insn(unmodified_block, Insn::LoadField { recv: ep, id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::CInt64 }); + let original_local = if level == 0 { Some(state.getlocal(ep_offset)) } else { None }; + // `block_handler & 1 == 1` accepts both ISEQ (0b01) and ifunc + // (0b11) handlers. Keep a compile-time check that this shortcut + // does not accidentally accept symbol block handlers. + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + + let profiled_block_summary = profiles.payload.profile.get_operand_types(exit_state.insn_idx) + .and_then(|types| types.first()) + .map(TypeDistributionSummary::new); + + let mut profiled_handlers = Vec::new(); + if let Some(summary) = profiled_block_summary.as_ref() { + if summary.is_monomorphic() || summary.is_polymorphic() || summary.is_skewed_polymorphic() { + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { + break; + } + if let Some(profiled_handler) = ProfiledBlockHandlerFamily::from_profiled_type(profiled_type) { + if !profiled_handlers.contains(&profiled_handler) { + profiled_handlers.push(profiled_handler); + } + } + } + } + } + + match profiled_handlers.as_slice() { + // No supported profiled families. Keep the generic fallback iseq/ifunc fallback + // for sites we do not specialize, such as no-profile and megamorphic sites. + [] => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyFallbackMiss, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + // A single supported profiled family. Emit a monomorphic fast path + [profiled_handler] => match profiled_handler { + ProfiledBlockHandlerFamily::Nil => { + fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: SideExitReason::BlockParamProxyNotNil, state: exit_id, recompile: None }); + let nil_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![nil_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyNotIseqOrIfunc, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + }, + // Multiple supported profiled families. Emit a polymorphic dispatch + _ => { + let profiled_blocks = profiled_handlers.iter() + .map(|&kind| (kind, fun.new_block(branch_insn_idx))) + .collect::<Vec<_>>(); + + let mut current_block = unmodified_block; + + for &(kind, profiled_block) in &profiled_blocks { + match kind { + ProfiledBlockHandlerFamily::Nil => { + let none_handler = fun.push_insn(current_block, Insn::Const { + val: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), + }); + let is_none = fun.push_insn(current_block, Insn::IsBitEqual { + left: block_handler, + right: none_handler, + }); + + let next_block = fun.new_block(branch_insn_idx); + + fun.push_insn(current_block, Insn::CondBranch { + val: is_none, + if_true: BranchEdge { target: profiled_block, args: vec![] }, + if_false: BranchEdge { target: next_block, args: vec![] }, + }); + + current_block = next_block; + + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + let tag_mask = fun.push_insn(current_block, Insn::Const { val: Const::CInt64(0x1) }); + let tag_bits = fun.push_insn(current_block, Insn::IntAnd { + left: block_handler, + right: tag_mask, + }); + let is_iseq_or_ifunc = fun.push_insn(current_block, Insn::IsBitEqual { + left: tag_bits, + right: tag_mask, + }); + let next_block = fun.new_block(branch_insn_idx); + fun.push_insn(current_block, Insn::CondBranch { + val: is_iseq_or_ifunc, + if_true: BranchEdge { target: profiled_block, args: vec![] }, + if_false: BranchEdge { target: next_block, args: vec![] }, + }); + current_block = next_block; + + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + }, + } + } + + fun.push_insn(current_block, Insn::SideExit { state: exit_id, reason: SideExitReason::BlockParamProxyProfileNotCovered, recompile: None }); + } + } + + // Continue compilation from the merged continuation block at the next + // instruction. + if let Some(local_param) = join_local { + state.setlocal(ep_offset, local_param); + } + state.stack_push(join_result); + block = join_block; + } + YARVINSN_getblockparam => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let branch_insn_idx = exit_state.insn_idx as u32; + + // If the block param is already a Proc (modified), read it from EP. + // Otherwise, convert it to a Proc and store it to EP. + let modified_block = fun.new_block(branch_insn_idx); + let unmodified_block = fun.new_block(branch_insn_idx); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_param = fun.push_insn(join_block, Insn::Param); + + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); + + fun.push_insn(block, Insn::CondBranch { + val: is_modified, + if_true: BranchEdge { target: modified_block, args: vec![] }, + if_false: BranchEdge { target: unmodified_block, args: vec![] } + }); + + // Push modified block: read Proc from EP. + let modified_val = fun.get_local_from_ep(modified_block, ep, ep_offset, level, types::BasicObject); + fun.push_insn(modified_block, Insn::Jump(BranchEdge { target: join_block, args: vec![modified_val] })); + + // Push unmodified block: convert block handler to Proc. + let unmodified_val = fun.push_insn(unmodified_block, Insn::GetBlockParam { + ep_offset, + level, + state: exit_id, + }); + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args: vec![unmodified_val] })); + + // Continue compilation from the join block at the next instruction. + if level == 0 { + state.setlocal(ep_offset, join_param); + } + state.stack_push(join_param); + block = join_block; + } + YARVINSN_pop => { state.stack_pop()?; } + YARVINSN_dup => { state.stack_push(state.stack_top()?); } + YARVINSN_dupn => { + // Duplicate the top N element of the stack. As we push, n-1 naturally + // points higher in the original stack. + let n = get_arg(pc, 0).as_usize(); + for _ in 0..n { + state.stack_push(state.stack_topn(n-1)?); + } + } + YARVINSN_swap => { + let right = state.stack_pop()?; + let left = state.stack_pop()?; + state.stack_push(right); + state.stack_push(left); + } + YARVINSN_setn => { + let n = get_arg(pc, 0).as_usize(); + let top = state.stack_top()?; + state.stack_setn(n, top); + } + YARVINSN_topn => { + let n = get_arg(pc, 0).as_usize(); + let top = state.stack_topn(n)?; + state.stack_push(top); + } + YARVINSN_adjuststack => { + let mut n = get_arg(pc, 0).as_usize(); + while n > 0 { + state.stack_pop()?; + n -= 1; + } + } + YARVINSN_opt_neq => { + // NB: opt_neq has two cd; get_arg(0) is for eq and get_arg(1) is for neq + let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + let argc = crate::profile::num_arguments_on_stack(cd); + assert_eq!(flags & VM_CALL_ARGS_BLOCKARG, 0); + + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + } + YARVINSN_opt_hash_freeze => { + let klass = HASH_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_opt_ary_freeze => { + let klass = ARRAY_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_opt_str_freeze => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_opt_str_uminus => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_UMINUS; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_leave => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + fun.push_insn(block, Insn::Return { val: state.stack_pop()? }); + break; // Don't enqueue the next block as a successor + } + YARVINSN_throw => { + fun.push_insn(block, Insn::Throw { throw_state: get_arg(pc, 0).as_u32(), val: state.stack_pop()?, state: exit_id }); + break; // Don't enqueue the next block as a successor + } + + // These are opt_send_without_block and all the opt_* instructions + // specialized to a certain method that could also be serviced + // using the general send implementation. The optimizer start from + // a general send for all of these later in the pipeline. + YARVINSN_opt_nil_p | + YARVINSN_opt_plus | + YARVINSN_opt_minus | + YARVINSN_opt_mult | + YARVINSN_opt_div | + YARVINSN_opt_mod | + YARVINSN_opt_eq | + YARVINSN_opt_lt | + YARVINSN_opt_le | + YARVINSN_opt_gt | + YARVINSN_opt_ge | + YARVINSN_opt_ltlt | + YARVINSN_opt_aset | + YARVINSN_opt_length | + YARVINSN_opt_size | + YARVINSN_opt_aref | + YARVINSN_opt_empty_p | + YARVINSN_opt_succ | + YARVINSN_opt_and | + YARVINSN_opt_or | + YARVINSN_opt_not | + YARVINSN_opt_regexpmatch2 | + YARVINSN_opt_send_without_block => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + let argc = crate::profile::num_arguments_on_stack(cd); + let mid = unsafe { rb_vm_ci_mid(call_info) }; + + // Check for calls to directives + if argc == 0 + && (mid == ID!(induce_side_exit_bang) || mid == ID!(induce_compile_failure_bang) || mid == ID!(induce_breakpoint_bang)) + && fun.type_of(state.stack_top()?) + .ruby_object() + .is_some_and(|obj| obj == VALUE(state::ZJIT_MODULE.load(Ordering::Relaxed))) + { + + if mid == ID!(induce_side_exit_bang) + && state::zjit_module_method_match_serial(ID!(induce_side_exit_bang), &state::INDUCE_SIDE_EXIT_SERIAL) + { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::DirectiveInduced, recompile: None }); + break; // End the block + } + if mid == ID!(induce_compile_failure_bang) + && state::zjit_module_method_match_serial(ID!(induce_compile_failure_bang), &state::INDUCE_COMPILE_FAILURE_SERIAL) + { + return Err(ParseError::DirectiveInduced); + } + if mid == ID!(induce_breakpoint_bang) + && state::zjit_module_method_match_serial(ID!(induce_breakpoint_bang), &state::INDUCE_BREAKPOINT_SERIAL) + { + fun.push_insn(block, Insn::BreakPoint); + state.stack_pop()?; // pop the receiver (::RubyVM::ZJIT) + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); + } + } + + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + + { + fn new_branch_block( + fun: &mut Function, + cd: *const rb_call_data, + argc: usize, + opcode: u32, + new_type: Type, + insn_idx: u32, + exit_state: &FrameState, + locals_count: usize, + stack_count: usize, + join_block: BlockId, + ) -> BlockId { + let block = fun.new_block(insn_idx); + let self_param = fun.push_insn(block, Insn::Param); + let mut state = exit_state.clone(); + state.locals.clear(); + state.stack.clear(); + state.locals.extend((0..locals_count).map(|_| fun.push_insn(block, Insn::Param))); + state.stack.extend((0..stack_count).map(|_| fun.push_insn(block, Insn::Param))); + let snapshot = fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + let args = state.stack_pop_n(argc).unwrap(); + let recv = state.stack_pop().unwrap(); + let refined_recv = fun.push_insn(block, Insn::RefineType { val: recv, new_type }); + state.replace(recv, refined_recv); + let send = fun.push_insn(block, Insn::Send { recv: refined_recv, cd, block: None, args, state: snapshot, reason: Uncategorized(opcode) }); + state.stack_push(send); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: state.as_args(self_param) })); + block + } + let branch_insn_idx = exit_state.insn_idx as u32; + let locals_count = state.locals.len(); + let stack_count = state.stack.len(); + let recv = state.stack_topn(argc as usize)?; // args are on top + let entry_args = state.as_args(self_param); + if let Some(summary) = fun.polymorphic_summary(&profiles, recv, exit_state.insn_idx) { + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + // Dedup by expected type so immediate/heap variants + // under the same Ruby class can still get separate branches. + let mut seen_types = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { break; } + let expected = Type::from_profiled_type(profiled_type); + if seen_types.iter().any(|ty: &Type| ty.bit_equal(expected)) { + continue; + } + seen_types.push(expected); + let has_type = fun.push_insn(block, Insn::HasType { val: recv, expected }); + let iftrue_block = + new_branch_block(&mut fun, cd, argc as usize, opcode, expected, branch_insn_idx, &exit_state, locals_count, stack_count, join_block); + let target = BranchEdge { target: iftrue_block, args: entry_args.clone() }; + let fall_through = fun.new_block(insn_idx); + fun.push_insn(block, Insn::CondBranch { + val: has_type, + if_true: target, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + block = fall_through; + } + // Continue compilation from the join block at the next instruction. + // Make a copy of the current state without the args (pop the receiver + // and push the result) because we just use the locals/stack sizes to + // make the right number of Params + let mut join_state = state.clone(); + join_state.stack_pop_n(argc as usize)?; + queue.push_back((join_state, join_block, insn_idx, local_inval)); + // In the fallthrough case, do a generic interpreter send and then join. + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let reason = SendWithoutBlockPolymorphicFallback; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason }); + state.stack_push(send); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: state.as_args(self_param) })); + break; // End the block + } + } + + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + } + YARVINSN_send => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; + let recv = state.stack_pop()?; + let block_handler = if !blockiseq.is_null() { + Some(BlockHandler::BlockIseq(blockiseq)) + } else if block_arg { + Some(BlockHandler::BlockArg) + } else { + None + }; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + + if let Some(BlockHandler::BlockIseq(_)) = block_handler { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_sendforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: SendForwardNotSpecialized }); + state.stack_push(send_forward); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokesuper => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; + let recv = state.stack_pop()?; + let blockiseq: IseqPtr = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::InvokeSuper { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokesuperforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: InvokeSuperForwardNotSpecialized }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokeblock => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; + + // Check if this is a monomorphic IFUNC block handler we can specialize + let block_handler_types = profiles.payload.profile.get_operand_types(exit_state.insn_idx); + let is_ifunc = (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT)) == 0 + && block_handler_types.is_some_and(|types| types.len() == 1 && { + let summary = TypeDistributionSummary::new(&types[0]); + summary.is_monomorphic() && unsafe { rb_IMEMO_TYPE_P(summary.bucket(0).class(), imemo_ifunc) == 1 } + }); + + let result = if is_ifunc { + // Load the block handler from LEP + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + let block_handler = fun.push_insn(block, Insn::LoadField { + recv: lep, + id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, + offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, + return_type: types::CInt64, + }); + + // Check IFUNC tag: (block_handler & 0x3) == 0x3 + let tag_mask = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let tag_bits = fun.push_insn(block, Insn::IntAnd { left: block_handler, right: tag_mask }); + let ifunc_tag = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let is_ifunc_match = fun.push_insn(block, Insn::IsBitEqual { left: tag_bits, right: ifunc_tag }); + + // Branch: on match, call InvokeBlockIfunc directly + let join_block = fun.new_block(insn_idx); + let join_param = fun.push_insn(join_block, Insn::Param); + let ifunc_block = fun.new_block(insn_idx); + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { + val: is_ifunc_match, + if_true: BranchEdge { target: ifunc_block, args: vec![] }, + if_false: BranchEdge { target: fall_through, args: vec![] }, + }); + + block = fall_through; + + let ifunc_result = fun.push_insn(ifunc_block, Insn::InvokeBlockIfunc { + cd, + block_handler, + args: args.clone(), + state: exit_id, + }); + fun.push_insn(ifunc_block, Insn::Jump(BranchEdge { target: join_block, args: vec![ifunc_result] })); + + // In the fallthrough case, use generic rb_vm_invokeblock and join + let fallback_result = fun.push_insn(block, Insn::InvokeBlock { + cd, args, state: exit_id, reason: InvokeBlockNotSpecialized, + }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); + + // Continue compilation from the join block + block = join_block; + join_param + } else { + fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }) + }; + state.stack_push(result); + } + YARVINSN_getglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let result = fun.push_insn(block, Insn::GetGlobal { id, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetGlobal { id, val, state: exit_id }); + } + YARVINSN_getinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // ic is in arg 1 + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_getivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_getivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + if let Some(summary) = fun.polymorphic_summary(&profiles, self_param, exit_state.insn_idx) { + self_param = fun.push_insn(block, Insn::GuardType { val: self_param, guard_type: types::HeapBasicObject, state: exit_id }); + let rbasic_flags = fun.load_rbasic_flags(block, self_param); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_param = fun.push_insn(join_block, Insn::Param); + // Dedup by expected shape so objects with different classes but the same shape can share code + // TODO(max): De-duplicate further by checking ivar offsets to allow + // different shapes with the same ivar layout to share code + let mut seen_shape_and_flags = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + // End of the buckets + if profiled_type.is_empty() { break; } + // Instance variable lookups on immediate values are always nil; don't bother + if profiled_type.flags().is_immediate() { continue; } + let expected_shape = profiled_type.shape(); + let (expected_rbasic_flags, rbasic_flags_mask) = profiled_type.rbasic_flags_and_mask(); + assert!(expected_shape.is_valid()); + // Too-complex shapes use hash tables for ivars; + // rb_shape_get_iv_index doesn't work for them. + // Let the fallthrough GetIvar handle these. + if expected_shape.is_complex() { continue; } + if seen_shape_and_flags.contains(&expected_rbasic_flags) { continue; } + seen_shape_and_flags.push(expected_rbasic_flags); + let rbasic_flags_mask = fun.push_insn(block, Insn::Const { val: Const::CUInt64(rbasic_flags_mask) }); + // The expected shape can change over run, so we put it + // as a pointer to keep it stable in snapshot tests. + let expected_rbasic_flags = fun.push_insn(block, Insn::Const { val: Const::CPtr(ptr::without_provenance(expected_rbasic_flags.to_usize())) }); + let expected_rbasic_flags = fun.push_insn(block, Insn::RefineType { val: expected_rbasic_flags, new_type: types::CUInt64 }); + let masked = fun.push_insn(block, Insn::IntAnd { left: rbasic_flags, right: rbasic_flags_mask}); + let has_shape_and_type = fun.push_insn(block, Insn::IsBitEqual { left: masked, right: expected_rbasic_flags }); + let iftrue_block = fun.new_block(insn_idx); + let target = BranchEdge { target: iftrue_block, args: vec![] }; + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: has_shape_and_type, + if_true: target, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + + block = fall_through; + let result = fun.load_ivar(iftrue_block, self_param, profiled_type, id, exit_id); + fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + } + // In the fallthrough case, do a generic interpreter getivar and then join. + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + state.stack_push(join_param); + // Continue compilation from the join block at the next instruction. + // Make a copy of the current state without the args (pop the receiver + // and push the result) because we just use the locals/stack sizes to + // make the right number of Params + block = join_block; + } else { + // Possibly monomorphic case; handled in optimize_getivar + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + state.stack_push(result); + } + } + YARVINSN_setinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_setivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_setivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id }); + // SetIvar will raise if self is an immediate. If it raises, we will have + // exited JIT code. So upgrade the type within JIT code to a heap object. + self_param = fun.push_insn(block, Insn::RefineType { val: self_param, new_type: types::HeapBasicObject }); + } + YARVINSN_getclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::GetClassVar { id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetClassVar { id, val, ic, state: exit_id }); + } + YARVINSN_opt_reverse => { + // Reverse the order of the top N stack items. + let n = get_arg(pc, 0).as_usize(); + for i in 0..n/2 { + let bottom = state.stack_topn(n - 1 - i)?; + let top = state.stack_topn(i)?; + state.stack_setn(i, bottom); + state.stack_setn(n - 1 - i, top); + } + } + YARVINSN_newrange => { + let flag = RangeType::from(get_arg(pc, 0).as_u32()); + let high = state.stack_pop()?; + let low = state.stack_pop()?; + let insn_id = fun.push_insn(block, Insn::NewRange { low, high, flag, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_invokebuiltin => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + + let mut args = vec![]; + for _ in 0..bf.argc { + args.push(state.stack_pop()?); + } + args.push(self_param); + args.reverse(); + + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_opt_invokebuiltin_delegate | + YARVINSN_opt_invokebuiltin_delegate_leave => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let index = get_arg(pc, 1).as_usize(); + let argc = bf.argc as usize; + + let mut args = vec![self_param]; + for &local in state.locals().skip(index).take(argc) { + args.push(local); + } + + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_objtostring => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let argc = crate::profile::num_arguments_on_stack(cd); + assert_eq!(0, argc, "objtostring should not have args"); + + let recv = state.stack_pop()?; + let objtostring = fun.push_insn(block, Insn::ObjToString { val: recv, cd, state: exit_id }); + state.stack_push(objtostring) + } + YARVINSN_anytostring => { + let str = state.stack_pop()?; + let val = state.stack_pop()?; + + let anytostring = fun.push_insn(block, Insn::AnyToString { val, str, state: exit_id }); + state.stack_push(anytostring); + } + YARVINSN_getspecial => { + let key = get_arg(pc, 0).as_u64(); + let svar = get_arg(pc, 1).as_u64(); + + if svar == 0 { + // TODO: Handle non-backref + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key), recompile: None }); + // End the block + break; + } else if svar & 0x01 != 0 { + // Handle symbol backrefs like $&, $`, $', $+ + let shifted_svar: u8 = (svar >> 1).try_into().unwrap(); + let symbol_type = SpecialBackrefSymbol::try_from(shifted_svar).expect("invalid backref symbol"); + let result = fun.push_insn(block, Insn::GetSpecialSymbol { symbol_type, state: exit_id }); + state.stack_push(result); + } else { + // Handle number backrefs like $1, $2, $3 + let result = fun.push_insn(block, Insn::GetSpecialNumber { nth: svar, state: exit_id }); + state.stack_push(result); + } + } + YARVINSN_expandarray => { + let num = get_arg(pc, 0).as_u64(); + let flag = get_arg(pc, 1).as_u64(); + if flag != 0 { + // We don't (yet) handle 0x01 (rest args), 0x02 (post args), or 0x04 + // (reverse?) + // + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + let val = state.stack_pop()?; + let array = fun.push_insn(block, Insn::GuardType { val, guard_type: types::ArrayExact, state: exit_id, }); + let length = fun.push_insn(block, Insn::ArrayLength { array }); + let expected = fun.push_insn(block, Insn::Const { val: Const::CInt64(num as i64) }); + fun.push_insn(block, Insn::GuardGreaterEq { left: length, right: expected, reason: SideExitReason::ExpandArray, state: exit_id }); + for i in (0..num).rev() { + // We do not emit a length guard here because in-bounds is already + // ensured by the expandarray length check above. + let index = fun.push_insn(block, Insn::Const { val: Const::CInt64(i.try_into().unwrap()) }); + let element = fun.push_insn(block, Insn::ArrayAref { array, index }); + state.stack_push(element); + } + } + _ => { + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + } + + if insn_idx_to_block.contains_key(&insn_idx) { + let target = insn_idx_to_block[&insn_idx]; + fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args(self_param) })); + queue.push_back((state, target, insn_idx, local_inval)); + break; // End the block + } + } + } + + // Populate the entries superblock with an Entries instruction targeting all entry blocks + fun.seal_entries(); + + fun.set_param_types(); + fun.infer_types(); + + match get_option!(dump_hir_init) { + Some(DumpHIR::WithoutSnapshot) => println!("Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)), + Some(DumpHIR::All) => println!("Initial HIR:\n{}", FunctionPrinter::with_snapshot(&fun)), + Some(DumpHIR::Debug) => println!("Initial HIR:\n{:#?}", &fun), + None => {}, + } + + fun.profiles = Some(profiles); + if let Err(err) = crate::stats::trace_compile_phase("validate", || fun.validate()) { + debug!("ZJIT: {err:?}: Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)); + return Err(ParseError::Validation(err)); + } + Ok(fun) +} + +/// Compile an entry_block for the interpreter +fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_block: &HashMap<u32, BlockId>) { + let mut entry_block = fun.entry_block; + let (self_param, entry_state) = compile_entry_state(fun); + let mut pc: Option<InsnId> = None; + let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + + // Check-and-jump for each missing optional PC + let mut iter = jit_entry_insns.iter().peekable(); + while let Some(&jit_entry_insn) = iter.next() { + if jit_entry_insn == all_opts_passed_insn_idx { + continue; + } + let target_block = insn_idx_to_block.get(&jit_entry_insn) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + // Load PC once at the start of the block, shared among all cases + let pc = *pc.get_or_insert_with(|| fun.push_insn(entry_block, Insn::LoadPC)); + let expected_pc = fun.push_insn(entry_block, Insn::Const { + val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), + }); + let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); + + let next_insn_idx = **iter.peek().expect("last entry is skipped so there is always a next"); + let fall_through = fun.new_block(next_insn_idx); + + fun.push_insn(entry_block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + entry_block = fall_through; + } + + // Terminate the block with a jump to the block with all optionals passed + let target_block = insn_idx_to_block.get(&all_opts_passed_insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile initial locals for an entry_block for the interpreter +fn compile_entry_state(fun: &mut Function) -> (InsnId, FrameState) { + let entry_block = fun.entry_block; + fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); + + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + let self_param = fun.push_insn(entry_block, Insn::LoadSelf); + let mut entry_state = FrameState::new(iseq); + // If the ISEQ does not escape EP, we can assume EP + 1 == SP + // TODO: This should maybe also consider if the EP has historically been escaped in this iseq. + // (see: https://github.com/Shopify/ruby/issues/774) + let use_sp = !iseq_escapes_ep(iseq); + let mut base: Option<InsnId> = None; + for local_idx in 0..num_locals(iseq) { + if local_idx < param_size { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let return_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact + } else { + types::BasicObject + }; + let recv = *base.get_or_insert_with(|| { + let base_insn = if use_sp { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(entry_block, base_insn) + }); + let val = if use_sp { + fun.get_local_from_sp(entry_block, recv, ep_offset_u32, return_type) + } else { + fun.get_local_from_ep(entry_block, recv, ep_offset_u32, 0, return_type) + }; + entry_state.locals.push(val); + } else { + entry_state.locals.push(fun.push_insn(entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +/// Compile a jit_entry_block +fn compile_jit_entry_block(fun: &mut Function, jit_entry_idx: usize, target_block: BlockId) { + let jit_entry_block = fun.jit_entry_blocks[jit_entry_idx]; + fun.push_insn(jit_entry_block, Insn::EntryPoint { jit_entry_idx: Some(jit_entry_idx) }); + + // Prepare entry_state with basic block params + let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block, jit_entry_idx); + + if get_option!(stats) { + fun.count_iseq_calls(jit_entry_block); + } + // Jump to target_block + fun.push_insn(jit_entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile params and initial locals for a jit_entry_block +fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_entry_idx: usize) -> (InsnId, FrameState) { + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let opt_num: usize = params.opt_num.try_into().expect("iseq param opt_num >= 0"); + let lead_num: usize = params.lead_num.try_into().expect("iseq param lead_num >= 0"); + let passed_opt_num = jit_entry_idx; + + // If the iseq has keyword parameters, the keyword bits local will be appended to the local table. + let kw_bits_idx: Option<usize> = if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if !keyword.is_null() { + Some(unsafe { (*keyword).bits_start } as usize) + } else { + None + } + } else { + None + }; + + let mut arg_idx: u32 = 0; + let self_param = fun.push_insn(jit_entry_block, Insn::LoadArg { idx: arg_idx, id: FieldName::SelfParam, val_type: types::BasicObject }); + arg_idx += 1; + let mut entry_state = FrameState::new(iseq); + let mut ep: Option<InsnId> = None; + for local_idx in 0..num_locals(iseq) { + if (lead_num + passed_opt_num..lead_num + opt_num).contains(&local_idx) { + // Omitted optionals are locals, so they start as nils before their code run + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } else if Some(local_idx) == kw_bits_idx { + // Read the kw_bits value written by the caller to the callee frame. + // This tells us which optional keywords were NOT provided and need their defaults evaluated. + // Note: The caller writes kw_bits to memory via gen_send_iseq_direct but does NOT pass it + // as a C argument, so we must read it from EP memory rather than Param. + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let ep = *ep.get_or_insert_with(|| fun.push_insn(jit_entry_block, Insn::GetEP { level: 0 })); + entry_state.locals.push(fun.get_local_from_ep( + jit_entry_block, + ep, + ep_offset_u32, + 0, + types::BasicObject, + )); + } else if local_idx < param_size { + let id = unsafe { rb_zjit_local_id(iseq, local_idx.try_into().unwrap()) }; + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::LoadArg { idx: arg_idx, id: id.into(), val_type: types::BasicObject })); + arg_idx += 1; + } else { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +pub struct Dominators { + /// Immediate dominator for each block, indexed by BlockId. + /// idom(root) = root (self-loop is sentinel), idom[unreachable] == IDOM_NONE. + idoms: Vec<BlockId>, +} + +/// Sentinel value for "no idom computed yet". +const IDOM_NONE: BlockId = BlockId(usize::MAX); + +impl Dominators { + pub fn new(f: &Function) -> Self { + let mut cfi = ControlFlowInfo::new(f); + Self::with_cfi(f, &mut cfi) + } + + /// Compute immediate dominators using the "engineered algorithm" from + /// Cooper, Harvey & Kennedy, "A Simple, Fast Dominance Algorithm" (2001), + /// Figure 3: <https://www.cs.tufts.edu/~nr/cs257/archive/keith-cooper/dom14.pdf> + pub fn with_cfi(f: &Function, cfi: &mut ControlFlowInfo) -> Self { + let rpo = f.reverse_post_order(); + let num_blocks = f.blocks.len(); + + // Map BlockId -> RPO index for O(1) lookup in intersect. + let mut rpo_order = vec![usize::MAX; num_blocks]; + for (idx, &block) in rpo.iter().enumerate() { + rpo_order[block.0] = idx; + } + + // Initialize idom: root's idom is itself, everything else is undefined. + let mut idoms = vec![IDOM_NONE; num_blocks]; + let root = f.entries_block; + idoms[root.0] = root; + + let mut changed = true; + while changed { + changed = false; + for &block in &rpo { + if block == root { continue; } + + // Find the first predecessor that already has an idom computed. + let preds: Vec<BlockId> = cfi.predecessors(block).collect(); + let mut new_idom = IDOM_NONE; + for &p in &preds { + if idoms[p.0] != IDOM_NONE { + new_idom = p; + break; + } + } + if new_idom == IDOM_NONE { continue; } + + // Intersect with remaining processed predecessors. + for &p in &preds { + if p == new_idom { continue; } + if idoms[p.0] != IDOM_NONE { + new_idom = Self::intersect(&idoms, &rpo_order, p, new_idom); + } + } + + if idoms[block.0] != new_idom { + idoms[block.0] = new_idom; + changed = true; + } + } + } + + Self { idoms } + } + + /// Walk up the dominator tree from two fingers until they meet. + /// Uses RPO indices: a node with a *lower* RPO index is *higher* in the tree. + fn intersect(idoms: &[BlockId], rpo_order: &[usize], mut b1: BlockId, mut b2: BlockId) -> BlockId { + while b1 != b2 { + while rpo_order[b1.0] > rpo_order[b2.0] { + b1 = idoms[b1.0]; + } + while rpo_order[b2.0] > rpo_order[b1.0] { + b2 = idoms[b2.0]; + } + } + b1 + } + + /// Return the immediate dominator of `block`. + pub fn idom(&self, block: BlockId) -> BlockId { + self.idoms[block.0] + } + + /// Return true if `left` is dominated by `right`. + pub fn is_dominated_by(&self, left: BlockId, right: BlockId) -> bool { + if self.idom(left) == IDOM_NONE { return false; } + let mut block = left; + loop { + if block == right { return true; } + if self.idom(block) == block { return false; } + block = self.idom(block); + } + } + + /// Compute the full dominator set for `block` by walking the idom chain to the root. + /// Returns dominators sorted by BlockId (ascending). Only used in tests; + /// production code should use `idom()` or `is_dominated_by()` instead. + pub fn dominators(&self, block: BlockId) -> Vec<BlockId> { + let mut doms = Vec::new(); + if self.idom(block) != IDOM_NONE { + let mut b = block; + loop { + doms.push(b); + if self.idom(b) == b { break; } + b = self.idom(b); + } + } + doms.sort(); + doms + } +} + +pub struct ControlFlowInfo<'a> { + function: &'a Function, + successor_map: HashMap<BlockId, Vec<BlockId>>, + predecessor_map: HashMap<BlockId, Vec<BlockId>>, +} + +impl<'a> ControlFlowInfo<'a> { + pub fn new(function: &'a Function) -> Self { + let mut successor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let mut predecessor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + + for block_id in function.reverse_post_order() { + let mut successors = function.successors(block_id); + successors.dedup(); + + // Update predecessors for successor blocks. + for &succ_id in &successors { + predecessor_map + .entry(succ_id) + .or_default() + .push(block_id); + } + + // Store successors for this block. + successor_map.insert(block_id, successors); + } + + Self { + function, + successor_map, + predecessor_map, + } + } + + pub fn is_succeeded_by(&self, left: BlockId, right: BlockId) -> bool { + self.successor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn is_preceded_by(&self, left: BlockId, right: BlockId) -> bool { + self.predecessor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn predecessors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.predecessor_map.get(&block).into_iter().flatten().copied() + } + + pub fn successors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.successor_map.get(&block).into_iter().flatten().copied() + } +} + +pub struct LoopInfo<'a> { + cfi: &'a ControlFlowInfo<'a>, + dominators: &'a Dominators, + loop_depths: HashMap<BlockId, u32>, + loop_headers: BlockSet, + back_edge_sources: BlockSet, +} + +impl<'a> LoopInfo<'a> { + pub fn new(cfi: &'a ControlFlowInfo<'a>, dominators: &'a Dominators) -> Self { + let mut loop_headers: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let mut loop_depths: HashMap<BlockId, u32> = HashMap::new(); + let mut back_edge_sources: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let rpo = cfi.function.reverse_post_order(); + + for &block in &rpo { + loop_depths.insert(block, 0); + } + + // Collect loop headers. + for &block in &rpo { + // Initialize the loop depths. + for predecessor in cfi.predecessors(block) { + if dominators.is_dominated_by(predecessor, block) { + // Found a loop header, so then identify the natural loop. + loop_headers.insert(block); + back_edge_sources.insert(predecessor); + let loop_blocks = Self::find_natural_loop(cfi, block, predecessor); + // Increment the loop depth. + for loop_block in &loop_blocks { + *loop_depths.get_mut(loop_block).expect("Loop block should be populated.") += 1; + } + } + } + } + + Self { + cfi, + dominators, + loop_depths, + loop_headers, + back_edge_sources, + } + } + + fn find_natural_loop( + cfi: &ControlFlowInfo, + header: BlockId, + back_edge_source: BlockId, + ) -> HashSet<BlockId> { + // todo(aidenfoxivey): Reimplement using BlockSet + let mut loop_blocks = HashSet::new(); + let mut stack = vec![back_edge_source]; + + loop_blocks.insert(header); + loop_blocks.insert(back_edge_source); + + while let Some(block) = stack.pop() { + for pred in cfi.predecessors(block) { + // Pushes to stack only if `pred` wasn't already in `loop_blocks`. + if loop_blocks.insert(pred) { + stack.push(pred) + } + } + } + + loop_blocks + } + + pub fn loop_depth(&self, block: BlockId) -> u32 { + self.loop_depths.get(&block).copied().unwrap_or(0) + } + + pub fn is_back_edge_source(&self, block: BlockId) -> bool { + self.back_edge_sources.get(block) + } + + pub fn is_loop_header(&self, block: BlockId) -> bool { + self.loop_headers.get(block) + } +} + +#[cfg(test)] +mod union_find_tests { + use super::UnionFind; + + #[test] + fn test_find_returns_self() { + let mut uf = UnionFind::new(); + assert_eq!(uf.find(3usize), 3); + } + + #[test] + fn test_find_returns_target() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + assert_eq!(uf.find(3usize), 4); + } + + #[test] + fn test_find_halts_with_identity_make_equal_to() { + let mut uf = UnionFind::<usize>::new(); + uf.make_equal_to(0, 0); + assert_eq!(uf.find(0), 0); + } + + #[test] + fn test_find_returns_transitive_target() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + uf.make_equal_to(4, 5); + assert_eq!(uf.find(3usize), 5); + assert_eq!(uf.find(4usize), 5); + } + + #[test] + fn test_find_compresses_path() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + uf.make_equal_to(4, 5); + assert_eq!(uf.at(3usize), Some(4)); + assert_eq!(uf.find(3usize), 5); + assert_eq!(uf.at(3usize), Some(5)); + } +} + +#[cfg(test)] +mod rpo_tests { + use super::*; + + #[test] + fn one_block() { + let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; + let entry = function.entry_block; + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry]); + } + + #[test] + fn jump() { + let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; + let entry = function.entry_block; + let exit = function.new_block(0); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, exit]); + } + + #[test] + fn diamond_iftrue() { + let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: side, args: vec![] }, + if_false: BranchEdge { target: exit, args: vec![] } + }); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, side, exit]); + } + + #[test] + fn diamond_iffalse() { + let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, side, exit]); + } + + #[test] + fn a_loop() { + let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; + let entry = function.entry_block; + function.push_insn(entry, Insn::Jump(BranchEdge { target: entry, args: vec![] })); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry]); + } +} + +#[cfg(test)] +mod validation_tests { + use super::*; + + #[track_caller] + fn assert_matches_err(res: Result<(), ValidationError>, expected: ValidationError) { + match res { + Err(validation_err) => { + assert_eq!(validation_err, expected); + } + Ok(_) => panic!("Expected validation error"), + } + } + + #[test] + fn one_block_no_terminator() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::BlockHasNoTerminator(entry)); + } + + #[test] + fn one_block_terminator_not_at_end() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let insn_id = function.push_insn(entry, Insn::Return { val }); + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Unreachable); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::TerminatorNotAtEnd(entry, insn_id, 1)); + } + + #[test] + fn iftrue_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let fall_through = function.new_block(1); + function.push_insn(fall_through, Insn::Unreachable); + function.push_insn(side, Insn::Unreachable); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: side, args: vec![val, val, val] }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn iffalse_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let fall_through = function.new_block(1); + function.push_insn(fall_through, Insn::Unreachable); + function.push_insn(side, Insn::Unreachable); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![val, val, val] }, + }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn jump_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Jump ( BranchEdge { target: side, args: vec![val, val, val] } )); + function.push_insn(side, Insn::Unreachable); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn not_defined_within_bb() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + // Create an instruction without making it belong to anything. + let dangling = function.new_insn(Insn::Const{val: Const::CBool(true)}); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: dangling, state: InsnId(0usize) }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, dangling)); + } + + #[test] + fn using_non_output_insn() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let const_ = function.push_insn(function.entry_block, Insn::Const{val: Const::CBool(true)}); + // Ret is a non-output instruction. + let ret = function.push_insn(function.entry_block, Insn::Return { val: const_ }); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: ret, state: InsnId(0usize) }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, ret)); + } + + #[test] + fn not_dominated_by_diamond() { + // This tests that one branch is missing a definition which fails. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val1 = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::CondBranch { + val: val1, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let val2 = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + let const_ = function.push_insn(exit, Insn::Const{val: Const::CBool(true)}); + function.push_insn(exit, Insn::Return { val: const_ }); + + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(exit, val2, v0)); + }); + } + + #[test] + fn dominated_by_diamond() { + // This tests that both branches with a definition succeeds. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] } + }); + let _val = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + let const_ = function.push_insn(exit, Insn::Const{val: Const::CBool(true)}); + function.push_insn(exit, Insn::Return { val: const_ }); + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + // Just checking that we don't panic. + assert!(function.validate_definite_assignment().is_ok()); + }); + } + + #[test] + fn instruction_appears_twice_in_same_block() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn_id(block, val); + function.push_insn(block, Insn::Return { val }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val)); + } + + #[test] + fn instruction_appears_twice_with_different_ids() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val0 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let val1 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.make_equal_to(val1, val0); + function.push_insn(block, Insn::Return { val: val0 }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val0)); + } + + #[test] + fn instruction_appears_twice_in_different_blocks() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let exit = function.new_block(0); + function.push_insn(block, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn_id(exit, val); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(exit, val)); + } +} + +#[cfg(test)] +mod infer_tests { + use super::*; + + #[track_caller] + fn assert_subtype(left: Type, right: Type) { + assert!(left.is_subtype(right), "{left} is not a subtype of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Type, right: Type) { + assert!(left.bit_equal(right), "{left} != {right}"); + } + + #[test] + fn test_const() { + let mut function = Function::new(std::ptr::null()); + let val = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(function.entry_block, Insn::Unreachable); + assert_bit_equal(function.infer_type(val), types::NilClass); + } + + #[test] + fn test_nil() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let nil = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: nil }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(false)); + }); + } + + #[test] + fn test_false() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let false_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qfalse) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: false_ }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(false)); + }); + } + + #[test] + fn test_truthy() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let true_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qtrue) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: true_ }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(true)); + }); + } + + #[test] + fn newarray() { + let mut function = Function::new(std::ptr::null()); + // Fake FrameState index of 0usize + let val = function.push_insn(function.entry_block, Insn::NewArray { elements: vec![], state: InsnId(0usize) }); + assert_bit_equal(function.infer_type(val), types::ArrayExact); + } + + #[test] + fn arraydup() { + let mut function = Function::new(std::ptr::null()); + // Fake FrameState index of 0usize + let arr = function.push_insn(function.entry_block, Insn::NewArray { elements: vec![], state: InsnId(0usize) }); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: arr, state: InsnId(0usize) }); + assert_bit_equal(function.infer_type(val), types::ArrayExact); + } + + #[test] + fn diamond_iffalse_merge_fixnum() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(4)) }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![v1] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let param = function.push_insn(exit, Insn::Param); + function.push_insn(exit, Insn::Unreachable); + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + }); + assert_bit_equal(function.type_of(param), Type::fixnum(3)); + } + + #[test] + fn self_loop_param_rotation_reaches_full_union() { + // bb_entry: jump bb_loop(c1, c2, c3, c4) // 4 distinct types + // bb_loop(p1, p2, p3, p4): + // jump bb_loop(p2, p3, p4, p1) // 4-cycle rotation + // + // Every param transitively flows into every other across enough trips + // around the loop, so the fixpoint for every param is the full union + // of all four input types. The fixpoint loop must not exit while a + // branch arm is still widening a param's type. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let loop_block = function.new_block(0); + + let c1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qtrue) }); + let c2 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); + let c3 = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let c4 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(7)) }); + function.push_insn(entry, Insn::Jump(BranchEdge { + target: loop_block, + args: vec![c1, c2, c3, c4], + })); + + let p1 = function.push_insn(loop_block, Insn::Param); + let p2 = function.push_insn(loop_block, Insn::Param); + let p3 = function.push_insn(loop_block, Insn::Param); + let p4 = function.push_insn(loop_block, Insn::Param); + function.push_insn(loop_block, Insn::Jump(BranchEdge { + target: loop_block, + args: vec![p2, p3, p4, p1], + })); + + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + }); + + let full = types::TrueClass + .union(types::FalseClass) + .union(types::NilClass) + .union(types::Fixnum); + assert_bit_equal(function.type_of(p1), full); + assert_bit_equal(function.type_of(p2), full); + assert_bit_equal(function.type_of(p3), full); + assert_bit_equal(function.type_of(p4), full); + } + + #[test] + fn diamond_iffalse_merge_bool() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(Qtrue) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![v1] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let param = function.push_insn(exit, Insn::Param); + function.push_insn(exit, Insn::Unreachable); + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + assert_bit_equal(function.type_of(param), types::TrueClass); + }); + } +} diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs new file mode 100644 index 0000000000..74c0af710e --- /dev/null +++ b/zjit/src/hir/opt_tests.rs @@ -0,0 +1,16642 @@ +#[cfg(test)] +mod hir_opt_tests { + use crate::hir::*; + + use crate::{hir_strings, options::*}; + use insta::assert_snapshot; + use crate::hir::tests::hir_build_tests::assert_contains_opcode; + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[test] + fn test_fold_iftrue_away() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v25:Fixnum[3] = Const Value(3) + Return v25 + "); + } + + #[test] + fn test_fold_iftrue_into_jump() { + eval(" + def test + cond = false + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:FalseClass = Const Value(false) + CheckInterrupts + v35:Fixnum[4] = Const Value(4) + Return v35 + "); + } + + #[test] + fn test_fold_fixnum_add() { + eval(" + def test + 1 + 2 + 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v33:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_sub() { + eval(" + def test + 5 - 3 - 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v33:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_sub_large_negative_result() { + eval(" + def test + 0 - 1073741825 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[1073741825] = Const Value(1073741825) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v24:Fixnum[-1073741825] = Const Value(-1073741825) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult() { + eval(" + def test + 6 * 7 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[6] = Const Value(6) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v24:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult_zero() { + eval(" + def test(n) + 0 * n + n * 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, *@0x1010, cme:0x1018) + v36:Fixnum = GuardType v10, Fixnum + v46:Fixnum[0] = Const Value(0) + v47:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1040, cme:0x1048) + v48:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v48 + "); + } + + #[test] + fn test_fold_fixnum_div() { + eval(" + def test + 7 / 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_fold_fixnum_div_zero() { + eval(" + def test + 7 / 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v23:Fixnum = FixnumDiv v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_div_negative() { + eval(" + def test + 7 / -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v24:Fixnum[-3] = Const Value(-3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_fold_fixnum_div_negative_one_overflow() { + eval(&format!(" + def test + {RUBY_FIXNUM_MIN} / -1 + end + ")); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4611686018427387904] = Const Value(-4611686018427387904) + v12:Fixnum[-1] = Const Value(-1) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v23:Fixnum = FixnumDiv v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_mod_zero_by_zero() { + eval(" + def test + 0 % 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v23:Fixnum = FixnumMod v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_mod_non_zero_by_zero() { + eval(" + def test + 11 % 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[11] = Const Value(11) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v23:Fixnum = FixnumMod v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_mod_zero_by_non_zero() { + eval(" + def test + 0 % 11 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[11] = Const Value(11) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod() { + eval(" + def test + 11 % 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[11] = Const Value(11) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative_numerator() { + eval(" + def test + -7 % 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-7] = Const Value(-7) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative_denominator() { + eval(" + def test + 7 % -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[-2] = Const Value(-2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative() { + eval(" + def test + -7 % -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-7] = Const Value(-7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[-1] = Const Value(-1) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_xor() { + eval(" + def test + 2 ^ 5 + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v23:Fixnum[7] = Const Value(7) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_xor_same_negative_number() { + eval(" + def test + 123 ^ -123 + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + v12:Fixnum[-123] = Const Value(-123) + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v23:Fixnum[-2] = Const Value(-2) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_and() { + eval(" + def test + 4 & -7 + end + "); + + assert_snapshot!(inspect("test"), @"0"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[-7] = Const Value(-7) + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_and_with_negative_self() { + eval(" + def test + -4 & 7 + end + "); + + assert_snapshot!(inspect("test"), @"4"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4] = Const Value(-4) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v25:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or() { + eval(" + def test + 4 | 1 + end + "); + + assert_snapshot!(inspect("test"), @"5"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or_with_negative_self() { + eval(" + def test + -4 | 1 + end + "); + + assert_snapshot!(inspect("test"), @"-3"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4] = Const Value(-4) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[-3] = Const Value(-3) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or_with_negative_other() { + eval(" + def test + 4 | -1 + end + "); + + assert_snapshot!(inspect("test"), @"-1"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[-1] = Const Value(-1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[-1] = Const Value(-1) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_less() { + eval(" + def test + if 1 < 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_less_equal() { + eval(" + def test + if 1 <= 2 && 2 <= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v58:TrueClass = Const Value(true) + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_greater() { + eval(" + def test + if 2 > 1 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_greater_equal() { + eval(" + def test + if 2 >= 1 && 2 >= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v58:TrueClass = Const Value(true) + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_eq_false() { + eval(" + def test + if 1 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:FalseClass = Const Value(false) + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_eq_true() { + eval(" + def test + if 2 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_true() { + eval(" + def test + if 1 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_false() { + eval(" + def test + if 2 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:FalseClass = Const Value(false) + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + Return v33 + "); + } + + #[test] + fn test_fold_unbox_fixnum() { + eval(" + def test(arr) = arr[0] + test([1,2,3]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_guard_greater_eq() { + eval(" + def test(arr) = arr[0] + test([1,2,3]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_guard_greater_eq_side_exit() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-10] = Const CInt64(-10) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn neq_with_side_effect_not_elided () { + let result = eval(" + class CustomEq + attr_reader :count + + def ==(o) + @count = @count.to_i + 1 + self.equal?(o) + end + end + + def test(object) + # intentionally unused, but also can't assign to underscore + object != object + nil + end + + custom = CustomEq.new + test(custom) + test(custom) + + custom.count + "); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :object@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :object@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(CustomEq@0x1008) + PatchPoint MethodRedefined(CustomEq@0x1008, !=@0x1010, cme:0x1018) + v30:ObjectSubclass[class_exact:CustomEq] = GuardType v10, ObjectSubclass[class_exact:CustomEq] + v31:BoolExact = CCallWithFrame v30, :BasicObject#!=@0x1040, v30 + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_replace_guard_if_known_fixnum() { + eval(" + def test(a) + a + 1 + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:Fixnum = FixnumAdd v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_param_forms_get_bb_param() { + eval(" + def rest(*array) = array + def kw(k:) = k + def kw_rest(**k) = k + def post(*rest, post) = post + def block(&b) = nil + "); + assert_snapshot!(hir_strings!("rest", "kw", "kw_rest", "block", "post"), @" + fn rest@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :array@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :array@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + Return v10 + + fn kw@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k@0x1000 + v4:BasicObject = LoadField v2, :<empty>@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :k@1 + v9:CPtr = GetEP 0 + v10:BasicObject = LoadField v9, :<empty>@0x1002 + Jump bb3(v7, v8, v10) + bb3(v12:BasicObject, v13:BasicObject, v14:BasicObject): + CheckInterrupts + Return v13 + + fn kw_rest@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :k@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + Return v10 + + fn block@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :b@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :b@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + CheckInterrupts + Return v14 + + fn post@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :rest@0x1000 + v4:BasicObject = LoadField v2, :post@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :rest@1 + v9:BasicObject = LoadArg :post@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_optimize_top_level_call_into_send_direct() { + eval(" + def foo = [] + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_iseq() { + eval(" + def foo = 1 + alias bar foo + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_cfunc() { + eval(" + alias bar itself + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_inline_nonparam_local_return() { + // Methods that return non-parameter local variables should NOT be inlined, + // because the local variable index will be out of bounds for args. + // The method must have a parameter so param_size > 0, and return a local + // that's not a parameter so local_idx >= param_size. + // Use dead code (if false) to create a local without initialization instructions, + // resulting in just getlocal + leave which enters the inlining code path. + eval(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc() { + eval(" + class C < Array + alias fun_new_map map + end + def test(o) = o.fun_new_map {|e| e } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, fun_new_map@0x1010, cme:0x1018) + v27:ArraySubclass[class_exact:C] = GuardType v10, ArraySubclass[class_exact:C] + v28:BasicObject = SendDirect v27, 0x1040, :fun_new_map (0x1050) + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :o@0x1000 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc_from_module() { + eval(" + class C + include Enumerable + def each; yield 1; end + alias bar map + end + def test(o) = o.bar { |x| x } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, bar@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v29:BasicObject = CCallWithFrame v28, :Enumerable#bar@0x1040, block=0x1048 + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :o@0x1000 + CheckInterrupts + Return v29 + "); + } + + // Regression test: when specialized_instruction is disabled, the compiler + // doesn't convert `send` to `opt_send_without_block`, so a no-block call + // reaches ZJIT as `YARVINSN_send` with a null blockiseq. This becomes + // `Send { blockiseq: Some(null_ptr) }` which must be normalized to None in + // reduce_send_to_ccall, otherwise CCallWithFrame gens wrong block handler. + #[test] + fn test_send_to_cfunc_without_specialized_instruction() { + eval_with_options(" + def test(a) = a.length + test([1,2,3]); test([1,2,3]) + ", "{ specialized_instruction: false }"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v24:ArrayExact = GuardType v10, ArrayExact + v25:CInt64 = ArrayLength v24 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_nonexistent_top_level_call() { + eval(" + def foo + end + def test + foo + end + test; test + undef :foo + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :foo # SendFallbackReason: Send: unsupported method type Null + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_private_top_level_call() { + eval(" + def foo = [] + private :foo + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_top_level_call_with_overloaded_cme() { + eval(" + def test + Integer(3) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :Integer (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_top_level_call_with_args_into_send_direct() { + eval(" + def foo(a, b) = [] + def test + foo 1, 2 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_top_level_sends_into_send_direct() { + eval(" + def foo = [] + def bar = [] + def test + foo + bar + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v23:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v24:BasicObject = SendDirect v23, 0x1038, :foo (0x1048) + PatchPoint MethodRedefined(Object@0x1000, bar@0x1050, cme:0x1058) + v27:BasicObject = SendDirect v23, 0x1038, :bar (0x1048) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_direct_no_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_direct_one_optional_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_direct_all_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3, 4 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_call_with_correct_and_too_many_args_for_method() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4) = [a, b, c, d] + def test = [target(), target(10, 20, 30), begin; target(10, 20, 30, 40, 50) rescue ArgumentError; end] + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v44:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v45:BasicObject = SendDirect v44, 0x1038, :target (0x1048) + v14:Fixnum[10] = Const Value(10) + v16:Fixnum[20] = Const Value(20) + v18:Fixnum[30] = Const Value(30) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:BasicObject = SendDirect v44, 0x1038, :target (0x1048), v14, v16, v18 + v24:Fixnum[10] = Const Value(10) + v26:Fixnum[20] = Const Value(20) + v28:Fixnum[30] = Const Value(30) + v30:Fixnum[40] = Const Value(40) + v32:Fixnum[50] = Const Value(50) + v34:BasicObject = Send v44, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count + v37:ArrayExact = NewArray v45, v48, v34 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_optimize_variadic_ccall() { + eval(" + def test + puts 'Hello' + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + PatchPoint MethodRedefined(Object@0x1008, puts@0x1010, cme:0x1018) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] + v23:BasicObject = CCallVariadic v22, :Kernel#puts@0x1040, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_fixnum_add_if_redefined() { + eval(" + class Integer + def +(other) + 100 + end + end + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum + v29:Fixnum[100] = Const Value(100) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_both_profiled() { + eval(" + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAdd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_left_profiled() { + eval(" + def test(a) = a + 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:Fixnum = FixnumAdd v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_right_profiled() { + eval(" + def test(a) = 1 + a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v27:Fixnum = GuardType v10, Fixnum + v28:Fixnum = FixnumAdd v14, v27 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn integer_aref_with_fixnum_emits_fixnum_aref() { + eval(" + def test(a, b) = a[b] + test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAref v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn elide_fixnum_aref() { + eval(" + def test + 1[2] + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_too_many_args() { + eval(" + def test = 1[2, 3] + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v14:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v24:BasicObject = CCallVariadic v10, :Integer#[]@0x1038, v12, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_non_fixnum() { + eval(r#" + def test = 1["x"] + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v24:BasicObject = CCallVariadic v10, :Integer#[]@0x1040, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_both_profiled() { + eval(" + def test(a, b) = a < b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:Fixnum = GuardType v13, Fixnum + v30:BoolExact = FixnumLt v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_left_profiled() { + eval(" + def test(a) = a < 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:BoolExact = FixnumLt v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_right_profiled() { + eval(" + def test(a) = 1 < a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v27:Fixnum = GuardType v10, Fixnum + v28:BoolExact = FixnumLt v14, v27 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_literals() { + eval(" + def test() + a = 2 + (1..a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v26:RangeExact = NewRangeFixnum v17 NewRangeInclusive v13 + CheckInterrupts + Return v26 + "); + } + + + #[test] + fn test_optimize_new_range_fixnum_exclusive_literals() { + eval(" + def test() + a = 2 + (1...a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v26:RangeExact = NewRangeFixnum v17 NewRangeExclusive v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_high_guarded() { + eval(" + def test(a) + (1..a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v14 NewRangeInclusive v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_high_guarded() { + eval(" + def test(a) + (1...a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v14 NewRangeExclusive v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_low_guarded() { + eval(" + def test(a) + (a..10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v23 NewRangeInclusive v15 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_low_guarded() { + eval(" + def test(a) + (a...10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v23 NewRangeExclusive v15 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_new_array() { + eval(" + def test() + c = [] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_opt_aref_array() { + eval(" + arr = [1,2,3] + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + assert_snapshot!(inspect("test [1,2,3]"), @"1"); + } + + #[test] + fn test_opt_aref_hash() { + eval(" + arr = {0 => 4} + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v27:HashExact = GuardType v10, HashExact + v28:BasicObject = HashAref v27, v15 + CheckInterrupts + Return v28 + "); + assert_snapshot!(inspect("test({0 => 4})"), @"4"); + } + + #[test] + fn test_eliminate_new_range() { + eval(" + def test() + c = (1..2) + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:RangeExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_do_not_eliminate_new_range_non_fixnum() { + eval(" + def test() + _ = (-'a'..'b') + 0 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:StringExact = StringCopy v16 + v19:RangeExact = NewRange v14 NewRangeInclusive v17 + PatchPoint NoEPEscape(test) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_new_array_with_elements() { + eval(" + def test(a) + c = [a] + 5 + end + test(1); test(2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:ArrayExact = NewArray v12 + v22:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_eliminate_new_hash() { + eval(" + def test() + c = {} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_eliminate_new_hash_with_elements() { + eval(" + def test(aval, bval) + c = {a: aval, b: bval} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :aval@0x1000 + v4:BasicObject = LoadField v2, :bval@0x1001 + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :aval@1 + v10:BasicObject = LoadArg :bval@2 + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:NilClass): + v20:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + v23:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v26:HashExact = NewHash v20: v14, v23: v15 + PatchPoint NoEPEscape(test) + v32:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_eliminate_array_dup() { + eval(" + def test + c = [1, 2] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_hash_dup() { + eval(" + def test + c = {a: 1, b: 2} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_putself() { + eval(" + def test() + c = self + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_eliminate_string_copy() { + eval(r#" + def test() + c = "abc" + 5 + end + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_fixnum_add() { + eval(" + def test(a, b) + a + b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_sub() { + eval(" + def test(a, b) + a - b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, -@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_mul() { + eval(" + def test(a, b) + a * b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, *@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_div() { + eval(" + def test(a, b) + a / b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v34:Fixnum = FixnumDiv v32, v33 + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_mod() { + eval(" + def test(a, b) + a % b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, %@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v34:Fixnum = FixnumMod v32, v33 + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_lt() { + eval(" + def test(a, b) + a < b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_le() { + eval(" + def test(a, b) + a <= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_gt() { + eval(" + def test(a, b) + a > b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_ge() { + eval(" + def test(a, b) + a >= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_eq() { + eval(" + def test(a, b) + a == b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ==@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_neq() { + eval(" + def test(a, b) + a != b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, !=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v34:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_get_constant_path() { + eval(" + def test() + C + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v14:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_do_not_eliminate_getconstant() { + eval(" + def test(klass) + klass::ARGV + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :klass@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :klass@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:FalseClass = Const Value(false) + v17:BasicObject = GetConstant v10, :ARGV, v15 + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn kernel_itself_const() { + eval(" + def test(x) = x.itself + test(0) # profile + test(1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, itself@0x1010, cme:0x1018) + v23:Fixnum = GuardType v10, Fixnum + CheckInterrupts + Return v23 + "); + } + + #[test] + fn kernel_itself_known_type() { + eval(" + def test = [].itself + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn eliminate_kernel_itself() { + eval(" + def test + x = [].itself + 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_module_name() { + eval(" + module M; end + def test + x = M.name + 1 + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v29:ModuleExact[M@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020) + v34:StringExact|NilClass = CCall v29, :Module#name@0x1048 + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_array_length() { + eval(" + def test + [].length + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn normal_class_type_inference() { + eval(" + class C; end + def test = C + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v18:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn core_classes_type_inference() { + eval(" + def test = [String, Class, Module, BasicObject] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v26:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Class) + v29:ClassSubclass[Class@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint StableConstantNames(0x1020, Module) + v32:ClassSubclass[Module@0x1028] = Const Value(VALUE(0x1028)) + PatchPoint StableConstantNames(0x1030, BasicObject) + v35:ClassSubclass[BasicObject@0x1038] = Const Value(VALUE(0x1038)) + v18:ArrayExact = NewArray v26, v29, v32, v35 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn module_instances_are_module_exact() { + eval(" + def test = [Enumerable, Kernel] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Enumerable) + v22:ModuleExact[Enumerable@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Kernel) + v25:ModuleSubclass[Kernel@0x1018] = Const Value(VALUE(0x1018)) + v14:ArrayExact = NewArray v22, v25 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn module_subclasses_are_not_module_exact() { + eval(" + class ModuleSubclass < Module; end + MY_MODULE = ModuleSubclass.new + def test = MY_MODULE + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_MODULE) + v18:ModuleSubclass[MY_MODULE@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn eliminate_array_size() { + eval(" + def test + [].size + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn kernel_itself_argc_mismatch() { + eval(" + def test = 1.itself(0) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[0] = Const Value(0) + v14:BasicObject = Send v10, :itself, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_kernel_block_given_p() { + eval(" + def test = block_given? + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:CPtr = GetEP 0 + v21:BoolExact = IsBlockGiven v20 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_kernel_block_given_p_in_block() { + eval(" + TEST = proc { block_given? } + TEST.call + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:FalseClass = Const Value(false) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_elide_kernel_block_given_p() { + eval(" + def test + block_given? + 5 + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v23:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn const_send_direct_integer() { + eval(" + def test(x) = 1.zero? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, zero?@0x1010, cme:0x1018) + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn class_known_send_direct_array() { + eval(" + def test(x) + a = [1,2,3] + a.first + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:ArrayExact = ArrayDup v17 + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, first@0x1018, cme:0x1020) + v32:BasicObject = InvokeBuiltin leaf <inline_expr>, v18 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn send_direct_to_module() { + eval(" + module M; end + def test = M.class + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v20:ModuleExact[M@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020) + v26:ClassSubclass[Module@0x1010] = Const Value(VALUE(0x1010)) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_to_instance_method() { + eval(" + class C + def foo = [] + end + + def test(c) = c.foo + c = C.new + test c + test c + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :c@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :c@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v24:BasicObject = SendDirect v23, 0x1040, :foo (0x1050) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_send_direct_iseq_with_block() { + let result = eval(" + def foo(a, b, &block) = block.call(a, b) + def test = foo(1, 2) { |a, b| a + b } + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(3), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn reload_local_across_send() { + eval(" + def foo(&block) = 1 + def test + a = 1 + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v34:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v36:Fixnum[1] = Const Value(1) + PatchPoint NoEPEscape(test) + v21:CPtr = LoadSP + v22:BasicObject = LoadField v21, :a@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn reload_local_across_send_after_ep_escape() { + eval(" + def foo(&block) = 1 + def test + a = 1 + lambda { a } + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + SetLocal :a, l0, EP@3, v13 + PatchPoint MethodRedefined(Object@0x1000, lambda@0x1008, cme:0x1010) + v45:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v46:BasicObject = CCallWithFrame v45, :Kernel#lambda@0x1038, block=0x1040 + v20:CPtr = GetEP 0 + v21:BasicObject = LoadField v20, :a@0x1048 + PatchPoint MethodRedefined(Object@0x1000, foo@0x1049, cme:0x1050) + v32:CPtr = GetEP 0 + v33:BasicObject = LoadField v32, :a@0x1048 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_rest() { + enable_zjit_stats(); + eval(" + def foo(*args) = 1 + def test = foo 1 + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:Fixnum[1] = Const Value(1) + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_rest + v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v17 + "); + } + + #[test] + fn specialize_call_to_post_param_iseq() { + eval(" + def foo(opt=80, post) = post + def test = foo(10) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn specialize_call_to_iseq_with_optional_between_required_params() { + let result = eval(" + def foo(lead, opt=80, post) = lead + opt + post + def test = foo(10, 20) + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(110), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + v13:Fixnum[20] = Const Value(20) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_to_iseq_with_multiple_required_kw() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_to_iseq_with_required_kw_reorder() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendDirect v25, 0x1038, :foo (0x1048), v13, v15, v11 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_to_iseq_with_positional_and_required_kw_reorder() { + eval(" + def foo(x, a:, b:) = [x, a, b] + def test = foo(0, b: 2, a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendDirect v25, 0x1038, :foo (0x1048), v11, v15, v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_with_positional_and_optional_kw() { + eval(" + def foo(x, a: 1) = [x, a] + def test = foo(0, a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_req_kw() { + eval(" + def foo(r, x = 2, a:, b:) = [x, a] + def test = [foo(1, a: 3, b: 4), foo(1, 2, b: 4, a: 3)] # with and without the optional, change kw order + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v15:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v38:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v11, v13, v15 + v20:Fixnum[1] = Const Value(1) + v22:Fixnum[2] = Const Value(2) + v24:Fixnum[4] = Const Value(4) + v26:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v42:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v20, v22, v26, v24 + v30:ArrayExact = NewArray v38, v42 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_kw_optional() { + eval(" + def foo(r, x = 2, a:, b: 4) = [r, x, a, b] + def test = [foo(1, a: 3), foo(1, 2, b: 40, a: 30)] # with and without the optionals + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v34:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v38:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v11, v13, v34 + v18:Fixnum[1] = Const Value(1) + v20:Fixnum[2] = Const Value(2) + v22:Fixnum[40] = Const Value(40) + v24:Fixnum[30] = Const Value(30) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v42:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v18, v20, v24, v22 + v28:ArrayExact = NewArray v38, v42 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_call_with_pos_optional_and_maybe_too_many_args() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v49:BasicObject = SendDirect v48, 0x1038, :target (0x1048), v11 + v16:Fixnum[10] = Const Value(10) + v18:Fixnum[20] = Const Value(20) + v20:Fixnum[30] = Const Value(30) + v22:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v52:BasicObject = SendDirect v48, 0x1038, :target (0x1048), v16, v18, v20, v22 + v27:Fixnum[10] = Const Value(10) + v29:Fixnum[20] = Const Value(20) + v31:Fixnum[30] = Const Value(30) + v33:Fixnum[40] = Const Value(40) + v35:Fixnum[50] = Const Value(50) + v37:Fixnum[60] = Const Value(60) + v39:BasicObject = Send v48, :target, v27, v29, v31, v33, v35, v37 # SendFallbackReason: Too many arguments for LIR + v41:ArrayExact = NewArray v49, v52, v39 + CheckInterrupts + Return v41 + "); + } + + #[test] + fn test_send_call_to_iseq_with_optional_kw() { + eval(" + def foo(a: 1) = a + def test = foo(a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_kwrest() { + enable_zjit_stats(); + eval(" + def foo(**args) = 1 + def test = foo(a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:Fixnum[1] = Const Value(1) + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_kwrest + v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_send_hash_to_kwarg_only_method() { + eval(r#" + def callee(a:) = a + def test = callee({a: 1}) + begin; test; rescue ArgumentError; end + begin; test; rescue ArgumentError; end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + v14:BasicObject = Send v6, :callee, v12 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_send_hash_to_optional_kwarg_only_method() { + eval(r#" + def callee(a: nil) = a + def test = callee({a: 1}) + begin; test; rescue ArgumentError; end + begin; test; rescue ArgumentError; end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + v14:BasicObject = Send v6, :callee, v12 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn specialize_call_to_iseq_with_optional_param_kw_using_default() { + eval(" + def foo(int: 1) = int + 1 + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v17 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_call_kwsplat() { + enable_zjit_stats(); + eval(" + def foo(a:) = a + def test = foo(**{a: 1}) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v15:HashExact = HashDup v14 + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_caller_kw_splat + v18:BasicObject = Send v7, :foo, v15 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_param_kwrest() { + enable_zjit_stats(); + eval(" + def foo(**kwargs) = kwargs.keys + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_kwrest + v14:BasicObject = Send v7, :foo # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn dont_optimize_ccall_with_kwarg() { + eval(" + def test = sprintf('%s', a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = Send v6, :sprintf, v12, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn dont_optimize_ccall_with_block_and_kwarg() { + eval(" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + test %(a\nb\nc) + test %() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact = NewArray + v22:TrueClass = Const Value(true) + v24:BasicObject = Send v12, 0x1008, :each_line, v22 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v27:CPtr = LoadSP + v28:BasicObject = LoadField v27, :s@0x1000 + v29:BasicObject = LoadField v27, :a@0x1030 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_empty_ic() { + eval(" + def test = Kernel + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_invalidated_ic() { + eval(" + def test = Kernel + test + Kernel = 5 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn replace_get_constant_path_with_const() { + eval(" + def test = Kernel + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v18:ModuleSubclass[Kernel@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn replace_nested_get_constant_path_with_const() { + eval(" + module Foo + module Bar + class C + end + end + end + def test = Foo::Bar::C + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo::Bar::C) + v18:ClassSubclass[Foo::Bar::C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_no_initialize() { + eval(" + class C; end + def test = C.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v43:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v46:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_initialize() { + eval(" + class C + def initialize x + @x = x + end + end + def test = C.new 1 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v46:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v49:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v53:BasicObject = SendDirect v49, 0x1068, :initialize (0x1078), v15 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_object() { + eval(" + def test = Object.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Object) + v43:ClassSubclass[Object@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Object@0x1008, new@0x1009, cme:0x1010) + v46:ObjectExact = ObjectAllocClass Object:VALUE(0x1008) + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_basic_object() { + eval(" + def test = BasicObject.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, BasicObject) + v43:ClassSubclass[BasicObject@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(BasicObject@0x1008, new@0x1009, cme:0x1010) + v46:BasicObjectExact = ObjectAllocClass BasicObject:VALUE(0x1008) + PatchPoint NoSingletonClass(BasicObject@0x1008) + PatchPoint MethodRedefined(BasicObject@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_hash() { + eval(" + def test = Hash.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Hash) + v43:ClassSubclass[Hash@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Hash@0x1008, new@0x1009, cme:0x1010) + v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) + v47:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, initialize@0x1038, cme:0x1040) + v52:BasicObject = SendDirect v46, 0x1068, :initialize (0x1078), v47 + CheckInterrupts + Return v46 + "); + assert_snapshot!(inspect("test"), @"{}"); + } + + #[test] + fn test_opt_new_array() { + eval(" + def test = Array.new 1 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Array) + v46:ClassSubclass[Array@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Array@0x1008, new@0x1009, cme:0x1010) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v53:BasicObject = CCallVariadic v46, :Array.new@0x1040, v15 + CheckInterrupts + Return v53 + "); + } + + #[test] + fn test_opt_new_set() { + eval(" + def test = Set.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Set) + v43:ClassSubclass[Set@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Set@0x1008, new@0x1009, cme:0x1010) + v17:HeapBasicObject = ObjectAlloc v43 + PatchPoint NoSingletonClass(Set@0x1008) + PatchPoint MethodRedefined(Set@0x1008, initialize@0x1038, cme:0x1040) + v49:SetExact = GuardType v17, SetExact + v50:BasicObject = CCallVariadic v49, :Set#initialize@0x1068 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_string() { + eval(" + def test = String.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v43:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(String@0x1008, new@0x1009, cme:0x1010) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v54:BasicObject = CCallVariadic v43, :String.new@0x1040 + CheckInterrupts + Return v54 + "); + } + + #[test] + fn test_opt_new_regexp() { + eval(" + def test = Regexp.new '' + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Regexp) + v47:ClassSubclass[Regexp@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v16:StringExact = StringCopy v15 + PatchPoint MethodRedefined(Regexp@0x1008, new@0x1018, cme:0x1020) + v50:RegexpExact = ObjectAllocClass Regexp:VALUE(0x1008) + PatchPoint NoSingletonClass(Regexp@0x1008) + PatchPoint MethodRedefined(Regexp@0x1008, initialize@0x1048, cme:0x1050) + v55:BasicObject = CCallVariadic v50, :Regexp#initialize@0x1078, v16 + CheckInterrupts + Return v50 + "); + } + + #[test] + fn test_inline_class_allocate() { + eval(" + class C; end + def test = C.allocate + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v20:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v24:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_args() { + eval(" + class C; end + def test = C.allocate(1) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v22:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = Send v22, :allocate, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_singleton_class() { + eval(" + class C; end + SC = C.singleton_class + def test = SC.allocate + test rescue 0 + "); + // Not specialized: singleton classes are not leaf allocators + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, SC) + v20:ClassSubclass[Class@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v24:BasicObject = CCallWithFrame v20, :Class.allocate@0x1048 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v31:CInt64 = ArrayLength v19 + v32:Fixnum = BoxFixnum v31 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, size@0x1010, cme:0x1018) + v31:CInt64 = ArrayLength v19 + v32:Fixnum = BoxFixnum v31 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:CPtr = GetEP 0 + v18:CUInt64 = LoadField v17, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v19:CBool = IsBlockParamModified v18 + CondBranch v19, bb4(), bb5() + bb4(): + v21:BasicObject = LoadField v17, :block@0x1002 + Jump bb6(v21, v21) + bb5(): + v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) + v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v25, v10) + bb6(v15:BasicObject, v16:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + tap(&block) + end + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :block@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22) + bb5(): + v24:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v24) + bb6(v17:BasicObject): + v32:CPtr = GetEP 0 + v33:CUInt64 = LoadField v32, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v34:CBool = IsBlockParamModified v33 + CondBranch v34, bb7(), bb8() + bb7(): + v36:BasicObject = LoadField v32, :block@0x1002 + Jump bb9(v36, v36) + bb8(): + v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) + v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v40, v17) + bb9(v30:BasicObject, v31:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + tap(&block) + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v14:CPtr = GetEP 1 + v15:CUInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CBool = IsBlockParamModified v15 + CondBranch v16, bb4(), bb5() + bb4(): + v18:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v18) + bb5(): + v20:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v20) + bb6(v13:BasicObject): + v27:CPtr = GetEP 1 + v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v29:CBool = IsBlockParamModified v28 + CondBranch v29, bb7(), bb8() + bb7(): + v31:BasicObject = LoadField v27, :block@0x1001 + Jump bb9(v31) + bb8(): + v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) + v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v35) + bb9(v26:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb9() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb9(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb8(), bb10() + bb8(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb6(v16:BasicObject, v17:BasicObject): + v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v38 + bb10(): + SideExit BlockParamProxyProfileNotCovered + "); + } + + #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CPtr = GetEP 0 + v16:CUInt64 = LoadField v15, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v17:CBool = IsBlockParamModified v16 + CondBranch v17, bb4(), bb5() + bb4(): + v19:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v19) + bb5(): + v21:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v21) + bb6(v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:CPtr = GetEP 1 + v12:CUInt64 = LoadField v11, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v13:CBool = IsBlockParamModified v12 + CondBranch v13, bb4(), bb5() + bb4(): + v15:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v15) + bb5(): + v17:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v17) + bb6(v10:BasicObject): + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setblockparam() { + eval(" + def test(&block) + block = nil + end + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + SetLocal :block, l0, EP@3, v14 + v18:CPtr = GetEP 0 + v19:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CInt64[512] = Const CInt64(512) + v21:CInt64 = IntOr v19, v20 + StoreField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001, v21 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = nil + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + SetLocal :block, l1, EP@3, v10 + v14:CPtr = GetEP 1 + v15:CInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CInt64[512] = Const CInt64(512) + v17:CInt64 = IntOr v15, v16 + StoreField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000, v17 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_true() { + eval(" + @foo = 4 + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) recompile + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_false() { + eval(" + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) recompile + v19:NilClass = Const Value(nil) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_proc_call() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.call(1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, call@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_proc_aref() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p[2] + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, []@0x1010, cme:0x1018) + v26:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] + v27:BasicObject = InvokeProc v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_specialize_proc_yield() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.yield(3) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, yield@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_proc_eqq() { + eval(" + p = proc { |x| x > 0 } + def test(p) + p === 1 + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, ===@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_specialize_proc_call_splat() { + eval(" + p = proc { } + def test(p) + empty = [] + p.call(*empty) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :p@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact = NewArray + v23:ArrayExact = ToArray v17 + v25:BasicObject = Send v12, :call, v23 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_specialize_proc_call_kwarg() { + eval(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:BasicObject = Send v10, :call, v15 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_dont_specialize_definedivar_with_t_data() { + eval(" + class C < Range + def test = defined?(@a) + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_definedivar() { + set_call_threshold(3); + eval(" + class C + def test = defined?(@a) + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_definedivar() { + eval(r#" + class C + def test = defined?(@a) + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_already_in_shape() { + eval(" + @foo = 4 + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v21:HeapBasicObject = GuardType v6, HeapBasicObject + v22:CShape = LoadField v21, :shape_id@0x1000 + v23:CShape[0x1001] = GuardBitEquals v22, CShape(0x1001) recompile + StoreField v21, :@foo@0x1002, v10 + WriteBarrier v21, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_with_shape_transition() { + eval(" + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v21:HeapBasicObject = GuardType v6, HeapBasicObject + v22:CShape = LoadField v21, :shape_id@0x1000 + v23:CShape[0x1001] = GuardBitEquals v22, CShape(0x1001) recompile + StoreField v21, :@foo@0x1002, v10 + WriteBarrier v21, v10 + v26:CShape[0x1003] = Const CShape(0x1003) + StoreField v21, :shape_id@0x1000, v26 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_multiple_monomorphic_setivar_with_shape_transition() { + eval(r#" + klass = Class.new do + def test + @foo = 1 + @bar = 2 + end + end + + # Grow class max_iv_count so fresh instances can keep both writes + # on the embedded fast path. + warm = klass.new + warm.instance_variable_set(:@warm1, 1) + warm.instance_variable_set(:@warm2, 2) + + obj = klass.new + obj.test + TEST = klass.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v28:HeapBasicObject = GuardType v6, HeapBasicObject + v29:CShape = LoadField v28, :shape_id@0x1000 + v30:CShape[0x1001] = GuardBitEquals v29, CShape(0x1001) recompile + StoreField v28, :@foo@0x1002, v10 + WriteBarrier v28, v10 + v33:CShape[0x1003] = Const CShape(0x1003) + StoreField v28, :shape_id@0x1000, v33 + v14:HeapBasicObject = RefineType v28, HeapBasicObject + v17:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + StoreField v14, :@bar@0x1004, v17 + WriteBarrier v14, v17 + v40:CShape[0x1005] = Const CShape(0x1005) + StoreField v14, :shape_id@0x1000, v40 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_dont_specialize_setivar_with_t_data() { + eval(" + class C < Range + def test = @a = 5 + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_setivar() { + set_call_threshold(3); + eval(" + class C + def test = @a = 5 + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_setivar() { + eval(r#" + class C + def test = @a = 5 + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_setivar_when_next_shape_is_complex() { + eval(r#" + class AboutToBeTooComplex + def test = @abc = 5 + end + SHAPE_MAX_VARIATIONS = 8 # see shape.h + SHAPE_MAX_VARIATIONS.times do + AboutToBeTooComplex.new.instance_variable_set(:"@a#{_1}", 1) + end + AboutToBeTooComplex.new.test + TEST = AboutToBeTooComplex.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@abc, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_hash() { + eval(" + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_hash_freeze_if_redefined() { + eval(" + class Hash + def freeze; end + end + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_hash() { + eval(" + def test = {}.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_hash() { + eval(" + def test = {}.dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, dup@0x1008, cme:0x1010) + v23:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = Send v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_hash_with_args() { + eval(" + def test = {}.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + v12:NilClass = Const Value(nil) + v14:BasicObject = Send v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_ary() { + eval(" + def test = [].freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_ary() { + eval(" + def test = [].freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_ary() { + eval(" + def test = [].dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, dup@0x1008, cme:0x1010) + v23:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = Send v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_ary_with_args() { + eval(" + def test = [].freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v12:NilClass = Const Value(nil) + v14:BasicObject = Send v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_str() { + eval(" + def test = ''.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_str() { + eval(" + def test = ''.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_str() { + eval(" + def test = ''.dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v24:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = Send v24, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_elide_freeze_str_with_args() { + eval(" + def test = ''.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:NilClass = Const Value(nil) + v15:BasicObject = Send v11, :freeze, v13 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_elide_uminus_with_frozen_str() { + eval(" + def test = -'' + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_uminus_with_refrozen_str() { + eval(" + def test = -''.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_uminus_with_unfrozen_str() { + eval(" + def test = -''.dup + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v24:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = Send v24, :-@ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_objtostring_anytostring_string() { + eval(r##" + def test = "#{('foo')}" + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + v21:StringExact = StringConcat v10, v14 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_objtostring_anytostring_with_non_string() { + eval(r##" + def test = "#{1}" + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled() { + eval(" + def test(a) + \"#{a}\" + end + test('foo'); test('foo') + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + v29:String = GuardType v10, String + v22:StringExact = StringConcat v14, v29 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled_string_subclass() { + eval(" + class MyString < String; end + + def test(a) + \"#{a}\" + end + foo = MyString.new('foo') + test(MyString.new(foo)); test(MyString.new(foo)) + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(MyString@0x1010) + v29:String = GuardType v10, String + v22:StringExact = StringConcat v14, v29 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_objtostring_profiled_nonstring_falls_back_to_send() { + eval(" + def test(a) + \"#{a}\" + end + test([1,2,3]); test([1,2,3]) # No fast path for array + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v28:ArrayExact = GuardType v10, ArrayExact + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, to_s@0x1018, cme:0x1020) + v34:BasicObject = CCallWithFrame v28, :Array#to_s@0x1048 + v20:String = AnyToString v28, str: v34 + v22:StringExact = StringConcat v14, v20 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_branchnil_nil() { + eval(" + def test + x = nil + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + CheckInterrupts + v21:NilClass = Const Value(nil) + Return v21 + "); + } + + #[test] + fn test_branchnil_truthy() { + eval(" + def test + x = 1 + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + v23:Fixnum[1] = RefineType v13, NotNil + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + Return v23 + "); + } + + #[test] + fn test_dont_eliminate_load_from_non_frozen_array() { + eval(r##" + S = [4,5,6] + def test = S[0] + test + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v23 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v23, v30 + CheckInterrupts + Return v34 + "); + // TODO(max): Check the result of `S[0] = 5; test` using `inspect` to make sure that we + // actually do the load at run-time. + } + + #[test] + fn test_eliminate_load_from_frozen_array_in_bounds() { + eval(r##" + def test = [4,5,6].freeze[1] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v34:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative() { + eval(r##" + def test = [4,5,6].freeze[-3] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-3] = Const Value(-3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-3] = Const CInt64(-3) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-10] = Const CInt64(-10) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + SideExit GuardLess + "); + } + + #[test] + fn test_dont_optimize_array_aref_if_redefined() { + eval(r##" + class Array + def [](index) = [] + end + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:BasicObject = SendDirect v11, 0x1040, :[] (0x1050), v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_array_aset_if_redefined() { + eval(r##" + class Array + def []=(*args); :redefined; end + end + + def test(arr) + arr[1] = 10 + end + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[10] = Const Value(10) + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_dont_optimize_array_max_if_redefined() { + eval(r##" + class Array + def max = [] + end + def test = [4,5,6].max + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) + v21:BasicObject = SendDirect v11, 0x1040, :max (0x1050) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_set_type_from_constant() { + eval(" + MY_SET = Set.new + + def test = MY_SET + + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_SET) + v18:SetExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_regexp_type() { + eval(" + def test = /a/ + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_bmethod_send_direct() { + eval(" + define_method(:zero) { :b } + define_method(:one) { |arg| arg } + + def test = one(zero) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint MethodRedefined(Object@0x1000, zero@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v28:StaticSymbol[:b] = Const Value(VALUE(0x1038)) + PatchPoint MethodRedefined(Object@0x1000, one@0x1040, cme:0x1048) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_symbol_block_bmethod() { + eval(" + define_method(:identity, &:itself) + def test = identity(100) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[100] = Const Value(100) + v13:BasicObject = Send v6, :identity, v11 # SendFallbackReason: Bmethod: Proc object is not defined by an ISEQ + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_call_bmethod_with_block() { + eval(" + define_method(:bmethod) { :b } + def test = (bmethod {}) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :bmethod # SendFallbackReason: Send: unsupported method type Bmethod + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_call_shareable_bmethod() { + eval(" + class Foo + class << self + define_method(:identity, &(Ractor.make_shareable ->(val){val})) + end + end + def test = Foo.identity(100) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo) + v22:ClassSubclass[Foo@0x1008] = Const Value(VALUE(0x1008)) + v12:Fixnum[100] = Const Value(100) + PatchPoint MethodRedefined(Class@0x1010, identity@0x1018, cme:0x1020) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_nil_nil_specialized_to_ccall() { + eval(" + def test = nil.nil? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v21:TrueClass = Const Value(true) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_nil_nil_specialized_to_ccall() { + eval(" + def test + nil.nil? + 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v17:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_non_nil_nil_specialized_to_ccall() { + eval(" + def test = 1.nil? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v21:FalseClass = Const Value(false) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_non_nil_nil_specialized_to_ccall() { + eval(" + def test + 1.nil? + 2 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v17:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_guard_nil_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1008, nil?@0x1010, cme:0x1018) + v24:NilClass = GuardType v10, NilClass + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_false_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(false) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1008, nil?@0x1010, cme:0x1018) + v24:FalseClass = GuardType v10, FalseClass + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_true_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(true) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1008, nil?@0x1010, cme:0x1018) + v24:TrueClass = GuardType v10, TrueClass + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_symbol_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(:foo) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, nil?@0x1010, cme:0x1018) + v24:StaticSymbol = GuardType v10, StaticSymbol + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_fixnum_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, nil?@0x1010, cme:0x1018) + v24:Fixnum = GuardType v10, Fixnum + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_float_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1.0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, nil?@0x1010, cme:0x1018) + v24:Flonum = GuardType v10, Flonum + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_string_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test('foo') + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, nil?@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basicobject_not_truthy() { + eval(" + def test(a) = !a + + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, !@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basicobject_not_false() { + eval(" + def test(a) = !a + + test(false) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1008, !@0x1010, cme:0x1018) + v24:FalseClass = GuardType v10, FalseClass + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basicobject_not_nil() { + eval(" + def test(a) = !a + + test(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1008, !@0x1010, cme:0x1018) + v24:NilClass = GuardType v10, NilClass + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basicobject_not_falsy() { + eval(" + def test(a) = !(if a then false else nil end) + + # TODO(max): Make this not GuardType NilClass and instead just reason + # statically + test(false) + test(true) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb6(), bb4(v9, v17) + bb6(): + v19:Truthy = RefineType v10, Truthy + v21:FalseClass = Const Value(false) + CheckInterrupts + Jump bb5(v9, v19, v21) + bb4(v25:BasicObject, v26:Falsy): + v29:NilClass = Const Value(nil) + Jump bb5(v25, v26, v29) + bb5(v31:BasicObject, v32:BasicObject, v33:Falsy): + PatchPoint MethodRedefined(NilClass@0x1008, !@0x1010, cme:0x1018) + v45:NilClass = GuardType v33, NilClass + v46:TrueClass = Const Value(true) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_specialize_array_empty_p() { + eval(" + def test(a) = a.empty? + + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, empty?@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact + v26:CInt64 = ArrayLength v25 + v27:CInt64[0] = Const CInt64(0) + v28:CBool = IsBitEqual v26, v27 + v29:BoolExact = BoxBool v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_specialize_hash_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test({}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, empty?@0x1010, cme:0x1018) + v25:HashExact = GuardType v10, HashExact + v26:BoolExact = CCall v25, :Hash#empty?@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basic_object_eq_to_ccall() { + eval(" + class C; end + def test(a, b) = a == b + + test(C.new, C.new) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ==@0x1010, cme:0x1018) + v29:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] + v30:CBool = IsBitEqual v29, v13 + v31:BoolExact = BoxBool v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_guard_fixnum_and_fixnum() { + eval(" + def test(x, y) = x & y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, &@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAnd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_guard_fixnum_or_fixnum() { + eval(" + def test(x, y) = x | y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, |@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumOr v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_method_redefinition_patch_point_on_top_level_method() { + eval(" + def foo; end + def test = foo + + test; test + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_embedded() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:BasicObject = LoadField v23, :@foo@0x1042 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_getivar_complex() { + eval(r#" + class C + attr_reader :foo + def initialize + 1000.times do |i| + instance_variable_set("@v#{i}", i) + end + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v24:BasicObject = GetIvar v23, :@foo + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_getivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // GetIvar stays as a C call because iseq_to_hir handles polymorphic + // branching at parse time for getinstancevariable. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def foo = @foo + end + + c = C.new + c.foo # profile + c.foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.foo } # trigger shape guard exits -> recompile + 100.times { c.foo } # run recompiled version (version 2) + "); + // After recompilation, iseq_to_hir generates polymorphic branches at + // parse time using the exit-profiled shapes: two optimized LoadField + // fast paths plus a GetIvar C call fallback. + assert_snapshot!(hir_string_proc("C.new.method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:BasicObject = LoadField v11, :@foo@0x1004 + Jump bb4(v28) + bb8(): + v30:BasicObject = GetIvar v11, :@foo + Jump bb4(v30) + bb4(v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_definedivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // DefinedIvar stays as a C call fallback to avoid more shape guard exits. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def has_foo = defined?(@foo) + end + + c = C.new + c.has_foo # profile + c.has_foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.has_foo } # trigger shape guard exits -> recompile + 100.times { c.has_foo } # run recompiled version (version 2) + "); + assert_snapshot!(hir_string_proc("C.new.method(:has_foo)"), @" + fn has_foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // SetIvar stays as a C call fallback to avoid more shape guard exits. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def foo = @foo = 5 + end + + c = C.new + c.foo # profile + c.foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.foo } # trigger shape guard exits -> recompile + 100.times { c.foo } # run recompiled version (version 2) + "); + assert_snapshot!(hir_string_proc("C.new.method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setivar_shape_guard_attr_writer_no_recompile() { + // attr_writer SetIvar has no inline cache and may target a receiver + // operand other than CFP self, so don't recompile here yet. + eval(" + class C + attr_writer :foo + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + end + + class D + def write(obj) + obj.foo = 5 + end + end + + c = C.new + d = D.new + d.write(c) # profile + d.write(c) # compile (version 1 with shape guard) + e = C.new(true) # same class, different shape + 100.times { d.write(e) } # shape guard exits, but no recompile + "); + assert_snapshot!(hir_string_proc("D.new.method(:write)"), @" + fn write@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :obj@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_optimize_getivar_on_module_embedded() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:Module = GuardType v6, Module + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + PatchPoint RootBoxOnly + v21:RubyValue = LoadField v17, :fields_obj@0x1002 + v22:BasicObject = LoadField v21, :@foo@0x1003 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_getivar_on_module_complex() { + eval(r#" + module M + @foo = 42 + for i in 0...1000 + instance_variable_set("@v#{i}", i) + end + def self.test = @foo + end + M.test + "#); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_side_exit_assertion() { + eval(" + def side_exit = ::RubyVM::ZJIT.induce_side_exit! + side_exit + "); + std::panic::catch_unwind(|| assert_compiles("side_exit")).expect_err("Should panic because the program should side exit"); + } + + #[test] + fn test_optimize_getivar_on_class_embedded() { + eval(" + class C + @foo = 42 + def self.test = @foo + end + C.test + "); + assert_snapshot!(assert_compiles("C.test"), @"42"); + assert_snapshot!(hir_string_proc("C.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:Class = GuardType v6, Class + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + PatchPoint RootBoxOnly + v21:RubyValue = LoadField v17, :fields_obj@0x1002 + v22:BasicObject = LoadField v21, :@foo@0x1003 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_getivar_on_class_complex() { + eval(r#" + class C + @foo = 42 + for i in 0...1000 + instance_variable_set("@v#{i}", i) + end + def self.test = @foo + end + C.test + "#); + assert_snapshot!(hir_string_proc("C.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_on_t_struct() { + // Range is T_STRUCT (not T_DATA): falls back to CCall + eval(" + class C < Range + def test = @a + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:HeapBasicObject = GuardType v6, HeapBasicObject + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:CAttrIndex[0] = Const CAttrIndex(0) + v21:BasicObject = CCall v17, :rb_ivar_get_at_no_ractor_check@0x1008, v20 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data() { + // T_DATA uses fields_obj for instance variables. + eval(" + class C < Thread + def test = @a + end + obj = C.new { } + obj.join + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:TData = GuardType v6, TData + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:RubyValue = LoadField v17, :fields_obj@0x1002 + v21:BasicObject = LoadField v20, :@a@0x1002 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data_complex_fields() { + // T_DATA with enough ivars to force heap field storage + eval(" + class C < Thread + def test = @var1000 + end + obj = C.new { } + obj.join + 1000.times { |i| obj.instance_variable_set(:\"@var#{i}\", 1) } + obj.instance_variable_set(:@var1000, 42) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@var1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit UnhandledYARVInsn(getinstancevariable) + "); + } + + #[test] + fn test_optimize_attr_reader_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + class << self + attr_reader :foo + end + def self.test = foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :foo # SendFallbackReason: Single-ractor mode required + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_polymorphic() { + set_call_threshold(3); + eval(r#" + class C + def foo_then_many + @foo = 1 + 10.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 2 + end + + def many_then_foo + 10.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 3 + @foo = 4 + end + + def foo = @foo + 1 + end + + O1 = C.new + O1.foo_then_many + O2 = C.new + O2.many_then_foo + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:15: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:CPtr = LoadField v11, :as_heap@0x1004 + v29:BasicObject = LoadField v28, :@foo@0x1000 + Jump bb4(v29) + bb8(): + v31:BasicObject = GetIvar v11, :@foo + Jump bb4(v31) + bb4(v13:BasicObject): + v34:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v45:Fixnum = GuardType v13, Fixnum + v46:Fixnum = FixnumAdd v45, v34 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_optimize_getivar_skewed_polymorphic() { + // Use threshold=6 so we get 5 profile samples. + // 4 calls with shape A, 1 with shape B = 80% skew (>= 75% threshold). + set_call_threshold(6); + eval(r#" + class C + def foo_then_many + @foo = 1 + 100.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 2 + end + + def many_then_foo + 100.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 3 + @foo = 4 + end + + def foo = @foo + 1 + end + + O1 = C.new + O1.foo_then_many + O2 = C.new + O2.many_then_foo + O1.foo + O1.foo + O1.foo + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:15: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:CPtr = LoadField v11, :as_heap@0x1002 + v21:BasicObject = LoadField v20, :@foo@0x1000 + Jump bb4(v21) + bb6(): + v23:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v24:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v25 = RefineType v24, CUInt64 + v26:CInt64 = IntAnd v12, v23 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:BasicObject = LoadField v11, :@foo@0x1004 + Jump bb4(v29) + bb8(): + v44:CShape = LoadField v11, :shape_id@0x1005 + v45:CShape[0x1006] = GuardBitEquals v44, CShape(0x1006) recompile + v46:CPtr = LoadField v11, :as_heap@0x1002 + v47:BasicObject = LoadField v46, :@foo@0x1000 + Jump bb4(v47) + bb4(v13:BasicObject): + v34:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v50:Fixnum = GuardType v13, Fixnum + v51:Fixnum = FixnumAdd v50, v34 + CheckInterrupts + Return v51 + "); + } + + #[test] + fn test_optimize_getivar_polymorphic_with_subclass() { + set_call_threshold(3); + eval(r#" + class C + def initialize + @foo = 3 + end + + def foo = @foo + 1 + end + + class D < C + def initialize + super + @bar = 4 + end + end + + O1 = C.new + O2 = D.new + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v28) + bb8(): + v30:BasicObject = GetIvar v11, :@foo + Jump bb4(v30) + bb4(v13:BasicObject): + v33:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v44:Fixnum = GuardType v13, Fixnum + v45:Fixnum = FixnumAdd v44, v33 + CheckInterrupts + Return v45 + "); + } + + #[test] + fn test_getivar_polymorphic_t_class_and_t_data() { + set_call_threshold(3); + eval(r#" + module Reader + def test = @a + end + + class A + extend Reader + @a = 0 + end + + ARGF.instance_eval do + extend Reader + @a = :a + end + + A.test + ARGF.test + "#); + assert_snapshot!(assert_compiles("[A.test, ARGF.test]"), @"[0, :a]"); + assert_snapshot!(hir_string_proc("Reader.instance_method(:test)"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:RubyValue = LoadField v11, :fields_obj@0x1002 + v21:BasicObject = LoadField v20, :@a@0x1002 + Jump bb4(v21) + bb6(): + v23:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v24:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v25 = RefineType v24, CUInt64 + v26:CInt64 = IntAnd v12, v23 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + PatchPoint RootBoxOnly + v30:RubyValue = LoadField v11, :fields_obj@0x1004 + v31:BasicObject = LoadField v30, :@a@0x1002 + Jump bb4(v31) + bb8(): + v33:BasicObject = GetIvar v11, :@a + Jump bb4(v33) + bb4(v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_dont_optimize_attr_accessor_polymorphic() { + set_call_threshold(3); + eval(" + class C + attr_reader :foo, :bar + + def foo_then_bar + @foo = 1 + @bar = 2 + end + + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + + O1 = C.new + O1.foo_then_bar + O2 = C.new + O2.bar_then_foo + def test(o) = o.foo + test O1 + test O2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:20: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:ObjectSubclass[class_exact:C] = RefineType v18, ObjectSubclass[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v37:BasicObject = GetIvar v20, :@foo + Jump bb4(v16, v17, v37) + bb6(): + v24:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v24) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_optimize_getivar_with_complex_shape() { + eval(r#" + class C + attr_accessor :foo + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + def test(o) = o.foo + test obj + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v24:BasicObject = GetIvar v23, :@foo + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_send_with_block() { + eval(r#" + def test = [1, 2, 3].map { |x| x * 2 } + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) + v22:BasicObject = SendDirect v11, 0x1040, :map (0x1050) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_send_variadic_with_block() { + eval(r#" + A = [1, 2, 3] + B = ["a", "b", "c"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v38:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, B) + v41:ArrayExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(Array@0x1020) + PatchPoint MethodRedefined(Array@0x1020, zip@0x1028, cme:0x1030) + v46:BasicObject = CCallVariadic v38, :Array#zip@0x1058, v41 + PatchPoint NoEPEscape(test) + v24:CPtr = LoadSP + v25:BasicObject = LoadField v24, :result@0x1060 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_do_not_optimize_send_with_block_forwarding() { + eval(r#" + def test(&block) = [].map(&block) + test { |x| x }; test { |x| x } + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil() { + eval(r#" + def test(&block) = [].map(&block) + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[0] = GuardBitEquals v24, CInt64(0) + v26:NilClass = Const Value(nil) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil_nested() { + eval(r#" + def test(&block) + proc do + [].map(&block) + end + end + test; test + "#); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v13:CPtr = GetEP 1 + v14:CUInt64 = LoadField v13, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v15:CBool = IsBlockParamModified v14 + CondBranch v15, bb4(), bb5() + bb4(): + v17:BasicObject = LoadField v13, :block@0x1001 + Jump bb6(v17) + bb5(): + v19:CInt64 = LoadField v13, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v20:CInt64 = GuardAnyBitSet v19, CUInt64(1) + v21:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v21) + bb6(v12:BasicObject): + v24:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_send_direct_iseq_with_block_no_callee_block_param() { + let result = eval(r#" + def foo + yield 1 + end + + def test = foo { |x| x * 2 } + test; test + "#); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_iseq_with_block_param_no_block() { + let result = eval(" + def foo(&blk) + blk ? blk.call : 42 + end + def test = foo + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(42), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_bmethod_with_block_param_no_block() { + let result = eval(" + define_method(:foo) { |&blk| + blk ? blk.call : 42 + } + def test = foo + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(42), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:BasicObject = SendDirect v19, 0x1038, :foo (0x1048) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_attr_reader_constant() { + eval(" + class C + attr_reader :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor_constant() { + eval(" + class C + attr_accessor :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_reader() { + eval(" + class C + attr_reader :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor_set() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + v35:CShape[0x1043] = Const CShape(0x1043) + StoreField v28, :shape_id@0x1040, v35 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_inline_attr_writer_set() { + eval(" + class C + attr_writer :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + v35:CShape[0x1043] = Const CShape(0x1043) + StoreField v28, :shape_id@0x1040, v35 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_inline_struct_aref_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v24:BasicObject = LoadField v23, :foo@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_inline_struct_aref_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v24:CPtr = LoadField v23, :as_heap@0x1040 + v25:BasicObject = LoadField v24, :foo@0x1041 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_elide_struct_aref() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) + o.foo + 5 + end + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_struct_aset_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:BasicObject = LoadField v2, :v@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :o@1 + v9:BasicObject = LoadArg :v@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] + v32:CUInt64 = LoadField v31, :RBASIC_FLAGS@0x1040 + v33:CUInt64 = GuardNoBitsSet v32, RUBY_FL_FREEZE=CUInt64(2048) + StoreField v31, :foo=@0x1041, v13 + WriteBarrier v31, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_inline_struct_aset_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:BasicObject = LoadField v2, :v@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :o@1 + v9:BasicObject = LoadArg :v@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] + v32:CUInt64 = LoadField v31, :RBASIC_FLAGS@0x1040 + v33:CUInt64 = GuardNoBitsSet v32, RUBY_FL_FREEZE=CUInt64(2048) + v34:CPtr = LoadField v31, :as_heap@0x1041 + StoreField v34, :foo=@0x1042, v13 + WriteBarrier v31, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_array_reverse_returns_array() { + eval(r#" + def test = [].reverse + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v21:ArrayExact = CCallWithFrame v10, :Array#reverse@0x1038 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_array_reverse_is_elidable() { + eval(r#" + def test + [].reverse + 5 + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_array_join_returns_string() { + eval(r#" + def test = [].join "," + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, join@0x1010, cme:0x1018) + v24:StringExact = CCallVariadic v10, :Array#join@0x1040, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_string_to_s_returns_string() { + eval(r#" + def test = "".to_s + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_string_literal_to_s() { + eval(r#" + def test = "foo".to_s + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_profiled_string_to_s() { + eval(r#" + def test(o) = o.to_s + test "foo" + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fixnum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test 5 + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v23:Fixnum = GuardType v10, Fixnum + v24:StringExact = CCallVariadic v23, :Integer#to_s@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_bignum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test (2**65) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v23:Bignum = GuardType v10, Bignum + v24:StringExact = CCallVariadic v23, :Integer#to_s@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_any_to_string_with_known_string_exact() { + eval(r##" + def test(x) = "#{x}" + test 123 + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v28:Fixnum = GuardType v10, Fixnum + PatchPoint MethodRedefined(Integer@0x1010, to_s@0x1018, cme:0x1020) + v33:StringExact = CCallVariadic v28, :Integer#to_s@0x1048 + v22:StringExact = StringConcat v14, v33 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_array_aref_fixnum_literal() { + eval(" + def test + arr = [1, 2, 3] + arr[0] + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v19:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v38:CInt64[0] = Const CInt64(0) + v32:CInt64 = ArrayLength v14 + v33:CInt64[0] = GuardLess v38, v32 + v37:BasicObject = ArrayAref v14, v33 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_array_aref_fixnum_profiled() { + eval(" + def test(arr, idx) + arr[idx] + end + test([1, 2, 3], 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :arr@1 + v9:BasicObject = LoadArg :idx@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v29:ArrayExact = GuardType v12, ArrayExact + v30:Fixnum = GuardType v13, Fixnum + v31:CInt64 = UnboxFixnum v30 + v32:CInt64 = ArrayLength v29 + v33:CInt64 = GuardLess v31, v32 + v34:CInt64 = AdjustBounds v33, v32 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + v37:BasicObject = ArrayAref v29, v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_array_aref_fixnum_array_subclass() { + eval(" + class C < Array; end + def test(arr, idx) + arr[idx] + end + test(C.new([1, 2, 3]), 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :arr@1 + v9:BasicObject = LoadArg :idx@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []@0x1010, cme:0x1018) + v29:ArraySubclass[class_exact:C] = GuardType v12, ArraySubclass[class_exact:C] + v30:Fixnum = GuardType v13, Fixnum + v31:CInt64 = UnboxFixnum v30 + v32:CInt64 = ArrayLength v29 + v33:CInt64 = GuardLess v31, v32 + v34:CInt64 = AdjustBounds v33, v32 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + v37:BasicObject = ArrayAref v29, v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_hash_aref_literal() { + eval(" + def test + arr = {1 => 3} + arr[1] + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v19:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v31:BasicObject = HashAref v14, v19 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_hash_aref_profiled() { + eval(" + def test(hash, key) + hash[key] + end + test({1 => 3}, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :hash@1 + v9:BasicObject = LoadArg :key@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v29:HashExact = GuardType v12, HashExact + v30:BasicObject = HashAref v29, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_optimize_hash_aref_subclass() { + eval(" + class C < Hash; end + def test(hash, key) + hash[key] + end + test(C.new({0 => 3}), 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :hash@1 + v9:BasicObject = LoadArg :key@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []@0x1010, cme:0x1018) + v29:HashSubclass[class_exact:C] = GuardType v12, HashSubclass[class_exact:C] + v30:BasicObject = CCallWithFrame v29, :Hash#[]@0x1040, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_does_not_fold_hash_aref_with_frozen_hash() { + eval(" + H = {a: 0}.freeze + def test = H[:a] + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, H) + v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:StaticSymbol[:a] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(Hash@0x1018) + PatchPoint MethodRedefined(Hash@0x1018, []@0x1020, cme:0x1028) + v28:BasicObject = HashAref v23, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_hash_aset_literal() { + eval(" + def test + h = {} + h[1] = 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + v24:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + HashAset v13, v22, v24 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_hash_aset_profiled() { + eval(" + def test(hash, key, val) + hash[key] = val + end + test({}, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :hash@1 + v10:BasicObject = LoadArg :key@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []=@0x1010, cme:0x1018) + v37:HashExact = GuardType v14, HashExact + HashAset v37, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_no_optimize_hash_aset_subclass() { + eval(" + class C < Hash; end + def test(hash, key, val) + hash[key] = val + end + test(C.new, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :hash@1 + v10:BasicObject = LoadArg :key@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []=@0x1010, cme:0x1018) + v37:HashSubclass[class_exact:C] = GuardType v14, HashSubclass[class_exact:C] + v38:BasicObject = CCallWithFrame v37, :Hash#[]=@0x1040, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_thread_current() { + eval(" + def test = Thread.current + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Thread) + v20:ClassSubclass[Thread@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, current@0x1018, cme:0x1020) + v24:CPtr = LoadEC + v25:CPtr = LoadField v24, :thread_ptr@0x1048 + v26:BasicObject = LoadField v25, :self@0x1049 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_array_aset_literal() { + eval(" + def test(arr) + arr[1] = 10 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []=@0x1010, cme:0x1018) + v33:ArrayExact = GuardType v10, ArrayExact + v34:CUInt64 = LoadField v33, :RBASIC_FLAGS@0x1040 + v35:CUInt64 = GuardNoBitsSet v34, RUBY_FL_FREEZE=CUInt64(2048) + v37:CUInt64 = GuardNoBitsSet v35, RUBY_ELTS_SHARED=CUInt64(4096) + v46:CInt64[1] = Const CInt64(1) + v39:CInt64 = ArrayLength v33 + v40:CInt64[1] = GuardLess v46, v39 + ArrayAset v33, v40, v19 + WriteBarrier v33, v19 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_array_aset_profiled() { + eval(" + def test(arr, index, val) + arr[index] = val + end + test([], 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :index@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arr@1 + v10:BasicObject = LoadArg :index@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []=@0x1010, cme:0x1018) + v37:ArrayExact = GuardType v14, ArrayExact + v38:Fixnum = GuardType v15, Fixnum + v39:CUInt64 = LoadField v37, :RBASIC_FLAGS@0x1040 + v40:CUInt64 = GuardNoBitsSet v39, RUBY_FL_FREEZE=CUInt64(2048) + v42:CUInt64 = GuardNoBitsSet v40, RUBY_ELTS_SHARED=CUInt64(4096) + v43:CInt64 = UnboxFixnum v38 + v44:CInt64 = ArrayLength v37 + v45:CInt64 = GuardLess v43, v44 + v46:CInt64 = AdjustBounds v45, v44 + v47:CInt64[0] = Const CInt64(0) + v48:CInt64 = GuardGreaterEq v46, v47 + ArrayAset v37, v48, v16 + WriteBarrier v37, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_array_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, index, val) + arr[index] = val + end + a = MyArray.new + test(a, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :index@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arr@1 + v10:BasicObject = LoadArg :index@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(MyArray@0x1008) + PatchPoint MethodRedefined(MyArray@0x1008, []=@0x1010, cme:0x1018) + v37:ArraySubclass[class_exact:MyArray] = GuardType v14, ArraySubclass[class_exact:MyArray] + v38:BasicObject = CCallVariadic v37, :Array#[]=@0x1040, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_array_ltlt() { + eval(" + def test(arr) + arr << 1 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, <<@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact + v28:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1040 + v29:CUInt64 = GuardNoBitsSet v28, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v27, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_array_push_single_arg() { + eval(" + def test(arr) + arr.push(1) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, push@0x1010, cme:0x1018) + v26:ArrayExact = GuardType v10, ArrayExact + v27:CUInt64 = LoadField v26, :RBASIC_FLAGS@0x1040 + v28:CUInt64 = GuardNoBitsSet v27, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v26, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_do_not_optimize_array_push_multi_arg() { + eval(" + def test(arr) + arr.push(1,2,3) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:Fixnum[2] = Const Value(2) + v19:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, push@0x1010, cme:0x1018) + v30:ArrayExact = GuardType v10, ArrayExact + v31:BasicObject = CCallVariadic v30, :Array#push@0x1040, v15, v17, v19 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_array_push_with_array_subclass() { + eval(" + class PushSubArray < Array + def <<(val) = super + end + test = PushSubArray.new + test << 1 + "); + assert_snapshot!(hir_string_proc("PushSubArray.new.method(:<<)"), @" + fn <<@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, <<@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:Array = GuardType v9, Array + v29:CUInt64 = LoadField v28, :RBASIC_FLAGS@0x1051 + v30:CUInt64 = GuardNoBitsSet v29, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v28, v10 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_pop_with_array_subclass() { + eval(" + class PopSubArray < Array + def pop = super + end + test = PopSubArray.new([1]) + test.pop + "); + assert_snapshot!(hir_string_proc("PopSubArray.new.method(:pop)"), @" + fn pop@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Array@0x1000, pop@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:Array = GuardType v6, Array + v24:CUInt64 = LoadField v23, :RBASIC_FLAGS@0x1049 + v25:CUInt64 = GuardNoBitsSet v24, RUBY_FL_FREEZE=CUInt64(2048) + v27:CUInt64 = GuardNoBitsSet v25, RUBY_ELTS_SHARED=CUInt64(4096) + v28:BasicObject = ArrayPop v23 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_aref_with_array_subclass_and_fixnum() { + eval(" + class ArefSubArray < Array + def [](idx) = super + end + test = ArefSubArray.new([1]) + test[0] + "); + assert_snapshot!(hir_string_proc("ArefSubArray.new.method(:[])"), @" + fn []@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :idx@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :idx@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:Array = GuardType v9, Array + v29:Fixnum = GuardType v10, Fixnum + v30:CInt64 = UnboxFixnum v29 + v31:CInt64 = ArrayLength v28 + v32:CInt64 = GuardLess v30, v31 + v33:CInt64 = AdjustBounds v32, v31 + v34:CInt64[0] = Const CInt64(0) + v35:CInt64 = GuardGreaterEq v33, v34 + v36:BasicObject = ArrayAref v28, v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_dont_optimize_array_aref_with_array_subclass_and_non_fixnum() { + eval(" + class ArefSubArrayRange < Array + def [](idx) = super + end + test = ArefSubArrayRange.new([1, 2, 3]) + test[0..1] + "); + assert_snapshot!(hir_string_proc("ArefSubArrayRange.new.method(:[])"), @" + fn []@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :idx@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :idx@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:BasicObject = CCallVariadic v9, :Array#[]@0x1058, v10 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_length() { + eval(" + def test(arr) = arr.length + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact + v26:CInt64 = ArrayLength v25 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_array_size() { + eval(" + def test(arr) = arr.size + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, size@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact + v26:CInt64 = ArrayLength v25 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_regexpmatch2() { + eval(r#" + def test(s) = s =~ /a/ + test("foo") + "#); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, =~@0x1018, cme:0x1020) + v27:StringExact = GuardType v10, StringExact + v28:BasicObject = CCallWithFrame v27, :String#=~@0x1048, v15 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_getbyte_fixnum() { + eval(r#" + def test(s, i) = s.getbyte(i) + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :i@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :i@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, getbyte@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact + v29:Fixnum = GuardType v13, Fixnum + v30:CInt64 = UnboxFixnum v29 + v31:CInt64 = LoadField v28, :len@0x1040 + v32:CInt64 = GuardLess v30, v31 + v33:CInt64 = AdjustBounds v32, v31 + v34:CInt64[0] = Const CInt64(0) + v35:CInt64 = GuardGreaterEq v33, v34 + v36:Fixnum = StringGetbyte v28, v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_elide_string_getbyte_fixnum() { + eval(r#" + def test(s, i) + s.getbyte(i) + 5 + end + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :i@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :i@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, getbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v12, StringExact + v33:Fixnum = GuardType v13, Fixnum + v34:CInt64 = UnboxFixnum v33 + v35:CInt64 = LoadField v32, :len@0x1040 + v36:CInt64 = GuardLess v34, v35 + v37:CInt64 = AdjustBounds v36, v35 + v38:CInt64[0] = Const CInt64(0) + v39:CInt64 = GuardGreaterEq v37, v38 + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_string_setbyte_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 127) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v14, StringExact + v33:Fixnum = GuardType v15, Fixnum + v34:Fixnum = GuardType v16, Fixnum + v35:CInt64 = UnboxFixnum v33 + v36:CInt64 = LoadField v32, :len@0x1040 + v37:CInt64 = GuardLess v35, v36 + v38:CInt64 = AdjustBounds v37, v36 + v39:CInt64[0] = Const CInt64(0) + v40:CInt64 = GuardGreaterEq v38, v39 + v41:CUInt64 = LoadField v32, :RBASIC_FLAGS@0x1041 + v42:CUInt64 = GuardNoBitsSet v41, RUBY_FL_FREEZE=CUInt64(2048) + v43:Fixnum = StringSetbyteFixnum v32, v33, v34 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_optimize_string_subclass_setbyte_fixnum() { + eval(r#" + class MyString < String + end + def test(s, idx, val) + s.setbyte(idx, val) + end + test(MyString.new('foo'), 0, 127) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1008) + PatchPoint MethodRedefined(MyString@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringSubclass[class_exact:MyString] = GuardType v14, StringSubclass[class_exact:MyString] + v33:Fixnum = GuardType v15, Fixnum + v34:Fixnum = GuardType v16, Fixnum + v35:CInt64 = UnboxFixnum v33 + v36:CInt64 = LoadField v32, :len@0x1040 + v37:CInt64 = GuardLess v35, v36 + v38:CInt64 = AdjustBounds v37, v36 + v39:CInt64[0] = Const CInt64(0) + v40:CInt64 = GuardGreaterEq v38, v39 + v41:CUInt64 = LoadField v32, :RBASIC_FLAGS@0x1041 + v42:CUInt64 = GuardNoBitsSet v41, RUBY_FL_FREEZE=CUInt64(2048) + v43:Fixnum = StringSetbyteFixnum v32, v33, v34 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_do_not_optimize_string_setbyte_non_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 3.14) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v14, StringExact + v33:BasicObject = CCallWithFrame v32, :String#setbyte@0x1040, v15, v16 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_specialize_string_empty() { + eval(r#" + def test(s) + s.empty? + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, empty?@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact + v26:CInt64 = LoadField v25, :len@0x1040 + v27:CInt64[0] = Const CInt64(0) + v28:CBool = IsBitEqual v26, v27 + v29:BoolExact = BoxBool v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_eliminate_string_empty() { + eval(r#" + def test(s) + s.empty? + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, empty?@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact + v20:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_integer_succ_with_fixnum() { + eval(" + def test(x) = x.succ + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, succ@0x1010, cme:0x1018) + v24:Fixnum = GuardType v10, Fixnum + v25:Fixnum[1] = Const Value(1) + v26:Fixnum = FixnumAdd v24, v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_bignum() { + eval(" + def test(x) = x.succ + test(4 << 70) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, succ@0x1010, cme:0x1018) + v24:Bignum = GuardType v10, Bignum + v25:BasicObject = CCallWithFrame v24, :Integer#succ@0x1040 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_inline_integer_ltlt_with_known_fixnum() { + eval(" + def test(x) = x << 5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:Fixnum = FixnumLShift v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_negative() { + eval(" + def test(x) = x << -5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1040, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_out_of_range() { + eval(" + def test(x) = x << 64 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1040, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_unknown_fixnum() { + eval(" + def test(x, y) = x << y + test(4, 5) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum + v29:BasicObject = CCallWithFrame v28, :Integer#<<@0x1040, v13 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_inline_integer_gtgt_with_known_fixnum() { + eval(" + def test(x) = x >> 5 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum + v26:Fixnum = FixnumRShift v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_negative() { + eval(" + def test(x) = x >> -5 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1040, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_out_of_range() { + eval(" + def test(x) = x >> 64 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1040, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_unknown_fixnum() { + eval(" + def test(x, y) = x >> y + test(4, 5) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum + v28:BasicObject = CCallWithFrame v27, :Integer#>>@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_append() { + eval(r#" + def test(x, y) = x << y + test("iron", "fish") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:String = GuardType v13, String + v31:StringExact = StringAppend v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_append_codepoint() { + eval(r#" + def test(x, y) = x << y + test("iron", 4) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:Fixnum = GuardType v13, Fixnum + v31:StringExact = StringAppendCodepoint v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_append_string_subclass() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test("iron", MyString.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:String = GuardType v13, String + v31:StringExact = StringAppend v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_do_not_optimize_string_subclass_append_string() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test(MyString.new, "iron") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1008) + PatchPoint MethodRedefined(MyString@0x1008, <<@0x1010, cme:0x1018) + v29:StringSubclass[class_exact:MyString] = GuardType v12, StringSubclass[class_exact:MyString] + v30:BasicObject = CCallWithFrame v29, :String#<<@0x1040, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_optimize_string_append_non_string() { + eval(r#" + def test = "iron" << :a + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, <<@0x1018, cme:0x1020) + v25:BasicObject = CCallWithFrame v11, :String#<<@0x1048, v13 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_many_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo(3, 3, 3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[3] = Const Value(3) + v14:Fixnum[3] = Const Value(3) + v16:Fixnum[3] = Const Value(3) + v18:BasicObject = Send v10, :foo, v12, v14, v16 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_string_ascii_only_p() { + eval(r#" + def test(x) = x.ascii_only? + test("iron") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ascii_only?@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact + v25:BoolExact = CCall v24, :String#ascii_only?@0x1040 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_few_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:BasicObject = Send v10, :foo # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_args() { + eval(" + def test = 4.succ 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = Send v10, :succ, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_integer_xor_with_fixnum() { + eval(" + def test(x, y) = x ^ y + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = GuardType v13, Fixnum + v29:Fixnum = FixnumXor v27, v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_eliminate_integer_xor() { + eval(r#" + def test(x, y) + x ^ y + 42 + end + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = GuardType v13, Fixnum + v23:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_lhs() { + eval(" + def test(x, y) = x ^ y + test(4 << 70, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Bignum = GuardType v12, Bignum + v28:BasicObject = CCallWithFrame v27, :Integer#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_rhs() { + eval(" + def test(x, y) = x ^ y + test(1, 4 << 70) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum + v28:BasicObject = CCallWithFrame v27, :Integer#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_boolean() { + eval(" + def test(x, y) = x ^ y + test(true, 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1008, ^@0x1010, cme:0x1018) + v27:TrueClass = GuardType v12, TrueClass + v28:BasicObject = CCallWithFrame v27, :TrueClass#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_args() { + eval(" + def test(x, y) = x.^() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_specialize_hash_size() { + eval(" + def test(hash) = hash.size + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :hash@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, size@0x1010, cme:0x1018) + v25:HashExact = GuardType v10, HashExact + v26:Fixnum = CCall v25, :Hash#size@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_eliminate_hash_size() { + eval(" + def test(hash) + hash.size + 5 + end + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :hash@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, size@0x1010, cme:0x1018) + v29:HashExact = GuardType v10, HashExact + v20:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_respond_to_p_true() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_no_method() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, respond_to_missing?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(C@0x1010, foo@0x1078, cme:0x1080) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_default_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v30:FalseClass = Const Value(false) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, false) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:FalseClass = Const Value(false) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_true_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, true) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_truthy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, 4) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_missing() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + PatchPoint MethodRedefined(C@0x1010, respond_to_missing?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(C@0x1010, foo@0x1078, cme:0x1080) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_do_not_optimize_redefined_respond_to_missing() { + eval(r#" + class C + def respond_to_missing?(method, include_private = false) + true + end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v27:BasicObject = CCallVariadic v26, :Kernel#respond_to?@0x1048, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putself() { + eval(r#" + def callee = self + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_string() { + eval(r#" + # frozen_string_literal: true + def callee = "abc" + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:StringExact[VALUE(0x1038)] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putnil() { + eval(r#" + def callee = nil + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_true() { + eval(r#" + def callee = true + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:TrueClass = Const Value(true) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_false() { + eval(r#" + def callee = false + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:FalseClass = Const Value(false) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_zero() { + eval(r#" + def callee = 0 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_one() { + eval(r#" + def callee = 1 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v20:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_parameter() { + eval(r#" + def callee(x) = x + def test = callee 3 + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_without_block_direct_last_parameter() { + eval(r#" + def callee(x, y, z) = z + def test = callee 1, 2, 3 + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_splat() { + eval(" + def foo = itself + + def test + # Use a local to inhibit compile.c peephole optimization to ensure callsites have VM_CALL_ARGS_SPLAT + empty = [] + foo(*empty) + ''.display(*empty) + itself(*empty) + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v19:ArrayExact = ToArray v13 + v21:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing + v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:StringExact = StringCopy v25 + PatchPoint NoEPEscape(test) + v31:ArrayExact = ToArray v13 + v33:BasicObject = Send v26, :display, v31 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v41:ArrayExact = ToArray v13 + v43:BasicObject = Send v8, :itself, v41 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_inline_symbol_to_sym() { + eval(r#" + def test(o) = o.to_sym + test :foo + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, to_sym@0x1010, cme:0x1018) + v22:StaticSymbol = GuardType v10, StaticSymbol + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_integer_to_i() { + eval(r#" + def test(o) = o.to_i + test 5 + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_i@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_send_with_block_with_no_params() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_with_block_with_one_param() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do |_| + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_with_block_with_multiple_params() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do |_a, _b| + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_inline_send_with_symbol_block() { + eval(r#" + def callee = 123 + public def the_block = 456 + def test + callee(&:the_block) + end + puts test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StaticSymbol[:the_block] = Const Value(VALUE(0x1000)) + v13:BasicObject = Send v6, &block, :callee, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_profile_stack_skips_block_arg() { + // Regression test: profile_stack must skip the &block arg on the stack when mapping + // profiled operand types. Without the fix, the receiver type would be mapped to the + // wrong stack slot, causing resolve_receiver_type to return NoProfile. + // With the fix, the receiver type is correctly resolved and the send gets past type + // resolution to hit the ARGS_BLOCKARG guard (ComplexArgPass) instead of NoProfile. + eval(" + def test(&block) = [].map(&block) + test { |x| x }; test { |x| x } + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eq_stringexact() { + eval(r#" + def test(l, r) = l == r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_string_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ==@0x1010, cme:0x1018) + v29:StringSubclass[class_exact:C] = GuardType v12, StringSubclass[class_exact:C] + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_stringexact_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_stringexact() { + eval(r#" + def test(l, r) = l === r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_string_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ===@0x1010, cme:0x1018) + v28:StringSubclass[class_exact:C] = GuardType v12, StringSubclass[class_exact:C] + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_string_equal_same_operand_true() { + eval(r#" + def test(s) = s == s + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v26:StringExact = GuardType v10, StringExact + v29:TrueClass = Const Value(true) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_string_eqq_same_operand_true() { + eval(r#" + def test(s) = s === s + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact + v28:TrueClass = Const Value(true) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_string_equal_frozen_local_same_operand_true() { + eval(r#" + def test + str = "a".freeze + str == str + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_fold_string_equal_frozen_distinct_literals_false() { + eval(r#" + def test + "a".freeze == "b".freeze + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v28:FalseClass = Const Value(false) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_not_fold_string_equal_true_without_pragma() { + eval(r#" + def test + "a" == "a" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v27:BoolExact = StringEqual v11, v14 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_not_fold_string_equal_false_without_pragma() { + eval(r#" + def test + "a" == "b" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v27:BoolExact = StringEqual v11, v14 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_string_equal_true_with_pragma() { + eval(r#" + # frozen_string_literal: true + def test + "a" == "a" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v26:TrueClass = Const Value(true) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_string_equal_false_with_pragma() { + eval(r#" + # frozen_string_literal: true + def test + "a" == "b" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_not_fold_string_equal_after_string_append_mutation() { + eval(r#" + def test + a = "a" + b = "a" + a << "a" + a == b + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:StringExact = StringCopy v16 + v21:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v22:StringExact = StringCopy v21 + v27:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v28:StringExact = StringCopy v27 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v50:StringExact = StringAppend v17, v28 + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v55:BoolExact = StringEqual v17, v22 + CheckInterrupts + Return v55 + "); + } + + #[test] + fn test_not_fold_string_equal_distinct_objects() { + eval(r#" + def test(s, t) = s == t + test("x", "x") + test("x", "x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :t@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :t@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_not_fold_string_equal_one_side_known_literal() { + eval(r#" + def test(s) = "a" == s + test("a") + test("a") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:StringExact = StringCopy v14 + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v29:String = GuardType v10, String + v30:BoolExact = StringEqual v15, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn opt_neq_string_nil_falls_back_to_basic_object_neq() { + eval(r#" + def test(str) + str != nil + end + + test("x") + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :str@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :str@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v27:StringExact = GuardType v10, StringExact + v28:BoolExact = CCallWithFrame v27, :BasicObject#!=@0x1040, v15 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_string_not_equal_distinct_objects() { + eval(r#" + def test(s, t) = s != t + test("x", "x") + test("x", "x") + "#); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :t@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :t@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v33:String = GuardType v13, String + v34:BoolExact = StringEqual v29, v33 + v35:TrueClass = Const Value(true) + v36:CBool = IsBitNotEqual v34, v35 + v37:BoolExact = BoxBool v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_fold_string_not_equal_same_operand_false() { + eval(r#" + def test(s) = s != s + test("x") + test("x") + "#); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v26:StringExact = GuardType v10, StringExact + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v35:TrueClass = Const Value(true) + v32:TrueClass = Const Value(true) + v33:CBool = IsBitNotEqual v35, v32 + v34:BoolExact = BoxBool v33 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_specialize_string_size() { + eval(r#" + def test(s) + s.size + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, size@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact + v26:Fixnum = CCall v25, :String#size@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_elide_string_size() { + eval(r#" + def test(s) + s.size + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, size@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact + v20:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact + v25:CInt64 = LoadField v24, :len@0x1040 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_elide_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018) + v28:StringExact = GuardType v10, StringExact + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_string_length() { + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, length@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact + v26:Fixnum = CCall v25, :String#length@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_class_eqq() { + eval(r#" + def test(o) = String === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v27:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Class@0x1018, ===@0x1020, cme:0x1028) + v31:BoolExact = IsA v10, v27 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_dont_specialize_module_eqq() { + eval(r#" + def test(o) = Kernel === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v27:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Module@0x1018, ===@0x1020, cme:0x1028) + v31:BoolExact = CCall v27, :Module#===@0x1050, v10 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_is_a_class() { + eval(r#" + def test(o) = o.is_a?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v25:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1011, cme:0x1018) + v29:StringExact = GuardType v10, StringExact + v30:BoolExact = IsA v29, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_is_a_module() { + eval(r#" + def test(o) = o.is_a?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v25:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, is_a?@0x1020, cme:0x1028) + v29:StringExact = GuardType v10, StringExact + v30:BasicObject = CCallWithFrame v29, :Kernel#is_a?@0x1050, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_is_a() { + eval(r#" + def test(o) + o.is_a?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v29:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, is_a?@0x1020, cme:0x1028) + v33:StringExact = GuardType v10, StringExact + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_elide_class_eqq() { + eval(r#" + def test(o) + Integer === o + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v31:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Class@0x1018, ===@0x1020, cme:0x1028) + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_kind_of_class() { + eval(r#" + def test(o) = o.kind_of?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v25:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1011, cme:0x1018) + v29:StringExact = GuardType v10, StringExact + v30:BoolExact = IsA v29, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_kind_of_module() { + eval(r#" + def test(o) = o.kind_of?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v25:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, kind_of?@0x1020, cme:0x1028) + v29:StringExact = GuardType v10, StringExact + v30:BasicObject = CCallWithFrame v29, :Kernel#kind_of?@0x1050, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_kind_of() { + eval(r#" + def test(o) + o.kind_of?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v29:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, kind_of?@0x1020, cme:0x1028) + v33:StringExact = GuardType v10, StringExact + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_fold_is_a_true() { + eval(r#" + def test = 5.is_a?(Integer) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v22:ClassSubclass[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Integer@0x1008, is_a?@0x1009, cme:0x1010) + v27:TrueClass = Const Value(true) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_is_a_false() { + eval(r#" + def test = 5.is_a?(String) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v22:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Integer@0x1010, is_a?@0x1018, cme:0x1020) + v27:FalseClass = Const Value(false) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_is_a_array_subclass_folds_to_true() { + eval(r#" + class C < Array; end + O = C.new + def test = O.is_a?(Array) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:ArraySubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Array) + v25:ClassSubclass[Array@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(C@0x1020) + PatchPoint MethodRedefined(C@0x1020, is_a?@0x1028, cme:0x1030) + v31:TrueClass = Const Value(true) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_is_a_user_defined_class_folds_to_true() { + eval(r#" + class C; end + O = C.new + def test = O.is_a?(C) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, C) + v25:ClassSubclass[C@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(C@0x1018) + PatchPoint MethodRedefined(C@0x1018, is_a?@0x1019, cme:0x1020) + v31:TrueClass = Const Value(true) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_is_a_symbol_folds_to_true() { + eval(r#" + O = :my_static_symbol + def test = O.is_a?(Symbol) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:StaticSymbol[:my_static_symbol] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Symbol) + v25:ClassSubclass[Symbol@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint MethodRedefined(Symbol@0x1018, is_a?@0x1019, cme:0x1020) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn counting_complex_feature_use_for_fallback() { + eval(" + define_method(:fancy) { |_a, *_b, kw: 100, **kw_rest, &block| } + def test = fancy(1) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:BasicObject = Send v6, :fancy, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn call_method_forwardable_param() { + eval(" + def forwardable(...) = itself(...) + def call_forwardable = forwardable + call_forwardable + "); + assert_snapshot!(hir_string("call_forwardable"), @" + fn call_forwardable@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :forwardable # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_string_length() { + eval(r#" + def test(s) + s.length + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, length@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact + v20:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_fold_self_class_respond_to_true() { + eval(r#" + class C + class << self + attr_accessor :_lex_actions + private :_lex_actions, :_lex_actions= + end + self._lex_actions = [1, 2, 3] + def initialize + if self.class.respond_to?(:_lex_actions, true) + :CORRECT + else + :oh_no_wrong + end + end + end + C.new # warm up + TEST = C.instance_method(:initialize) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn initialize@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v43:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] + v46:ClassSubclass[C@0x1000] = Const Value(VALUE(0x1000)) + v13:StaticSymbol[:_lex_actions] = Const Value(VALUE(0x1038)) + v15:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Class@0x1040, respond_to?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(Class@0x1040, _lex_actions@0x1078, cme:0x1080) + v52:TrueClass = Const Value(true) + CheckInterrupts + v26:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) + Return v26 + "); + } + + #[test] + fn test_fold_self_class_name() { + eval(r#" + class C; end + def test(o) = o.class.name + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, class@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v28:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1040, name@0x1048, cme:0x1050) + v32:StringExact|NilClass = CCall v28, :Module#name@0x1078 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_fold_kernel_class() { + eval(r#" + class C; end + def test(o) = o.class + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, class@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] + v26:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_fixnum_class() { + eval(r#" + def test = 5.class + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, class@0x1008, cme:0x1010) + v22:ClassSubclass[Integer@0x1000] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_singleton_class() { + eval(r#" + def test = self.class + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, class@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] + v21:ClassSubclass[Object@0x1038] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_print_nil_module_name() { + eval(r#" + X = [Module.new].freeze + def test = X[0] + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, X) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v37:ModuleExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v37 + "); + } + + #[test] + fn no_load_from_ep_right_after_entrypoint() { + let formatted = eval(" + def read_nil_local(a, _b, _c) + formatted ||= a + @formatted = formatted + -> { formatted } # the environment escapes + end + + def call + puts [], [], [], [] # fill VM stack with junk + read_nil_local(true, 1, 1) # expected direct send + end + + call # profile + call # compile + @formatted + "); + assert_eq!(Qtrue, formatted, "{}", formatted.obj_info()); + assert_snapshot!(hir_string("read_nil_local"), @" + fn read_nil_local@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :_b@0x1001 + v5:BasicObject = LoadField v2, :_c@0x1002 + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :_b@2 + v12:BasicObject = LoadArg :_c@3 + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:NilClass): + CheckInterrupts + SetLocal :formatted, l0, EP@3, v16 + PatchPoint SingleRactorMode + SetIvar v15, :@formatted, v16 + v47:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, lambda@0x1018, cme:0x1020) + v63:BasicObject = CCallWithFrame v47, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 + v50:CPtr = GetEP 0 + v51:BasicObject = LoadField v50, :a@0x1001 + v52:BasicObject = LoadField v50, :_b@0x1002 + v53:BasicObject = LoadField v50, :_c@0x1058 + v54:BasicObject = LoadField v50, :formatted@0x1059 + CheckInterrupts + Return v63 + "); + } + + #[test] + fn test_fold_load_field_frozen_constant_object() { + // Basic case: frozen constant object with attr_accessor + eval(" + class TestFrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + FROZEN_OBJ = TestFrozen.new.freeze + + def test = FROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_OBJ) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozen@0x1010) + PatchPoint MethodRedefined(TestFrozen@0x1010, a@0x1018, cme:0x1020) + v30:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_multiple_ivars() { + // Frozen object with multiple instance variables + eval(" + class TestMultiIvars + attr_accessor :a, :b, :c + def initialize + @a = 10 + @b = 20 + @c = 30 + end + end + + MULTI_FROZEN = TestMultiIvars.new.freeze + + def test = MULTI_FROZEN.b + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MULTI_FROZEN) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestMultiIvars@0x1010) + PatchPoint MethodRedefined(TestMultiIvars@0x1010, b@0x1018, cme:0x1020) + v30:Fixnum[20] = Const Value(20) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_string_value() { + // Frozen object with a string ivar + eval(r#" + class TestFrozenStr + attr_accessor :name + def initialize + @name = "hello" + end + end + + FROZEN_STR = TestFrozenStr.new.freeze + + def test = FROZEN_STR.name + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_STR) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenStr@0x1010) + PatchPoint MethodRedefined(TestFrozenStr@0x1010, name@0x1018, cme:0x1020) + v30:StringExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_nil_value() { + // Frozen object with nil ivar + eval(" + class TestFrozenNil + attr_accessor :value + def initialize + @value = nil + end + end + + FROZEN_NIL = TestFrozenNil.new.freeze + + def test = FROZEN_NIL.value + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_NIL) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenNil@0x1010) + PatchPoint MethodRedefined(TestFrozenNil@0x1010, value@0x1018, cme:0x1020) + v30:NilClass = Const Value(nil) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_fold_load_field_unfrozen_object() { + // Non-frozen object should NOT be folded + eval(" + class TestUnfrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + UNFROZEN_OBJ = TestUnfrozen.new + + def test = UNFROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, UNFROZEN_OBJ) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestUnfrozen@0x1010) + PatchPoint MethodRedefined(TestUnfrozen@0x1010, a@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:BasicObject = LoadField v20, :@a@0x104a + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_load_field_frozen_with_attr_reader() { + // Using attr_reader instead of attr_accessor + eval(" + class TestAttrReader + attr_reader :value + def initialize(v) + @value = v + end + end + + FROZEN_READER = TestAttrReader.new(42).freeze + + def test = FROZEN_READER.value + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_READER) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestAttrReader@0x1010) + PatchPoint MethodRedefined(TestAttrReader@0x1010, value@0x1018, cme:0x1020) + v30:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_symbol_value() { + // Frozen object with a symbol ivar + eval(" + class TestFrozenSym + attr_accessor :sym + def initialize + @sym = :hello + end + end + + FROZEN_SYM = TestFrozenSym.new.freeze + + def test = FROZEN_SYM.sym + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_SYM) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenSym@0x1010) + PatchPoint MethodRedefined(TestFrozenSym@0x1010, sym@0x1018, cme:0x1020) + v30:StaticSymbol[:hello] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_true_false() { + // Frozen object with boolean ivars + eval(" + class TestFrozenBool + attr_accessor :flag + def initialize + @flag = true + end + end + + FROZEN_TRUE = TestFrozenBool.new.freeze + + def test = FROZEN_TRUE.flag + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_TRUE) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenBool@0x1010) + PatchPoint MethodRedefined(TestFrozenBool@0x1010, flag@0x1018, cme:0x1020) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_fold_load_field_dynamic_receiver() { + // Dynamic receiver (not a constant) should NOT be folded even if object is frozen + eval(" + class TestDynamic + attr_accessor :val + def initialize + @val = 99 + end + end + + def test(obj) = obj.val + o = TestDynamic.new.freeze + test o + test o + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :obj@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(TestDynamic@0x1008) + PatchPoint MethodRedefined(TestDynamic@0x1008, val@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:TestDynamic] = GuardType v10, ObjectSubclass[class_exact:TestDynamic] + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:BasicObject = LoadField v23, :@val@0x1042 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_load_field_frozen_nested_access() { + // Accessing multiple fields from frozen constant in sequence + eval(" + class TestNestedAccess + attr_accessor :x, :y + def initialize + @x = 100 + @y = 200 + end + end + + NESTED_FROZEN = TestNestedAccess.new.freeze + + def test = NESTED_FROZEN.x + NESTED_FROZEN.y + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, NESTED_FROZEN) + v27:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, x@0x1018, cme:0x1020) + v53:Fixnum[100] = Const Value(100) + PatchPoint StableConstantNames(0x1048, NESTED_FROZEN) + v34:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, y@0x1050, cme:0x1058) + v55:Fixnum[200] = Const Value(200) + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) + v56:Fixnum[300] = Const Value(300) + CheckInterrupts + Return v56 + "); + } + + #[test] + fn test_dont_fold_load_field_with_primitive_return_type() { + eval(r#" + S = "abc".freeze + def test = S.bytesize + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v20:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, bytesize@0x1018, cme:0x1020) + v25:CInt64 = LoadField v20, :len@0x1048 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn optimize_call_to_private_method_iseq_with_fcall() { + eval(r#" + class C + def callprivate = secret + private def secret = 42 + end + C.new.callprivate + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprivate)"), @" + fn callprivate@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_iseq() { + eval(r#" + class C + private def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn optimize_call_to_private_method_cfunc_with_fcall() { + eval(r#" + class BasicObject + def callprivate = initialize rescue $! + end + Obj = BasicObject.new.callprivate + "#); + assert_snapshot!(hir_string_proc("BasicObject.instance_method(:callprivate)"), @" + fn callprivate@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(BasicObject@0x1000) + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v21:BasicObjectExact = GuardType v6, BasicObjectExact + v22:NilClass = Const Value(nil) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_cfunc() { + eval(r#" + Obj = BasicObject.new + def test = Obj.initialize rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:BasicObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :initialize # SendFallbackReason: Send: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn dont_optimize_call_to_private_top_level_method() { + eval(r#" + def toplevel_method = :OK + Obj = Object.new + def test = Obj.toplevel_method rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :toplevel_method # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn optimize_call_to_protected_method_iseq_with_fcall() { + eval(r#" + class C + def callprotected = secret + protected def secret = 42 + end + C.new.callprotected + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprotected)"), @" + fn callprotected@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_protected_method_iseq() { + eval(r#" + class C + protected def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + // Test that when a singleton class has been seen for a class, we skip the + // NoSingletonClass optimization to avoid an invalidation loop. + #[test] + fn test_skip_optimization_after_singleton_class_seen() { + // First, compile a function that uses the NoSingletonClass assumption + eval(r#" + def test(s, proc) + s.length + proc.call + s.length + end + test("hi", -> {}) + test("hi", -> {}) + "#); + let hir = hir_string("test"); + assert!(hir.contains("NoSingletonClass(String"), "{hir}"); + + // Now we break the assumption by defining a singleton method on a string. + eval(r#" + special_string = +"" + test(special_string, -> { def special_string.length = -1 }) + "#); + + // The output should NOT have NoSingletonClass patchpoint for String, and should + // fall back to SendWithoutBlock instead of the optimized CCall path. + let hir = hir_string("test"); + assert!(! hir.contains("NoSingletonClass(String"), "{hir}"); + assert_snapshot!(hir, @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :proc@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :proc@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:BasicObject = Send v12, :length # SendFallbackReason: Singleton class previously created for receiver class + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, call@0x1010, cme:0x1018) + v40:ObjectSubclass[class_exact:Proc] = GuardType v13, ObjectSubclass[class_exact:Proc] + v41:BasicObject = InvokeProc v40 + PatchPoint NoEPEscape(test) + v32:BasicObject = Send v12, :length # SendFallbackReason: Singleton class previously created for receiver class + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_no_singleton_class_busts_isolated_per_iseq() { + // First, compile a function that uses the NoSingletonClass assumption + eval(r#" + def will_bust(s, proc) + s.length + proc.call + s.length + end + + def call_length(s) = s.length + + will_bust("hi", -> {}) + will_bust("hi", -> {}) + "#); + let hir = hir_string("will_bust"); + assert!(hir.contains("NoSingletonClass(String"), "{hir}"); + + // Now we break the assumption by defining a singleton method on a string. + eval(r#" + special_string = +"" + will_bust(special_string, -> { def special_string.length = -1 }) + "#); + let hir = hir_string("will_bust"); + assert!(! hir.contains("NoSingletonClas(String"), "{hir}"); + + // But, the unrelated call_length() should still use NoSingletonClass + eval("call_length('profile')"); + let hir = hir_string("call_length"); + assert!(hir.contains("NoSingletonClass"), "{hir}"); + } + + #[test] + fn test_invokesuper_to_iseq_optimizes_to_direct() { + eval(" + class A + def foo + 'A' + end + end + + class B < A + def foo + super + end + end + + B.new.foo; B.new.foo + "); + + // A Ruby method as the target of `super` should optimize provided no block is given. + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendDirect but got:\n{hir}"); + assert!(hir.contains("SendDirect"), "Should optimize to SendDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:BasicObject = SendDirect v6, 0x1050, :foo (0x1060) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_invokesuper_from_a_block() { + _ = eval(" + define_method(:itself) { super() } + itself + "); + + assert_snapshot!(hir_string("itself"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call from within a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_with_positional_args_optimizes_to_direct() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + super(x) + 1 + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendDirect but got:\n{hir}"); + assert!(hir.contains("SendDirect"), "Should optimize to SendDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(A@0x1008, foo@0x1010, cme:0x1018) + v28:CPtr = GetEP 0 + v29:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v30:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v29, Value(VALUE(0x1048)) + v31:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v32:FalseClass = GuardBitEquals v31, Value(false) + v33:BasicObject = SendDirect v9, 0x1058, :foo (0x1068), v10 + v18:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1070, +@0x1078, cme:0x1080) + v36:Fixnum = GuardType v33, Fixnum + v37:Fixnum = FixnumAdd v36, v18 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_invokesuper_with_forwarded_splat_args_remains_invokesuper() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(*x) + super + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v18:BasicObject = InvokeSuper v9, 0x1008, v16 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_invokesuper_with_block_literal_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo + super { 'from subclass' } + end + end + + B.new.foo; B.new.foo + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for block literal:\n{hir}"); + + // With a block, we don't optimize to SendDirect + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call made with a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_to_cfunc_optimizes_to_ccall() { + eval(" + class C < Hash + def size + super + end + end + + C.new.size + "); + + let hir = hir_string_proc("C.new.method(:size)"); + assert!(!hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @" + fn size@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:Fixnum = CCall v6, :Hash#size@0x1050 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_inline_invokesuper_to_basicobject_initialize() { + eval(" + class C + def initialize + super + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:NilClass = Const Value(nil) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_invokesuper_to_variadic_cfunc_optimizes_to_ccall() { + eval(" + class MyString < String + def byteindex(needle, offset = 0) + super(needle, offset) + end + end + + MyString.new('hello world').byteindex('world', 0); MyString.new('hello world').byteindex('world', 0) + "); + + let hir = hir_string_proc("MyString.new('hello world').method(:byteindex)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to CCallVariadic but got:\n{hir}"); + assert!(hir.contains("CCallVariadic"), "Should optimize to CCallVariadic for variadic cfunc:\n{hir}"); + + assert_snapshot!(hir, @" + fn byteindex@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :needle@0x1000 + v4:BasicObject = LoadField v2, :offset@0x1001 + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1002)] = Const CPtr(0x1002) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:BasicObject = LoadArg :needle@1 + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:BasicObject): + v25:Fixnum[0] = Const Value(0) + Jump bb5(v20, v21, v25) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :needle@1 + v18:BasicObject = LoadArg :offset@2 + Jump bb5(v16, v17, v18) + bb5(v28:BasicObject, v29:BasicObject, v30:BasicObject): + PatchPoint MethodRedefined(String@0x1008, byteindex@0x1010, cme:0x1018) + v44:CPtr = GetEP 0 + v45:RubyValue = LoadField v44, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v46:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v45, Value(VALUE(0x1048)) + v47:RubyValue = LoadField v44, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v48:FalseClass = GuardBitEquals v47, Value(false) + v49:BasicObject = CCallVariadic v28, :String#byteindex@0x1058, v29, v30 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_invokesuper_with_blockarg_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo(&blk) + other_block = proc { 'different block' } + super(&other_block) + end + end + + B.new.foo { 'passed' }; B.new.foo { 'passed' } + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :blk@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :blk@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + PatchPoint NoSingletonClass(B@0x1008) + PatchPoint MethodRedefined(B@0x1008, proc@0x1010, cme:0x1018) + v39:ObjectSubclass[class_exact:B] = GuardType v11, ObjectSubclass[class_exact:B] + v40:BasicObject = CCallWithFrame v39, :Kernel#proc@0x1040, block=0x1048 + v19:CPtr = GetEP 0 + v20:BasicObject = LoadField v19, :blk@0x1050 + SetLocal :other_block, l0, EP@3, v40 + v27:CPtr = GetEP 0 + v28:BasicObject = LoadField v27, :other_block@0x1051 + v30:BasicObject = InvokeSuper v39, 0x1058, v28 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_invokesuper_with_symbol_to_proc_remains_invokesuper() { + eval(" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + super(items, &:succ) + end + end + + B.new.foo([1, 2, 3]); B.new.foo([1, 2, 3]) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for symbol-to-proc:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :items@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :items@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:StaticSymbol[:succ] = Const Value(VALUE(0x1008)) + v18:BasicObject = InvokeSuper v9, 0x1010, v10, v16 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_invokesuper_with_keyword_args_remains_invokesuper() { + eval(" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + B.new.foo('image data'); B.new.foo('image data') + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :content@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:BasicObject, v18:BasicObject): + v21:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v22:StringExact = StringCopy v21 + Jump bb5(v17, v22) + bb4(): + EntryPoint JIT(1) + v14:BasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :content@1 + Jump bb5(v14, v15) + bb5(v25:BasicObject, v26:BasicObject): + v32:BasicObject = InvokeSuper v25, 0x1010, v26 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb7(), bb6(v9, v17) + bb7(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v38:Fixnum[3] = Const Value(3) + Return v38 + bb6(v43:BasicObject, v44:Falsy): + v48:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v48 + "); + } + + #[test] + fn specialize_polymorphic_send_iseq() { + set_call_threshold(4); + eval(" + class C + def foo = 3 + end + + class D + def foo = 4 + end + + def test o + o.foo + 2 + end + + test C.new; test D.new; test C.new; test D.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v55:Fixnum[3] = Const Value(3) + Jump bb4(v16, v17, v55) + bb6(): + v24:CBool = HasType v10, ObjectSubclass[class_exact:D] + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + PatchPoint NoSingletonClass(D@0x1040) + PatchPoint MethodRedefined(D@0x1040, foo@0x1010, cme:0x1048) + v56:Fixnum[4] = Const Value(4) + Jump bb4(v25, v26, v56) + bb8(): + v33:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + v40:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1070, +@0x1078, cme:0x1080) + v59:Fixnum = GuardType v37, Fixnum + v60:Fixnum = FixnumAdd v59, v40 + CheckInterrupts + Return v60 + "); + } + + #[test] + fn specialize_polymorphic_send_with_immediate() { + set_call_threshold(4); + eval(" + class C; end + + def test o + o.itself + end + + test C.new; test 3; test C.new; test 4 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:ObjectSubclass[class_exact:C] = RefineType v18, ObjectSubclass[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, itself@0x1010, cme:0x1018) + Jump bb4(v16, v17, v20) + bb6(): + v24:CBool = HasType v10, Fixnum + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:Fixnum = RefineType v27, Fixnum + PatchPoint MethodRedefined(Integer@0x1040, itself@0x1010, cme:0x1018) + Jump bb4(v25, v26, v29) + bb8(): + v33:BasicObject = Send v10, :itself # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_fixnum_and_bignum() { + // Fixnum and Bignum both have class Integer, but they should be + // treated as different types for polymorphic dispatch because + // Fixnum is an immediate and Bignum is a heap object. + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + fixnum = 1 + bignum = 10**100 + test(fixnum) + test(bignum) + test(fixnum) + test(bignum) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, Fixnum + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:Fixnum = RefineType v18, Fixnum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v46:StringExact = CCallVariadic v20, :Integer#to_s@0x1040 + Jump bb4(v16, v17, v46) + bb6(): + v24:CBool = HasType v10, Bignum + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:Bignum = RefineType v27, Bignum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v49:StringExact = CCallVariadic v29, :Integer#to_s@0x1040 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_flonum_and_heap_float() { + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + flonum = 1.5 + heap_float = 1.7976931348623157e+308 + test(flonum) + test(heap_float) + test(flonum) + test(heap_float) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, Flonum + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:Flonum = RefineType v18, Flonum + PatchPoint MethodRedefined(Float@0x1008, to_s@0x1010, cme:0x1018) + v46:BasicObject = CCallWithFrame v20, :Float#to_s@0x1040 + Jump bb4(v16, v17, v46) + bb6(): + v24:CBool = HasType v10, HeapFloat + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:HeapFloat = RefineType v27, HeapFloat + PatchPoint MethodRedefined(Float@0x1008, to_s@0x1010, cme:0x1018) + v49:BasicObject = CCallWithFrame v29, :Float#to_s@0x1040 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_static_and_dynamic_symbol() { + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + static_sym = :foo + dynamic_sym = (\"zjit_dynamic_\" + Object.new.object_id.to_s).to_sym + test static_sym + test dynamic_sym + test static_sym + test dynamic_sym + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, StaticSymbol + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:StaticSymbol = RefineType v18, StaticSymbol + PatchPoint MethodRedefined(Symbol@0x1008, to_s@0x1010, cme:0x1018) + v48:StringExact = InvokeBuiltin leaf <inline_expr>, v20 + Jump bb4(v16, v17, v48) + bb6(): + v24:CBool = HasType v10, DynamicSymbol + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:DynamicSymbol = RefineType v27, DynamicSymbol + PatchPoint MethodRedefined(Symbol@0x1008, to_s@0x1010, cme:0x1018) + v49:StringExact = InvokeBuiltin leaf <inline_expr>, v29 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_iseq_duplicate_class_profiles() { + set_call_threshold(4); + eval(" + class C + def foo = 3 + end + + O1 = C.new + O1.instance_variable_set(:@foo, 1) + O2 = C.new + O2.instance_variable_set(:@bar, 2) + + def test o + o.foo + end + + test O1; test O2; test O1; test O2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v38:Fixnum[3] = Const Value(3) + Jump bb4(v16, v17, v38) + bb6(): + v24:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v24) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn upgrade_self_type_to_heap_after_setivar() { + // Snapshot the overflow path only when this build naturally keeps five + // ivars embedded and overflows on the next write. + let obj = eval(r#" + klass = Class.new do + def initialize + @v0 = 0 + @v1 = 1 + @v2 = 2 + @v3 = 3 + @v4 = 4 + end + + def test + @overflow = 1 + @after = 2 + end + end + + TEST = klass.instance_method(:test) + OBJ = klass.new + OBJ + "#); + // Skip builds where five ivars already force heap-backed storage. + if !obj.embedded_p() { + return; + } + + // Make sure the next write is the one that overflows into heap-backed + // storage, so this snapshot still exercises the self-type upgrade path. + let probe = eval(r#" + probe = OBJ.class.new + probe.instance_variable_set(:@overflow, 1) + probe + "#); + if probe.embedded_p() { + return; + } + eval("OBJ.test"); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@overflow, v10 + v14:HeapBasicObject = RefineType v6, HeapBasicObject + v17:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + v29:CShape = LoadField v14, :shape_id@0x1000 + v30:CShape[0x1001] = GuardBitEquals v29, CShape(0x1001) + v31:CPtr = LoadField v14, :as_heap@0x1002 + StoreField v31, :@after@0x1003, v17 + WriteBarrier v14, v17 + v34:CShape[0x1004] = Const CShape(0x1004) + StoreField v14, :shape_id@0x1000, v34 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn recompile_after_ep_escape_uses_ep_locals() { + // When a method creates a lambda, EP escapes to the heap. After + // invalidation and recompilation, the compiler must use EP-based + // locals (SetLocal/GetLocal) instead of SSA locals, because the + // spill target (stack) and the read target (heap EP) diverge. + eval(" + CONST = {}.freeze + def test_ep_escape(list, sep=nil, iter_method=:each) + sep ||= lambda { } + kwsplat = CONST + list.__send__(iter_method) {|*v| yield(*v) } + end + + test_ep_escape({a: 1}, nil, :each_pair) { |k, v| + test_ep_escape([1], lambda { }) { |x| } + } + test_ep_escape({a: 1}, nil, :each_pair) { |k, v| + test_ep_escape([1], lambda { }) { |x| } + } + "); + assert_snapshot!(hir_string("test_ep_escape"), @" + fn test_ep_escape@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :list@0x1000 + v4:BasicObject = LoadField v2, :sep@0x1001 + v5:BasicObject = LoadField v2, :iter_method@0x1002 + v6:NilClass = Const Value(nil) + v7:CPtr = LoadPC + v8:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v9:CBool = IsBitEqual v7, v8 + CondBranch v9, bb3(v1, v3, v4, v5, v6), bb9() + bb9(): + v11:CPtr[CPtr(0x1004)] = Const CPtr(0x1004) + v12:CBool = IsBitEqual v7, v11 + CondBranch v12, bb5(v1, v3, v4, v5, v6), bb10() + bb10(): + Jump bb7(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :list@1 + v18:NilClass = Const Value(nil) + v19:NilClass = Const Value(nil) + v20:NilClass = Const Value(nil) + Jump bb3(v16, v17, v18, v19, v20) + bb3(v36:BasicObject, v37:BasicObject, v38:BasicObject, v39:BasicObject, v40:NilClass): + v43:NilClass = Const Value(nil) + SetLocal :sep, l0, EP@5, v43 + Jump bb5(v36, v37, v43, v39, v40) + bb4(): + EntryPoint JIT(1) + v23:BasicObject = LoadArg :self@0 + v24:BasicObject = LoadArg :list@1 + v25:BasicObject = LoadArg :sep@2 + v26:NilClass = Const Value(nil) + v27:NilClass = Const Value(nil) + Jump bb5(v23, v24, v25, v26, v27) + bb5(v47:BasicObject, v48:BasicObject, v49:BasicObject, v50:BasicObject, v51:NilClass): + v54:StaticSymbol[:each] = Const Value(VALUE(0x1008)) + SetLocal :iter_method, l0, EP@4, v54 + Jump bb7(v47, v48, v49, v54, v51) + bb6(): + EntryPoint JIT(2) + v30:BasicObject = LoadArg :self@0 + v31:BasicObject = LoadArg :list@1 + v32:BasicObject = LoadArg :sep@2 + v33:BasicObject = LoadArg :iter_method@3 + v34:NilClass = Const Value(nil) + Jump bb7(v30, v31, v32, v33, v34) + bb7(v58:BasicObject, v59:BasicObject, v60:BasicObject, v61:BasicObject, v62:NilClass): + CheckInterrupts + v68:CBool = Test v60 + v69:Truthy = RefineType v60, Truthy + CondBranch v68, bb8(v58, v59, v69, v61, v62), bb11() + bb11(): + v71:Falsy = RefineType v60, Falsy + PatchPoint MethodRedefined(Object@0x1010, lambda@0x1018, cme:0x1020) + v118:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v58, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] + v119:BasicObject = CCallWithFrame v118, :Kernel#lambda@0x1048, block=0x1050 + v75:CPtr = GetEP 0 + v76:BasicObject = LoadField v75, :list@0x1001 + v78:BasicObject = LoadField v75, :iter_method@0x1058 + v79:BasicObject = LoadField v75, :kwsplat@0x1059 + SetLocal :sep, l0, EP@5, v119 + Jump bb8(v118, v76, v119, v78, v79) + bb8(v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1060, CONST) + v115:HashExact[VALUE(0x1068)] = Const Value(VALUE(0x1068)) + SetLocal :kwsplat, l0, EP@3, v115 + v96:CPtr = GetEP 0 + v97:BasicObject = LoadField v96, :list@0x1001 + v99:CPtr = GetEP 0 + v100:BasicObject = LoadField v99, :iter_method@0x1058 + v102:BasicObject = Send v97, 0x1070, :__send__, v100 # SendFallbackReason: Send: unsupported method type Optimized + v103:CPtr = GetEP 0 + v104:BasicObject = LoadField v103, :list@0x1001 + v105:BasicObject = LoadField v103, :sep@0x1002 + v106:BasicObject = LoadField v103, :iter_method@0x1058 + v107:BasicObject = LoadField v103, :kwsplat@0x1059 + CheckInterrupts + Return v102 + "); + } + + #[test] + fn test_array_each() { + eval("[1, 2, 3].each { |x| x }"); + assert_snapshot!(hir_string_proc("Array.instance_method(:each)"), @" + fn each@<internal:array>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + v15:TrueClass|NilClass = Defined yield, v13 + v17:CBool = Test v15 + CondBranch v17, bb9(), bb4(v8, v9) + bb9(): + v35:Fixnum[0] = Const Value(0) + Jump bb8(v8, v35) + bb8(v48:BasicObject, v49:Fixnum): + v84:Array = RefineType v48, Array + v85:CInt64 = ArrayLength v84 + v86:Fixnum = BoxFixnum v85 + v87:BoolExact = FixnumGe v49, v86 + v54:CBool = Test v87 + CondBranch v54, bb10(), bb7(v48, v49) + bb10(): + CheckInterrupts + Return v48 + bb7(v67:BasicObject, v68:Fixnum): + v88:Array = RefineType v67, Array + v89:CInt64 = UnboxFixnum v68 + v90:BasicObject = ArrayAref v88, v89 + v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: InvokeBlock: not yet specialized + v91:Fixnum[1] = Const Value(1) + v92:Fixnum = FixnumAdd v68, v91 + PatchPoint NoEPEscape(each) + Jump bb8(v67, v92) + bb4(v23:BasicObject, v24:NilClass): + v28:BasicObject = InvokeBuiltin <inline_expr>, v23 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_delete_duplicate_store() { + eval(" + class C + def initialize + a = 1 + @a = a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v35:HeapBasicObject = GuardType v8, HeapBasicObject + v36:CShape = LoadField v35, :shape_id@0x1000 + v37:CShape[0x1001] = GuardBitEquals v36, CShape(0x1001) recompile + StoreField v35, :@a@0x1002, v13 + WriteBarrier v35, v13 + v40:CShape[0x1003] = Const CShape(0x1003) + StoreField v35, :shape_id@0x1000, v40 + v20:HeapBasicObject = RefineType v35, HeapBasicObject + PatchPoint NoEPEscape(initialize) + PatchPoint SingleRactorMode + WriteBarrier v20, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_remove_duplicate_store_with_non_effectful_insns_between() { + eval(" + class C + def initialize + a = 1 + @a = a + b = 5 + b += a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v49:HeapBasicObject = GuardType v10, HeapBasicObject + v50:CShape = LoadField v49, :shape_id@0x1000 + v51:CShape[0x1001] = GuardBitEquals v50, CShape(0x1001) recompile + StoreField v49, :@a@0x1002, v16 + WriteBarrier v49, v16 + v54:CShape[0x1003] = Const CShape(0x1003) + StoreField v49, :shape_id@0x1000, v54 + v23:HeapBasicObject = RefineType v49, HeapBasicObject + v26:Fixnum[5] = Const Value(5) + PatchPoint NoEPEscape(initialize) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v65:Fixnum[6] = Const Value(6) + PatchPoint SingleRactorMode + WriteBarrier v23, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_remove_two_stores() { + eval(" + class C + def initialize + a = 1 + @a = a + @a = a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v43:HeapBasicObject = GuardType v8, HeapBasicObject + v44:CShape = LoadField v43, :shape_id@0x1000 + v45:CShape[0x1001] = GuardBitEquals v44, CShape(0x1001) recompile + StoreField v43, :@a@0x1002, v13 + WriteBarrier v43, v13 + v48:CShape[0x1003] = Const CShape(0x1003) + StoreField v43, :shape_id@0x1000, v48 + v20:HeapBasicObject = RefineType v43, HeapBasicObject + PatchPoint NoEPEscape(initialize) + PatchPoint SingleRactorMode + WriteBarrier v20, v13 + v28:HeapBasicObject = RefineType v20, HeapBasicObject + WriteBarrier v28, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_exit_from_function_stub_for_opt_keyword_callee() { + // We have a SendDirect to a callee that fails to compile, + // so the function stub has to take care of exiting to + // interpreter. + eval(" + def target(a = binding.local_variable_get(:a), b: nil) + ::RubyVM::ZJIT.induce_compile_failure! + [a, b] + end + + def entry = target(b: -1) + + raise 'wrong' unless [nil, -1] == entry + raise 'wrong' unless [nil, -1] == entry + "); + + crate::hir::tests::hir_build_tests::assert_compile_fails("target", ParseError::DirectiveInduced); + let hir = hir_string("entry"); + assert!(hir.contains("SendDirect"), "{hir}"); + } + + #[test] + fn test_exit_from_function_stub_for_lead_opt() { + // We have a SendDirect to a callee that fails to compile, + // so the function stub has to take care of exiting to + // interpreter. + let result = eval(" + def target(_required, a = a, b = b) + ::RubyVM::ZJIT.induce_compile_failure! + a + end + + def entry = target(1) + + entry + entry + "); + assert_eq!(Qnil, result); + + crate::hir::tests::hir_build_tests::assert_compile_fails("target", ParseError::DirectiveInduced); + let hir = hir_string("entry"); + assert!(hir.contains("SendDirect"), "{hir}"); + } + + #[test] + fn test_recompile_no_profile_send() { + // Test the SideExit -> recompile flow: a no-profile send becomes a SideExit, + // the exit profiles the send, triggers recompilation, and the new version + // optimizes it to SendDirect. + eval(" + def greet_recompile(x) = x.to_s + def test_no_profile_recompile(flag) + if flag + greet_recompile(42) + else + 'hello' + end + end + "); + + // With call_threshold=2, num_profiles=1: + // 1st call profiles (flag=false, so greet is never reached) + // 2nd call compiles (greet has no profile data -> SideExit recompile) + eval("test_no_profile_recompile(false); test_no_profile_recompile(false)"); + + // Now call with flag=true. This hits the SideExit, which profiles + // the send and invalidates the ISEQ for recompilation. + eval("test_no_profile_recompile(true)"); + + // After profiling via the side exit, rebuilding HIR should now + // have a SendDirect for greet_recompile instead of SideExit. + assert_snapshot!(hir_string("test_no_profile_recompile"), @r" + fn test_no_profile_recompile@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :flag@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :flag@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v23:Fixnum[42] = Const Value(42) + PatchPoint MethodRedefined(Object@0x1008, greet_recompile@0x1010, cme:0x1018) + v43:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] + v44:BasicObject = SendDirect v43, 0x1040, :greet_recompile (0x1050), v23 + CheckInterrupts + Return v44 + bb4(v30:BasicObject, v31:Falsy): + v35:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) + v36:StringExact = StringCopy v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_no_profile_send_on_final_version() { + // On the final ISEQ version (MAX_ISEQ_VERSIONS reached), no-profile sends should + // remain as Send fallbacks instead of being converted to SideExits, since recompilation + // is no longer possible and SideExits would fire every time without benefit. + // + // Use call_threshold=3 to ensure the method is auto-compiled before hir_string() builds + // the HIR. The auto-compile creates version 1, and hir_string() creates version 2 + // (= MAX_ISEQ_VERSIONS), so this is the final version. + set_call_threshold(3); + eval(" + def greet_final(x) = x.to_s + def test_final_version(flag) + if flag + greet_final(42) + else + 'hello' + end + end + "); + // Call enough times to trigger auto-compilation. flag=false so greet_final is never + // reached and has no profile data. + eval("3.times { test_final_version(false) }"); + + // On the final version, greet_final should be a Send fallback, not a SideExit. + assert_snapshot!(hir_string("test_final_version"), @r" + fn test_final_version@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :flag@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :flag@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v23:Fixnum[42] = Const Value(42) + v25:BasicObject = Send v9, :greet_final, v23 # SendFallbackReason: SendWithoutBlock: no profile data available + CheckInterrupts + Return v25 + bb4(v30:BasicObject, v31:Falsy): + v35:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v36:StringExact = StringCopy v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_invokeblock_ifunc() { + eval(" + class IFuncTestList + include Enumerable + def each + yield 1 + yield 2 + end + end + IFuncTestList.new.map { |x| x } + "); + assert_snapshot!(hir_string_proc("IFuncTestList.instance_method(:each)"), @" + fn each@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:CPtr = GetEP 0 + v13:CInt64 = LoadField v12, :VM_ENV_DATA_INDEX_SPECVAL@0x1000 + v14:CInt64[3] = Const CInt64(3) + v15:CInt64 = IntAnd v13, v14 + v16:CInt64[3] = Const CInt64(3) + v17:CBool = IsBitEqual v15, v16 + CondBranch v17, bb5(), bb6() + bb5(): + v20:BasicObject = InvokeBlockIfunc v13, v10 + Jump bb4(v20) + bb6(): + v22:BasicObject = InvokeBlock, v10 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb4(v22) + bb4(v18:BasicObject): + v27:Fixnum[2] = Const Value(2) + v29:CPtr = GetEP 0 + v30:CInt64 = LoadField v29, :VM_ENV_DATA_INDEX_SPECVAL@0x1000 + v31:CInt64[3] = Const CInt64(3) + v32:CInt64 = IntAnd v30, v31 + v33:CInt64[3] = Const CInt64(3) + v34:CBool = IsBitEqual v32, v33 + CondBranch v34, bb8(), bb9() + bb8(): + v37:BasicObject = InvokeBlockIfunc v30, v27 + Jump bb7(v37) + bb9(): + v39:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb7(v39) + bb7(v35:BasicObject): + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_dedup_guard_type() { + // Two subtractions on the same Fixnum operand `n` each require a + // GuardType n, Fixnum. The second guard is redundant and should be + // eliminated by fold_constants. + eval(" + def test(n) + (n - 1) + (n - 2) + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, -@0x1010, cme:0x1018) + v35:Fixnum = GuardType v10, Fixnum + v36:Fixnum = FixnumSub v35, v15 + v21:Fixnum[2] = Const Value(2) + v40:Fixnum = FixnumSub v35, v21 + PatchPoint MethodRedefined(Integer@0x1008, +@0x1040, cme:0x1048) + v44:Fixnum = FixnumAdd v36, v40 + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_dedup_guard_type_across_cfg_join() { + eval(" + def test(n, cond) + if cond + a = n + 1 + else + a = n + 2 + end + n + a + end + test(1, true); test(1, false) + "); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert_eq!( + guard_count, 2, + "expected 2 GuardType instructions after cross-block dedup, found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_forward_guard_through_conditional_branch() { + eval(" + def test(n, a, b) + if a + if b + n + 1 + else + n + 2 + end + else + n + 3 + end + end + test(1, true, true); test(1, true, false); test(1, false, false) + "); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert!( + guard_count <= 3, + "expected at most 3 GuardType instructions (one per leaf branch) after forwarding through conditional branches, found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_no_forward_when_no_guard_in_branches() { + let src = " + def test(n, cond) + a = if cond then 1 else 2 end + n + a + end + test(1, true); test(1, false) + "; + eval(src); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert_eq!( + guard_count, 1, + "expected 1 GuardType (merge block only), found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_infer_types_across_non_maximal_basic_blocks() { + // Previous worklist-based type inference only worked for maximal SSA. This is a regression + // test for hanging. + eval(" + class TheClass + def set_value_loop + i = 0 + while i < 10 + @levar ||= i + i += 1 + end + end + end + 3.times do |i| + TheClass.new.set_value_loop + end + "); + assert_snapshot!(hir_string_proc("TheClass.instance_method(:set_value_loop)"), @" + fn set_value_loop@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[0] = Const Value(0) + CheckInterrupts + Jump bb6(v8, v13) + bb6(v19:BasicObject, v20:Fixnum): + v24:Fixnum[10] = Const Value(10) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v110:BoolExact = FixnumLt v20, v24 + CheckInterrupts + v30:CBool = Test v110 + CondBranch v30, bb4(v19, v20), bb7() + bb4(v40:BasicObject, v41:Fixnum): + PatchPoint SingleRactorMode + v46:HeapBasicObject = GuardType v40, HeapBasicObject + v47:CUInt64 = LoadField v46, :RBASIC_FLAGS@0x1038 + v49:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v50:CPtr[CPtr(0x1039)] = Const CPtr(0x1039) + v51 = RefineType v50, CUInt64 + v52:CInt64 = IntAnd v47, v49 + v53:CBool = IsBitEqual v52, v51 + CondBranch v53, bb9(), bb10() + bb9(): + v55:BasicObject = LoadField v46, :@levar@0x103a + Jump bb8(v55) + bb10(): + v57:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v58:CPtr[CPtr(0x103b)] = Const CPtr(0x103b) + v59 = RefineType v58, CUInt64 + v60:CInt64 = IntAnd v47, v57 + v61:CBool = IsBitEqual v60, v59 + CondBranch v61, bb11(), bb12() + bb11(): + v63:NilClass = Const Value(nil) + Jump bb8(v63) + bb12(): + v97:CShape = LoadField v46, :shape_id@0x103c + v98:CShape[0x103d] = GuardBitEquals v97, CShape(0x103d) recompile + v99:BasicObject = LoadField v46, :@levar@0x103a + Jump bb8(v99) + bb8(v48:BasicObject): + CheckInterrupts + v69:CBool = Test v48 + CondBranch v69, bb5(v46, v41), bb13() + bb13(): + PatchPoint NoEPEscape(set_value_loop) + PatchPoint SingleRactorMode + v101:CShape = LoadField v46, :shape_id@0x103c + v102:CShape[0x103e] = GuardBitEquals v101, CShape(0x103e) recompile + StoreField v46, :@levar@0x103a, v41 + WriteBarrier v46, v41 + v105:CShape[0x103d] = Const CShape(0x103d) + StoreField v46, :shape_id@0x103c, v105 + v79:HeapBasicObject = RefineType v46, HeapBasicObject + Jump bb5(v79, v41) + bb5(v81:HeapBasicObject, v82:Fixnum): + PatchPoint NoEPEscape(set_value_loop) + v89:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x103f, cme:0x1040) + v114:Fixnum = FixnumAdd v82, v89 + Jump bb6(v81, v114) + bb7(): + v35:NilClass = Const Value(nil) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_float_nan_p_annotation() { + eval(r#" + def test(x) = x.nan? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, nan?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum + v24:BoolExact = CCall v23, :Float#nan?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_finite_p_annotation() { + eval(r#" + def test(x) = x.finite? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, finite?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum + v24:BoolExact = CCall v23, :Float#finite?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_infinite_p_annotation() { + eval(r#" + def test(x) = x.infinite? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, infinite?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum + v24:NilClass|Fixnum = CCall v23, :Float#infinite?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_integer_even_p_annotation() { + eval(r#" + def test(x) = x.even? + test(2) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, even?@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_integer_odd_p_annotation() { + eval(r#" + def test(x) = x.odd? + test(3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, odd?@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_zero_p_annotation() { + eval(r#" + def test(x) = x.zero? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, zero?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_positive_p_annotation() { + eval(r#" + def test(x) = x.positive? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, positive?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_negative_p_annotation() { + eval(r#" + def test(x) = x.negative? + test(-1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, negative?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + #[test] + fn test_float_add_inline() { + eval(r#" + def test(a, b) = a + b + test(1.0, 2.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, +@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatAdd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_mul_inline() { + eval(r#" + def test(a, b) = a * b + test(1.5, 2.5) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, *@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatMul v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_sub_inline() { + eval(r#" + def test(a, b) = a - b + test(5.0, 3.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, -@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatSub v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_div_inline() { + eval(r#" + def test(a, b) = a / b + test(10.0, 3.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, /@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatDiv v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_to_i_inline() { + eval(r#" + def test(a) = a.to_i + test(3.7) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, to_i@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum + v24:Integer = FloatToInt v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_mul_fixnum_inline() { + eval(r#" + def test(a, b) = a * b + test(1.5, 3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, *@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum + v29:Fixnum = GuardType v13, Fixnum + v30:Float = FloatMul v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_repeated_heap_object_guards() { + eval(r#" + C = Struct.new(:var) + def test(obj) + sum = 0 + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum + end + test(C.new(3)) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :obj@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, var@0x1010, cme:0x1018) + v138:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] + v139:BasicObject = LoadField v138, :var@0x1040 + PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) + v179:Fixnum = GuardType v139, Fixnum + v180:Fixnum = FixnumAdd v17, v179 + PatchPoint NoEPEscape(test) + v185:Fixnum = FixnumAdd v180, v179 + v190:Fixnum = FixnumAdd v185, v179 + v195:Fixnum = FixnumAdd v190, v179 + v200:Fixnum = FixnumAdd v195, v179 + v205:Fixnum = FixnumAdd v200, v179 + v210:Fixnum = FixnumAdd v205, v179 + v215:Fixnum = FixnumAdd v210, v179 + v220:Fixnum = FixnumAdd v215, v179 + v225:Fixnum = FixnumAdd v220, v179 + CheckInterrupts + Return v225 + "); + } + + #[test] + fn test_dont_fold_array_length() { + eval(r#" + A = [1, 2, 3, 4] + def test = A.length + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v21:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, length@0x1018, cme:0x1020) + v26:CInt64 = ArrayLength v21 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_frozen_array_length() { + eval(r#" + A = [1, 2, 3, 4].freeze + def test = A.length + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v21:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, length@0x1018, cme:0x1020) + v28:CInt64[4] = Const CInt64(4) + v27:Fixnum = BoxFixnum v28 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_elide_test_of_box_bool() { + eval(r#" + def test(a, b) + if a == b + 3 + else + 4 + end + end + test(:a, :b) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, ==@0x1010, cme:0x1018) + v48:StaticSymbol = GuardType v12, StaticSymbol + v49:CBool = IsBitEqual v48, v13 + v50:BoolExact = BoxBool v49 + CheckInterrupts + CondBranch v49, bb5(), bb4(v11, v48, v13) + bb5(): + v29:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v29 + bb4(v34:BasicObject, v35:StaticSymbol, v36:BasicObject): + v40:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v40 + "); + } +} diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs new file mode 100644 index 0000000000..7a1cd85c5f --- /dev/null +++ b/zjit/src/hir/tests.rs @@ -0,0 +1,6433 @@ +#[cfg(test)] +use super::*; + +#[cfg(test)] +mod snapshot_tests { + use super::*; + use insta::assert_snapshot; + + #[track_caller] + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[track_caller] + fn optimized_hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(&format!("{}.method(:{})", "self", method))); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[test] + fn test_remove_redundant_patch_points() { + eval(" + def test = 1 + 2 + 3 + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:2: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v13:Any = Snapshot FrameState { pc: 0x1008, stack: [v10, v12], locals: [] } + PatchPoint MethodRedefined(Integer@0x1010, +@0x1018, cme:0x1020) + v35:Fixnum[6] = Const Value(6) + v21:Any = Snapshot FrameState { pc: 0x1048, stack: [v35], locals: [] } + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [a=v12, b=v13] } + v15:Any = Snapshot FrameState { pc: 0x1010, stack: [], locals: [a=v12, b=v13] } + PatchPoint NoTracePoint + v17:Any = Snapshot FrameState { pc: 0x1018, stack: [v12], locals: [a=v12, b=v13] } + v18:Any = Snapshot FrameState { pc: 0x1020, stack: [v12, v13], locals: [a=v12, b=v13] } + v19:ArrayExact = NewArray v12, v13 + v20:Any = Snapshot FrameState { pc: 0x1028, stack: [v19], locals: [a=v12, b=v13] } + PatchPoint NoTracePoint + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_direct_with_reordered_kwargs_has_snapshot() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + v16:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15], locals: [] } + v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] + v26:BasicObject = SendDirect v25, 0x1048, :foo (0x1058), v13, v15, v11 + v18:Any = Snapshot FrameState { pc: 0x1060, stack: [v26], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_with_kwargs_in_order_has_snapshot() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] + v23:BasicObject = SendDirect v22, 0x1048, :foo (0x1058), v11, v13 + v16:Any = Snapshot FrameState { pc: 0x1060, stack: [v23], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_send_direct_with_many_kwargs_no_reorder_snapshot() { + eval(" + def foo(five, six, a:, b:, c:, d:, e:, f:) = [a, b, c, d, five, six, e, f] + def test = foo(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[5] = Const Value(5) + v13:Fixnum[6] = Const Value(6) + v15:Fixnum[4] = Const Value(4) + v17:Fixnum[3] = Const Value(3) + v19:Fixnum[1] = Const Value(1) + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[7] = Const Value(7) + v25:Fixnum[8] = Const Value(8) + v26:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15, v17, v19, v21, v23, v25], locals: [] } + v27:BasicObject = Send v6, :foo, v11, v13, v15, v17, v19, v21, v23, v25 # SendFallbackReason: Too many arguments for LIR + v28:Any = Snapshot FrameState { pc: 0x1010, stack: [v27], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v27 + "); + } +} + +#[cfg(test)] +pub(crate) mod hir_build_tests { + use super::*; + use crate::options::set_call_threshold; + use insta::assert_snapshot; + + fn iseq_contains_opcode(iseq: IseqPtr, expected_opcode: u32) -> bool { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + if opcode == expected_opcode { + return true; + } + insn_idx += insn_len(opcode as usize); + } + false + } + + #[track_caller] + pub fn assert_contains_opcode(method: &str, opcode: u32) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + + #[track_caller] + fn assert_contains_opcodes(method: &str, opcodes: &[u32]) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + for &opcode in opcodes { + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + } + + /// Combine multiple hir_string() results to match all of them at once, which allows + /// us to avoid running the set of zjit-test -> zjit-test-update multiple times. + #[macro_export] + macro_rules! hir_strings { + ($( $s:expr ),+ $(,)?) => {{ + vec![$( hir_string($s) ),+].join("\n") + }}; + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + pub fn assert_compile_fails(method: &str, reason: ParseError) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let result = iseq_to_hir(iseq); + assert!(result.is_err(), "Expected an error but successfully compiled to HIR: {}", FunctionPrinter::without_snapshot(&result.unwrap())); + assert_eq!(result.unwrap_err(), reason); + } + + #[test] + fn test_compile_optional() { + eval("def test(x=1) = 123"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:BasicObject, v18:BasicObject): + v21:Fixnum[1] = Const Value(1) + Jump bb5(v17, v21) + bb4(): + EntryPoint JIT(1) + v14:BasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :x@1 + Jump bb5(v14, v15) + bb5(v24:BasicObject, v25:BasicObject): + v29:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_putobject() { + eval("def test = 123"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_checkmatch_case() { + eval(r#" + def test(o) + case o + in Integer + 1 + else + 2 + end + end + test(1) + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + v18:BasicObject = GetConstantPath 0x1008 + v20:BasicObject = CheckMatch v10, v18, CASE + CheckInterrupts + v23:CBool = Test v20 + v24:Truthy = RefineType v20, Truthy + CondBranch v23, bb4(v9, v10, v14, v10), bb5() + bb4(v36:BasicObject, v37:BasicObject, v38:NilClass, v39:BasicObject): + v44:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v44 + bb5(): + v26:Falsy = RefineType v20, Falsy + v31:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_checkmatch_case_splat_array() { + eval(r#" + def test(o) + case o + when *[1, 2] + 1 + else + 2 + end + end + test(1) + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:ArrayExact = ArrayDup v16 + v19:BasicObject = CheckMatch v10, v17, CASE|ARRAY + CheckInterrupts + v22:CBool = Test v19 + v23:Truthy = RefineType v19, Truthy + CondBranch v22, bb4(v9, v10, v10), bb5() + bb4(v34:BasicObject, v35:BasicObject, v36:BasicObject): + v41:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v41 + bb5(): + v25:Falsy = RefineType v19, Falsy + v29:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_checkmatch_when_splat_array() { + eval(r#" + def test + case + when *[1, 2] + 1 + else + 2 + end + end + test + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:ArrayExact = ArrayDup v12 + v15:BasicObject = CheckMatch v10, v13, WHEN|ARRAY + CheckInterrupts + v18:CBool = Test v15 + v19:Truthy = RefineType v15, Truthy + CondBranch v18, bb4(v6), bb5() + bb4(v29:BasicObject): + v33:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v33 + bb5(): + v21:Falsy = RefineType v15, Falsy + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_new_array() { + eval("def test = []"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array_with_element() { + eval("def test(a) = [a]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = NewArray v10 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_new_range_inclusive_with_one_element() { + eval("def test(a) = (a..10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v17:RangeExact = NewRange v10 NewRangeInclusive v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_new_range_inclusive_with_two_elements() { + eval("def test(a, b) = (a..b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:RangeExact = NewRange v12 NewRangeInclusive v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_new_range_exclusive_with_one_element() { + eval("def test(a) = (a...10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v17:RangeExact = NewRange v10 NewRangeExclusive v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_new_range_exclusive_with_two_elements() { + eval("def test(a, b) = (a...b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:RangeExact = NewRange v12 NewRangeExclusive v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_array_dup() { + eval("def test = [1, 2, 3]"); + assert_contains_opcode("test", YARVINSN_duparray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_hash_dup() { + eval("def test = {a: 1, b: 2}"); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:HashExact = HashDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_new_hash_empty() { + eval("def test = {}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_hash_with_elements() { + eval("def test(aval, bval) = {a: aval, b: bval}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :aval@0x1000 + v4:BasicObject = LoadField v2, :bval@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :aval@1 + v9:BasicObject = LoadArg :bval@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v17:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + v20:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v23:HashExact = NewHash v17: v12, v20: v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_string_copy() { + eval("def test = \"hello\""); + assert_contains_opcode("test", YARVINSN_dupchilledstring); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_bignum() { + eval("def test = 999999999999999999999999999999999999"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_flonum() { + eval("def test = 1.5"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_heap_float() { + eval("def test = 1.7976931348623157e+308"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_static_sym() { + eval("def test = :foo"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_opt_plus() { + eval("def test = 1+2"); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v15:BasicObject = Send v10, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze; 5; end + end + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze; 5; end + end + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze; 5; end + end + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_uminus() { + eval(" + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@; 5; end + end + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS)) + "); + } + + #[test] + fn test_setlocal_getlocal() { + eval(" + def test + a = 1 + a + end + "); + assert_contains_opcodes("test", &[YARVINSN_getlocal_WC_0, YARVINSN_setlocal_WC_0]); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nested_setlocal_getlocal() { + eval(" + l3 = 3 + _unused = _unused1 = nil + 1.times do |l2| + _ = nil + l2 = 2 + 1.times do |l1| + l1 = 1 + define_method(:test) do + l1 = l2 + l2 = l1 + l2 + l3 = l2 + l3 + end + end + end + "); + assert_contains_opcodes( + "test", + &[YARVINSN_getlocal_WC_1, YARVINSN_setlocal_WC_1, + YARVINSN_getlocal, YARVINSN_setlocal]); + assert_snapshot!(hir_string("test"), @" + fn block (3 levels) in <compiled>@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:CPtr = GetEP 2 + v11:BasicObject = LoadField v10, :l2@0x1000 + SetLocal :l1, l1, EP@3, v11 + v16:CPtr = GetEP 1 + v17:BasicObject = LoadField v16, :l1@0x1001 + v19:CPtr = GetEP 2 + v20:BasicObject = LoadField v19, :l2@0x1000 + v23:BasicObject = Send v17, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l2, l2, EP@4, v23 + v28:CPtr = GetEP 2 + v29:BasicObject = LoadField v28, :l2@0x1000 + v31:CPtr = GetEP 3 + v32:BasicObject = LoadField v31, :l3@0x1002 + v35:BasicObject = Send v29, :+, v32 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l3, l3, EP@5, v35 + CheckInterrupts + Return v35 + " + ); + } + + #[test] + fn test_setlocal_in_default_args() { + eval(" + def test(a = (b = 1)) = [a, b] + "); + assert_contains_opcode("test", YARVINSN_setlocal_WC_0); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:NilClass): + v26:Fixnum[1] = Const Value(1) + Jump bb5(v20, v26, v26) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :a@1 + v18:NilClass = Const Value(nil) + Jump bb5(v16, v17, v18) + bb5(v31:BasicObject, v32:BasicObject, v33:NilClass|Fixnum): + v39:ArrayExact = NewArray v32, v33 + CheckInterrupts + Return v39 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_tracepoint() { + eval(" + def test(a = (b = 1)) = [a, b] + TracePoint.new(:line) {}.enable + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:NilClass): + SideExit UnhandledYARVInsn(trace_putobject_INT2FIX_1_) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :a@1 + v18:NilClass = Const Value(nil) + Jump bb5(v16, v17, v18) + bb5(v27:BasicObject, v28:BasicObject, v29:NilClass): + v35:ArrayExact = NewArray v28, v29 + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_side_exit() { + eval(" + def test(a = (def foo = nil)) = a + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:BasicObject, v18:BasicObject): + SideExit UnhandledYARVInsn(definemethod) + bb4(): + EntryPoint JIT(1) + v14:BasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :a@1 + Jump bb5(v14, v15) + bb5(v23:BasicObject, v24:BasicObject): + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_setlocal_cyclic_default_args() { + eval(" + def test = proc { |a=a| a } + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + Jump bb3(v6, v7) + bb4(): + EntryPoint JIT(1) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + Jump bb3(v10, v11) + bb3(v13:BasicObject, v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn defined_ivar() { + eval(" + def test = defined?(@foo) + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn if_defined_ivar() { + eval(" + def test + if defined?(@foo) + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:TrueClass|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + v13:CBool = Test v10 + v14:NilClass = RefineType v10, Falsy + CondBranch v13, bb5(), bb4(v6) + bb5(): + v16:TrueClass = RefineType v10, Truthy + v19:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v19 + bb4(v24:BasicObject): + v28:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn defined() { + eval(" + def test = return defined?(SeaChange), defined?(favourite), defined?($ruby) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined constant, v10 + v15:StringExact|NilClass = Defined func, v6 + v17:NilClass = Const Value(nil) + v19:StringExact|NilClass = Defined global-variable, v17 + v21:ArrayExact = NewArray v12, v15, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn defined_yield_in_method_local_iseq_returns_defined() { + eval(" + def test = defined?(yield) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined yield, v10 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn defined_yield_in_non_method_local_iseq_returns_nil() { + eval(" + define_method(:test) { defined?(yield) } + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_return_const() { + eval(" + def test(cond) + if cond + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_leave); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :cond@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :cond@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + bb4(v27:BasicObject, v28:Falsy): + v32:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_merge_const() { + eval(" + def test(cond) + if cond + result = 3 + else + result = 4 + end + result + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :cond@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :cond@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + CheckInterrupts + v19:CBool = Test v12 + v20:Falsy = RefineType v12, Falsy + CondBranch v19, bb6(), bb4(v11, v20, v13) + bb6(): + v22:Truthy = RefineType v12, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Jump bb5(v11, v22, v25) + bb4(v30:BasicObject, v31:Falsy, v32:NilClass): + v36:Fixnum[4] = Const Value(4) + Jump bb5(v30, v31, v36) + bb5(v39:BasicObject, v40:BasicObject, v41:Fixnum): + CheckInterrupts + Return v41 + "); + } + + #[test] + fn test_opt_plus_fixnum() { + eval(" + def test(a, b) = a + b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :+, v13 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_minus_fixnum() { + eval(" + def test(a, b) = a - b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_minus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :-, v13 # SendFallbackReason: Uncategorized(opt_minus) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_mult_fixnum() { + eval(" + def test(a, b) = a * b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :*, v13 # SendFallbackReason: Uncategorized(opt_mult) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_div_fixnum() { + eval(" + def test(a, b) = a / b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :/, v13 # SendFallbackReason: Uncategorized(opt_div) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_mod_fixnum() { + eval(" + def test(a, b) = a % b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :%, v13 # SendFallbackReason: Uncategorized(opt_mod) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_eq_fixnum() { + eval(" + def test(a, b) = a == b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :==, v13 # SendFallbackReason: Uncategorized(opt_eq) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_neq_fixnum() { + eval(" + def test(a, b) = a != b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :!=, v13 # SendFallbackReason: Uncategorized(opt_neq) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_lt_fixnum() { + eval(" + def test(a, b) = a < b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :<, v13 # SendFallbackReason: Uncategorized(opt_lt) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_le_fixnum() { + eval(" + def test(a, b) = a <= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :<=, v13 # SendFallbackReason: Uncategorized(opt_le) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_gt_fixnum() { + eval(" + def test(a, b) = a > b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :>, v13 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_loop() { + eval(" + def test + result = 0 + times = 10 + while times > 0 + result = result + 1 + times = times - 1 + end + result + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[0] = Const Value(0) + v20:Fixnum[10] = Const Value(10) + CheckInterrupts + Jump bb5(v10, v16, v20) + bb5(v26:BasicObject, v27:BasicObject, v28:BasicObject): + v32:Fixnum[0] = Const Value(0) + v35:BasicObject = Send v28, :>, v32 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + v38:CBool = Test v35 + v39:Truthy = RefineType v35, Truthy + CondBranch v38, bb4(v26, v27, v28), bb6() + bb4(v51:BasicObject, v52:BasicObject, v53:BasicObject): + v58:Fixnum[1] = Const Value(1) + v61:BasicObject = Send v52, :+, v58 # SendFallbackReason: Uncategorized(opt_plus) + v66:Fixnum[1] = Const Value(1) + v69:BasicObject = Send v53, :-, v66 # SendFallbackReason: Uncategorized(opt_minus) + Jump bb5(v51, v61, v69) + bb6(): + v41:Falsy = RefineType v35, Falsy + v43:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_opt_ge_fixnum() { + eval(" + def test(a, b) = a >= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :>=, v13 # SendFallbackReason: Uncategorized(opt_ge) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_display_types() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v19:CBool[true] = Test v13 + v20 = RefineType v13, Falsy + CondBranch v19, bb5(), bb4(v8, v20) + bb5(): + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v25 + bb4(v30, v31): + v35 = Const Value(4) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_send_without_block() { + eval(" + def bar(a, b) + a+b + end + def test + bar(2, 3) + end + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + v13:Fixnum[3] = Const Value(3) + v15:BasicObject = Send v6, :bar, v11, v13 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_send_with_block() { + eval(" + def test(a) + a.each {|item| + item + } + end + test([1,2,3]) + "); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:BasicObject = Send v10, 0x1008, :each # SendFallbackReason: Uncategorized(send) + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :a@0x1000 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_intern_interpolated_symbol() { + eval(r#" + def test + :"foo#{123}" + end + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[123] = Const Value(123) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + v21:Symbol = StringIntern v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn different_objects_get_addresses() { + eval("def test = unknown_method([0], [1], '2', '2')"); + + // The 2 string literals have the same address because they're deduped. + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:ArrayExact = ArrayDup v11 + v14:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:ArrayExact = ArrayDup v14 + v17:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v18:StringExact = StringCopy v17 + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + v23:BasicObject = Send v6, :unknown_method, v12, v15, v18, v21 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_cant_compile_splat() { + eval(" + def test(a) = foo(*a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v18:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_compile_block_arg() { + eval(" + def test(a) = foo(&a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v9, &block, :foo, v10 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_cant_compile_kwarg() { + eval(" + def test(a) = foo(a: 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:BasicObject = Send v9, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_cant_compile_kw_splat() { + eval(" + def test(a) = foo(**a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v9, :foo, v10 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v16 + "); + } + + // TODO(max): Figure out how to generate a call with TAILCALL flag + + #[test] + fn test_compile_super() { + eval(" + def test = super() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_compile_zsuper() { + eval(" + def test = super + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_cant_compile_super_nil_blockarg() { + eval(" + def test = super(&nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:NilClass = Const Value(nil) + v13:BasicObject = InvokeSuper v6, 0x1000, v11 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_compile_super_forward() { + eval(" + def test(...) = super(...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_super_forward_with_block() { + eval(" + def test(...) = super { |x| x } + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + PatchPoint NoEPEscape(test) + v19:CPtr = LoadSP + v20:BasicObject = LoadField v19, :...@0x1000 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_super_forward_with_use() { + eval(" + def test(...) = super(...) + 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + v18:Fixnum[1] = Const Value(1) + v21:BasicObject = Send v16, :+, v18 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_compile_super_forward_with_arg() { + eval(" + def test(...) = super(1, ...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_compile_forwardable() { + eval("def forwardable(...) = nil"); + assert_snapshot!(hir_string("forwardable"), @" + fn forwardable@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + CheckInterrupts + Return v14 + "); + } + + // TODO(max): Figure out how to generate a call with OPT_SEND flag + + #[test] + fn test_cant_compile_kw_splat_mut() { + eval(" + def test(a) = foo **a, b: 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + v17:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:BasicObject = Send v15, :core#hash_merge_kwd, v17, v10 # SendFallbackReason: Uncategorized(opt_send_without_block) + v24:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + v27:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v29:Fixnum[1] = Const Value(1) + v31:BasicObject = Send v24, :core#hash_merge_ptr, v22, v27, v29 # SendFallbackReason: Uncategorized(opt_send_without_block) + v33:BasicObject = Send v9, :foo, v31 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_cant_compile_splat_mut() { + eval(" + def test(*) = foo *, 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :*@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :*@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToNewArray v10 + v18:Fixnum[1] = Const Value(1) + v20:CUInt64 = LoadField v16, :RBASIC_FLAGS@0x1001 + v21:CUInt64 = GuardNoBitsSet v20, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v16, v18 + v24:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_compile_forwarding() { + eval(" + def test(...) = foo(...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: SendForward: not yet specialized + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_triple_dots_with_positional_args() { + eval(" + def test(a, ...) = foo(a, ...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + SideExit SplatKwNotProfiled + "); + } + + #[test] + fn test_opt_new() { + eval(" + class C; end + def test = C.new + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:NilClass = Const Value(nil) + v15:CBool = IsMethodCFunc v10, :new + CondBranch v15, bb6(), bb4(v6, v12, v10) + bb6(): + v17:HeapBasicObject = ObjectAlloc v10 + v19:BasicObject = Send v17, :initialize # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Jump bb5(v6, v17, v19) + bb4(v23:BasicObject, v24:NilClass, v25:BasicObject): + v28:BasicObject = Send v25, :new # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb5(v23, v28, v24) + bb5(v31:BasicObject, v32:BasicObject, v33:BasicObject): + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_newarray_send_max_no_elements() { + eval(" + def test = [].max + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v11:BasicObject = ArrayMax + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v20:BasicObject = ArrayMax v12, v13 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX)) + "); + } + + #[test] + fn test_opt_newarray_send_min_no_elements() { + eval(" + def test = [].min + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN) + v11:BasicObject = ArrayMin + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_min() { + eval(" + def test(a,b) = [a,b].min + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN) + v20:BasicObject = ArrayMin v12, v13 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_newarray_send_min_redefined() { + eval(" + class Array + alias_method :old_min, :min + def min + old_min * 2 + end + end + + def test(a,b) = [a,b].min + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN)) + "); + } + + #[test] + fn test_opt_newarray_send_hash() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH) + v33:Fixnum = ArrayHash v16, v17 + PatchPoint NoEPEscape(test) + v40:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v41:ArrayExact = ArrayDup v40 + v43:BasicObject = Send v15, :puts, v41 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH)) + "); + } + + #[test] + fn test_opt_newarray_send_pack() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v32:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v33:StringExact = StringCopy v32 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v36:String = ArrayPackBuffer v16, v17, fmt: v33 + PatchPoint NoEPEscape(test) + v43:ArrayExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v44:ArrayExact = ArrayDup v43 + v46:BasicObject = Send v15, :puts, v44 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_opt_newarray_send_pack_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v32:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v33:StringExact = StringCopy v32 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v30:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v31:StringExact = StringCopy v30 + v37:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v38:StringExact = StringCopy v37 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v42:String = ArrayPackBuffer v16, v17, fmt: v38, buf: v31 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v30:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v31:StringExact = StringCopy v30 + v37:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v38:StringExact = StringCopy v37 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_include_p() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v34:BoolExact = ArrayInclude v16, v17 | v17 + PatchPoint NoEPEscape(test) + v41:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v42:ArrayExact = ArrayDup v41 + v44:BasicObject = Send v15, :puts, v42 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v16:BoolExact = DupArrayInclude VALUE(0x1008) | v10 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + v22:BasicObject = Send v19, :length # SendFallbackReason: Uncategorized(opt_length) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + v22:BasicObject = Send v19, :size # SendFallbackReason: Uncategorized(opt_size) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_getconstant() { + eval(" + def test(klass) + klass::ARGV + end + "); + assert_contains_opcode("test", YARVINSN_getconstant); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :klass@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :klass@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:FalseClass = Const Value(false) + v17:BasicObject = GetConstant v10, :ARGV, v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + test + "); + assert_contains_opcode("test", YARVINSN_getinstancevariable); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setinstancevariable); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + v15:HeapBasicObject = RefineType v6, HeapBasicObject + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_set_ivar_rescue_frozen() { + let result = eval(" + class Foo + attr_accessor :bar + def initialize + @bar = 1 + freeze + end + end + + def test(foo) + begin + foo.bar = 2 + rescue FrozenError + end + end + + foo = Foo.new + test(foo) + test(foo) + + foo.bar + "); + assert_eq!(VALUE::fixnum_from_usize(1), result); + } + + #[test] + fn test_getclassvariable() { + eval(" + class Foo + def self.test = @@foo + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_getclassvariable), "iseq Foo.test does not contain getclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetClassVar :@@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setclassvariable() { + eval(" + class Foo + def self.test = @@foo = 42 + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_setclassvariable), "iseq Foo.test does not contain setclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[42] = Const Value(42) + SetClassVar :@@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setglobal() { + eval(" + def test = $foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + SetGlobal :$foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getglobal() { + eval(" + def test = $foo + test + "); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetGlobal :$foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CPtr = GetEP 0 + v16:CUInt64 = LoadField v15, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v17:CBool = IsBlockParamModified v16 + CondBranch v17, bb4(), bb5() + bb4(): + v19:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v19) + bb5(): + v21:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v21) + bb6(v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:CPtr = GetEP 0 + v18:CUInt64 = LoadField v17, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v19:CBool = IsBlockParamModified v18 + CondBranch v19, bb4(), bb5() + bb4(): + v21:BasicObject = LoadField v17, :block@0x1002 + Jump bb6(v21, v21) + bb5(): + v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) + v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v25, v10) + bb6(v15:BasicObject, v16:BasicObject): + v28:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + tap(&block) + end + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :block@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22) + bb5(): + v24:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v24) + bb6(v17:BasicObject): + v32:CPtr = GetEP 0 + v33:CUInt64 = LoadField v32, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v34:CBool = IsBlockParamModified v33 + CondBranch v34, bb7(), bb8() + bb7(): + v36:BasicObject = LoadField v32, :block@0x1002 + Jump bb9(v36, v36) + bb8(): + v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) + v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v40, v17) + bb9(v30:BasicObject, v31:BasicObject): + v43:BasicObject = Send v11, &block, :tap, v30 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + tap(&block) + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v14:CPtr = GetEP 1 + v15:CUInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CBool = IsBlockParamModified v15 + CondBranch v16, bb4(), bb5() + bb4(): + v18:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v18) + bb5(): + v20:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v20) + bb6(v13:BasicObject): + v27:CPtr = GetEP 1 + v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v29:CBool = IsBlockParamModified v28 + CondBranch v29, bb7(), bb8() + bb7(): + v31:BasicObject = LoadField v27, :block@0x1001 + Jump bb9(v31) + bb8(): + v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) + v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v35) + bb9(v26:BasicObject): + v38:BasicObject = Send v8, &block, :tap, v26 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v38 + "); + } + + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb9() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb9(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb8(), bb10() + bb8(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb6(v16:BasicObject, v17:BasicObject): + v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v38 + bb10(): + SideExit BlockParamProxyProfileNotCovered + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:CPtr = GetEP 1 + v12:CUInt64 = LoadField v11, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v13:CBool = IsBlockParamModified v12 + CondBranch v13, bb4(), bb5() + bb4(): + v15:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v15) + bb5(): + v17:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v17) + bb6(v10:BasicObject): + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setblockparam() { + eval(" + def test(&block) + block = nil + end + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + SetLocal :block, l0, EP@3, v14 + v18:CPtr = GetEP 0 + v19:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CInt64[512] = Const CInt64(512) + v21:CInt64 = IntOr v19, v20 + StoreField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001, v21 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = nil + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + SetLocal :block, l1, EP@3, v10 + v14:CPtr = GetEP 1 + v15:CInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CInt64[512] = Const CInt64(512) + v17:CInt64 = IntOr v15, v16 + StoreField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000, v17 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_splatkw_unprofiled_side_exits() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + SideExit SplatKwNotProfiled + "); + } + + #[test] + fn test_splatkw_nil_guards_nil() { + eval(" + def foo(a, ...) = a + def test(a, ...) = foo(a, ...) + test(1) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64[0] = GuardBitEquals v42, CInt64(0) + v44:NilClass = Const Value(nil) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + v47:NilClass = GuardType v20, NilClass + v49:BasicObject = Send v17, &block, :foo, v18, v29, v47, v34 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_splatkw_empty_hash_guards_hash() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + test(&proc {}) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + v32:HashExact = GuardType v12, HashExact + v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_splatkw_hash_guards_hash() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + test(a: 1, &proc {}) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + v32:HashExact = GuardType v12, HashExact + v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_splatkw_polymorphic_side_exits() { + set_call_threshold(3); + eval(" + def foo(a, ...) = a + def test(a, ...) = foo(a, ...) + test(1) + test(1, b: 2) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64[0] = GuardBitEquals v42, CInt64(0) + v44:NilClass = Const Value(nil) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + SideExit SplatKwPolymorphic + "); + } + + #[test] + fn test_splatkw_with_non_hash_side_exits() { + eval(" + def foo(a:) = a + def test(obj, &block) = foo(**obj, &block) + obj = Object.new + def obj.to_hash = { a: 1 } + test(obj) { 2 } + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + v4:BasicObject = LoadField v2, :block@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :obj@1 + v9:BasicObject = LoadArg :block@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :block@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + SideExit SplatKwNotNilOrHash + "); + } + + #[test] + fn test_splatarray_mut() { + eval(" + def test(a) = [*a] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_concattoarray() { + eval(" + def test(a) = [1, *a] + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:ArrayExact = NewArray v14 + v19:ArrayExact = ToArray v10 + ArrayExtend v16, v19 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_pushtoarray_one_element() { + eval(" + def test(a) = [*a, 1] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + v17:Fixnum[1] = Const Value(1) + v19:CUInt64 = LoadField v15, :RBASIC_FLAGS@0x1001 + v20:CUInt64 = GuardNoBitsSet v19, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v15, v17 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_pushtoarray_multiple_elements() { + eval(" + def test(a) = [*a, 1, 2, 3] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[2] = Const Value(2) + v21:Fixnum[3] = Const Value(3) + v23:CUInt64 = LoadField v15, :RBASIC_FLAGS@0x1001 + v24:CUInt64 = GuardNoBitsSet v23, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v15, v17 + ArrayPush v15, v19 + ArrayPush v15, v21 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_aset() { + eval(" + def test(a, b) = a[b] = 1 + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v17:NilClass = Const Value(nil) + v21:Fixnum[1] = Const Value(1) + v25:BasicObject = Send v12, :[]=, v13, v21 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_aref() { + eval(" + def test(a, b) = a[b] + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :[], v13 # SendFallbackReason: Uncategorized(opt_aref) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :empty? # SendFallbackReason: Uncategorized(opt_empty_p) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_succ() { + eval(" + def test(x) = x.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :succ # SendFallbackReason: Uncategorized(opt_succ) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :&, v13 # SendFallbackReason: Uncategorized(opt_and) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :|, v13 # SendFallbackReason: Uncategorized(opt_or) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_not() { + eval(" + def test(x) = !x + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :! # SendFallbackReason: Uncategorized(opt_not) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_regexpmatch2() { + eval(" + def test(regexp, matchee) = regexp =~ matchee + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :regexp@0x1000 + v4:BasicObject = LoadField v2, :matchee@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :regexp@1 + v9:BasicObject = LoadArg :matchee@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :=~, v13 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + CheckInterrupts + Return v20 + "); + } + + #[test] + // Tests for ConstBase requires either constant or class definition, both + // of which can't be performed inside a method. + fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias aliased __callee__ + end + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1000)) + v12:BasicObject = PutSpecialObject CBase + v14:StaticSymbol[:aliased] = Const Value(VALUE(0x1008)) + v16:StaticSymbol[:__callee__] = Const Value(VALUE(0x1010)) + v18:BasicObject = Send v10, :core#set_method_alias, v12, v14, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn opt_reverse() { + eval(" + def reverse_odd + a, b, c = @a, @b, @c + [a, b, c] + end + + def reverse_even + a, b, c, d = @a, @b, @c, @d + [a, b, c, d] + end + "); + assert_contains_opcode("reverse_odd", YARVINSN_opt_reverse); + assert_contains_opcode("reverse_even", YARVINSN_opt_reverse); + assert_snapshot!(hir_strings!("reverse_odd", "reverse_even"), @" + fn reverse_odd@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:NilClass = Const Value(nil) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9, v10) + bb3(v12:BasicObject, v13:NilClass, v14:NilClass, v15:NilClass): + PatchPoint SingleRactorMode + v20:BasicObject = GetIvar v12, :@a + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v12, :@b + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v12, :@c + PatchPoint NoEPEscape(reverse_odd) + v38:ArrayExact = NewArray v20, v23, v26 + CheckInterrupts + Return v38 + + fn reverse_even@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11, v12) + bb3(v14:BasicObject, v15:NilClass, v16:NilClass, v17:NilClass, v18:NilClass): + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v14, :@a + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v14, :@b + PatchPoint SingleRactorMode + v29:BasicObject = GetIvar v14, :@c + PatchPoint SingleRactorMode + v32:BasicObject = GetIvar v14, :@d + PatchPoint NoEPEscape(reverse_even) + v46:ArrayExact = NewArray v23, v26, v29, v32 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_branchnil() { + eval(" + def test(x) = x&.itself + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v17:CBool = IsNil v10 + v18:NilClass = Const Value(nil) + CondBranch v17, bb4(v9, v18, v18), bb5() + bb5(): + v20:NotNil = RefineType v10, NotNil + v22:BasicObject = Send v20, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb4(v9, v20, v22) + bb4(v24:BasicObject, v25:BasicObject, v26:BasicObject): + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_infer_nilability_from_branchif() { + eval(" + def test(x) + if x + x&.itself + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_branchnil); + // Note that IsNil has as its operand a value that we know statically *cannot* be nil + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb6(), bb4(v9, v17) + bb6(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v25:CBool[false] = IsNil v19 + v26:NilClass = Const Value(nil) + CondBranch v25, bb5(v9, v26, v26), bb7() + bb7(): + v28:Truthy = RefineType v19, NotNil + v30:BasicObject = Send v28, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v30 + bb4(v35:BasicObject, v36:Falsy): + v40:Fixnum[4] = Const Value(4) + Jump bb5(v35, v36, v40) + bb5(v42:BasicObject, v43:Falsy, v44:Fixnum[4]): + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb7(), bb6(v9, v17) + bb7(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v24:CBool[true] = Test v19 + v25 = RefineType v19, Falsy + CondBranch v24, bb8(), bb5(v9, v25) + bb8(): + v27:Truthy = RefineType v19, Truthy + CheckInterrupts + v32:CBool[true] = Test v27 + v33 = RefineType v27, Falsy + CondBranch v32, bb9(), bb4(v9, v33) + bb9(): + v35:Truthy = RefineType v27, Truthy + v38:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v38 + bb4(v63, v64): + v68 = Const Value(4) + CheckInterrupts + Return v68 + bb5(v53, v54): + v58 = Const Value(5) + CheckInterrupts + Return v58 + bb6(v43:BasicObject, v44:Falsy): + v48:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v48 + "); + } + + #[test] + fn test_invokebuiltin_delegate_annotated() { + assert_contains_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("Float"), @" + fn Float@<internal:kernel>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arg@0x1000 + v4:BasicObject = LoadField v2, :exception@0x1001 + v5:BasicObject = LoadField v2, :<empty>@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arg@1 + v10:BasicObject = LoadArg :exception@2 + v11:CPtr = GetEP 0 + v12:BasicObject = LoadField v11, :<empty>@0x1003 + Jump bb3(v8, v9, v10, v12) + bb3(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject): + v21:Float = InvokeBuiltin rb_f_float, v14, v15, v16 + Jump bb4(v14, v15, v16, v17, v21) + bb4(v23:BasicObject, v24:BasicObject, v25:BasicObject, v26:BasicObject, v27:Float): + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invokebuiltin_cexpr_annotated() { + assert_contains_opcode("class", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("class"), @" + fn class@<internal:kernel>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Class = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:Class): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_delegate_with_args() { + // Using an unannotated builtin to test InvokeBuiltin generation + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn open@<internal:dir>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :name@0x1000 + v4:BasicObject = LoadField v2, :encoding@0x1001 + v5:BasicObject = LoadField v2, :<empty>@0x1002 + v6:BasicObject = LoadField v2, :block@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :name@1 + v12:BasicObject = LoadArg :encoding@2 + v13:CPtr = GetEP 0 + v14:BasicObject = LoadField v13, :<empty>@0x1003 + v15:BasicObject = LoadArg :block@3 + v16:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v14, v15, v16) + bb3(v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:BasicObject, v23:NilClass): + v27:BasicObject = InvokeBuiltin dir_s_open, v18, v19, v20 + PatchPoint NoEPEscape(open) + v35:CPtr = GetEP 0 + v36:CUInt64 = LoadField v35, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v37:CBool = IsBlockParamModified v36 + CondBranch v37, bb5(), bb6() + bb5(): + v39:BasicObject = LoadField v35, :block@0x1005 + Jump bb7(v39, v39) + bb6(): + v41:CInt64 = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) + v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb7(v43, v22) + bb7(v33:BasicObject, v34:BasicObject): + CheckInterrupts + v47:CBool = Test v33 + v48:Falsy = RefineType v33, Falsy + CondBranch v47, bb8(), bb4(v18, v19, v20, v21, v34, v27) + bb8(): + v50:Truthy = RefineType v33, Truthy + v54:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + v57:BasicObject = InvokeBuiltin dir_s_close, v18, v27 + CheckInterrupts + Return v54 + bb4(v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject): + CheckInterrupts + Return v68 + "); + } + + #[test] + fn test_invokebuiltin_delegate_without_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "enable")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate_leave), "iseq GC.enable does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn enable@<internal:gc>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = InvokeBuiltin gc_enable, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_with_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "start")); + assert!(iseq_contains_opcode(iseq, YARVINSN_invokebuiltin), "iseq GC.start does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn start@<internal:gc>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :full_mark@0x1000 + v4:BasicObject = LoadField v2, :immediate_mark@0x1001 + v5:BasicObject = LoadField v2, :immediate_sweep@0x1002 + v6:BasicObject = LoadField v2, :<empty>@0x1003 + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :full_mark@1 + v11:BasicObject = LoadArg :immediate_mark@2 + v12:BasicObject = LoadArg :immediate_sweep@3 + v13:CPtr = GetEP 0 + v14:BasicObject = LoadField v13, :<empty>@0x1004 + Jump bb3(v9, v10, v11, v12, v14) + bb3(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject): + v27:FalseClass = Const Value(false) + v29:BasicObject = InvokeBuiltin gc_start_internal, v16, v17, v18, v19, v27 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_name() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "name")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn name@<internal:symbol>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_to_s() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "to_s")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn to_s@<internal:symbol>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dupn() { + eval(" + def test(x) = (x[0, 1] ||= 2) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + v17:Fixnum[0] = Const Value(0) + v19:Fixnum[1] = Const Value(1) + v22:BasicObject = Send v10, :[], v17, v19 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + v26:CBool = Test v22 + v27:Truthy = RefineType v22, Truthy + CondBranch v26, bb4(v9, v10, v14, v10, v17, v19, v27), bb5() + bb4(v41:BasicObject, v42:BasicObject, v43:NilClass, v44:BasicObject, v45:Fixnum[0], v46:Fixnum[1], v47:Truthy): + CheckInterrupts + Return v47 + bb5(): + v29:Falsy = RefineType v22, Falsy + v32:Fixnum[2] = Const Value(2) + v35:BasicObject = Send v10, :[]=, v17, v19, v32 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_objtostring_anytostring() { + eval(" + def test = \"#{1}\" + "); + assert_contains_opcode("test", YARVINSN_objtostring); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:StringExact = StringConcat v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:NilClass = Const Value(nil) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_toregexp() { + eval(r##" + def test = /#{1}#{2}#{3}/ + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:RegexpExact = ToRegexp v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_toregexp_with_options() { + eval(r##" + def test = /#{1}#{2}/mixn + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:RegexpExact = ToRegexp v15, v22, MULTILINE|IGNORECASE|EXTENDED|NOENCODING + CheckInterrupts + Return v24 + "); + } + + #[test] + fn throw() { + eval(" + define_method(:throw_return) { return 1 } + define_method(:throw_break) { break 2 } + "); + assert_contains_opcode("throw_return", YARVINSN_throw); + assert_contains_opcode("throw_break", YARVINSN_throw); + assert_snapshot!(hir_strings!("throw_return", "throw_break"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v12:Fixnum[1] = Const Value(1) + Throw TAG_RETURN, v12 + + fn block in <compiled>@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v12:Fixnum[2] = Const Value(2) + Throw TAG_BREAK, v12 + "); + } + + #[test] + fn test_invokeblock() { + eval(r#" + def test + yield + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = InvokeBlock # SendFallbackReason: InvokeBlock: not yet specialized + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_invokeblock_with_args() { + eval(r#" + def test(x, y) + yield x, y + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: InvokeBlock: not yet specialized + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_expandarray_no_splat() { + eval(r#" + def test(o) + a, b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :o@1 + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:NilClass, v16:NilClass): + v22:ArrayExact = GuardType v14, ArrayExact + v23:CInt64 = ArrayLength v22 + v24:CInt64[2] = Const CInt64(2) + v25:CInt64 = GuardGreaterEq v23, v24 + v26:CInt64[1] = Const CInt64(1) + v27:BasicObject = ArrayAref v22, v26 + v28:CInt64[0] = Const CInt64(0) + v29:BasicObject = ArrayAref v22, v28 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_expandarray_splat() { + eval(r#" + def test(o) + a, *b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :o@1 + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:NilClass, v16:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_expandarray_splat_post() { + eval(r#" + def test(o) + a, *b, c = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :o@1 + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass, v19:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_checkkeyword_tests_fixnum_bit() { + eval(r#" + def test(kw: 1 + 1) = kw + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :<empty>@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:CPtr = GetEP 0 + v10:BasicObject = LoadField v9, :<empty>@0x1002 + Jump bb3(v7, v8, v10) + bb3(v12:BasicObject, v13:BasicObject, v14:BasicObject): + v17:BoolExact = FixnumBitCheck v14, 0 + CheckInterrupts + v20:CBool = Test v17 + v21:TrueClass = RefineType v17, Truthy + CondBranch v20, bb4(v12, v13, v14), bb5() + bb5(): + v23:FalseClass = RefineType v17, Falsy + v25:Fixnum[1] = Const Value(1) + v27:Fixnum[1] = Const Value(1) + v30:BasicObject = Send v25, :+, v27 # SendFallbackReason: Uncategorized(opt_plus) + Jump bb4(v12, v30, v14) + bb4(v33:BasicObject, v34:BasicObject, v35:BasicObject): + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_checkkeyword_too_many_keywords_causes_side_exit() { + eval(r#" + def test(k1: k1, k2: k2, k3: k3, k4: k4, k5: k5, + k6: k6, k7: k7, k8: k8, k9: k9, k10: k10, k11: k11, + k12: k12, k13: k13, k14: k14, k15: k15, k16: k16, + k17: k17, k18: k18, k19: k19, k20: k20, k21: k21, + k22: k22, k23: k23, k24: k24, k25: k25, k26: k26, + k27: k27, k28: k28, k29: k29, k30: k30, k31: k31, + k32: k32, k33: k33) = k1 + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k1@0x1000 + v4:BasicObject = LoadField v2, :k2@0x1001 + v5:BasicObject = LoadField v2, :k3@0x1002 + v6:BasicObject = LoadField v2, :k4@0x1003 + v7:BasicObject = LoadField v2, :k5@0x1004 + v8:BasicObject = LoadField v2, :k6@0x1005 + v9:BasicObject = LoadField v2, :k7@0x1006 + v10:BasicObject = LoadField v2, :k8@0x1007 + v11:BasicObject = LoadField v2, :k9@0x1008 + v12:BasicObject = LoadField v2, :k10@0x1009 + v13:BasicObject = LoadField v2, :k11@0x100a + v14:BasicObject = LoadField v2, :k12@0x100b + v15:BasicObject = LoadField v2, :k13@0x100c + v16:BasicObject = LoadField v2, :k14@0x100d + v17:BasicObject = LoadField v2, :k15@0x100e + v18:BasicObject = LoadField v2, :k16@0x100f + v19:BasicObject = LoadField v2, :k17@0x1010 + v20:BasicObject = LoadField v2, :k18@0x1011 + v21:BasicObject = LoadField v2, :k19@0x1012 + v22:BasicObject = LoadField v2, :k20@0x1013 + v23:BasicObject = LoadField v2, :k21@0x1014 + v24:BasicObject = LoadField v2, :k22@0x1015 + v25:BasicObject = LoadField v2, :k23@0x1016 + v26:BasicObject = LoadField v2, :k24@0x1017 + v27:BasicObject = LoadField v2, :k25@0x1018 + v28:BasicObject = LoadField v2, :k26@0x1019 + v29:BasicObject = LoadField v2, :k27@0x101a + v30:BasicObject = LoadField v2, :k28@0x101b + v31:BasicObject = LoadField v2, :k29@0x101c + v32:BasicObject = LoadField v2, :k30@0x101d + v33:BasicObject = LoadField v2, :k31@0x101e + v34:BasicObject = LoadField v2, :k32@0x101f + v35:BasicObject = LoadField v2, :k33@0x1020 + v36:BasicObject = LoadField v2, :<empty>@0x1021 + Jump bb3(v1, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) + bb2(): + EntryPoint JIT(0) + v39:BasicObject = LoadArg :self@0 + v40:BasicObject = LoadArg :k1@1 + v41:BasicObject = LoadArg :k2@2 + v42:BasicObject = LoadArg :k3@3 + v43:BasicObject = LoadArg :k4@4 + v44:BasicObject = LoadArg :k5@5 + v45:BasicObject = LoadArg :k6@6 + v46:BasicObject = LoadArg :k7@7 + v47:BasicObject = LoadArg :k8@8 + v48:BasicObject = LoadArg :k9@9 + v49:BasicObject = LoadArg :k10@10 + v50:BasicObject = LoadArg :k11@11 + v51:BasicObject = LoadArg :k12@12 + v52:BasicObject = LoadArg :k13@13 + v53:BasicObject = LoadArg :k14@14 + v54:BasicObject = LoadArg :k15@15 + v55:BasicObject = LoadArg :k16@16 + v56:BasicObject = LoadArg :k17@17 + v57:BasicObject = LoadArg :k18@18 + v58:BasicObject = LoadArg :k19@19 + v59:BasicObject = LoadArg :k20@20 + v60:BasicObject = LoadArg :k21@21 + v61:BasicObject = LoadArg :k22@22 + v62:BasicObject = LoadArg :k23@23 + v63:BasicObject = LoadArg :k24@24 + v64:BasicObject = LoadArg :k25@25 + v65:BasicObject = LoadArg :k26@26 + v66:BasicObject = LoadArg :k27@27 + v67:BasicObject = LoadArg :k28@28 + v68:BasicObject = LoadArg :k29@29 + v69:BasicObject = LoadArg :k30@30 + v70:BasicObject = LoadArg :k31@31 + v71:BasicObject = LoadArg :k32@32 + v72:BasicObject = LoadArg :k33@33 + v73:CPtr = GetEP 0 + v74:BasicObject = LoadField v73, :<empty>@0x1022 + Jump bb3(v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63, v64, v65, v66, v67, v68, v69, v70, v71, v72, v74) + bb3(v76:BasicObject, v77:BasicObject, v78:BasicObject, v79:BasicObject, v80:BasicObject, v81:BasicObject, v82:BasicObject, v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject, v91:BasicObject, v92:BasicObject, v93:BasicObject, v94:BasicObject, v95:BasicObject, v96:BasicObject, v97:BasicObject, v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject, v102:BasicObject, v103:BasicObject, v104:BasicObject, v105:BasicObject, v106:BasicObject, v107:BasicObject, v108:BasicObject, v109:BasicObject, v110:BasicObject): + SideExit TooManyKeywordParameters + "); + } + + #[test] + fn test_array_each() { + assert_snapshot!(hir_string_proc("Array.instance_method(:each)"), @" + fn each@<internal:array>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + v15:TrueClass|NilClass = Defined yield, v13 + v17:CBool = Test v15 + v18:NilClass = RefineType v15, Falsy + CondBranch v17, bb9(), bb4(v8, v9) + bb9(): + v20:TrueClass = RefineType v15, Truthy + Jump bb6(v8, v9) + bb6(v30:BasicObject, v31:NilClass): + v35:Fixnum[0] = Const Value(0) + Jump bb8(v30, v35) + bb8(v48:BasicObject, v49:Fixnum): + v52:BoolExact = InvokeBuiltin rb_jit_ary_at_end, v48, v49 + v54:CBool = Test v52 + v55:FalseClass = RefineType v52, Falsy + CondBranch v54, bb10(), bb7(v48, v49) + bb10(): + v57:TrueClass = RefineType v52, Truthy + v59:NilClass = Const Value(nil) + CheckInterrupts + Return v48 + bb7(v67:BasicObject, v68:Fixnum): + v72:BasicObject = InvokeBuiltin rb_jit_ary_at, v67, v68 + v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: InvokeBlock: not yet specialized + v78:Fixnum = InvokeBuiltin rb_jit_fixnum_inc, v67, v68 + PatchPoint NoEPEscape(each) + Jump bb8(v67, v78) + bb4(v23:BasicObject, v24:NilClass): + v28:BasicObject = InvokeBuiltin <inline_expr>, v23 + Jump bb5(v23, v24, v28) + bb5(v40:BasicObject, v41:NilClass, v42:BasicObject): + CheckInterrupts + Return v42 + "); + } + + #[test] + fn test_induce_side_exit() { + eval(" + class NonTopLexicalScope + RubyVM = 0 + def test + RubyVM::ZJIT.induce_side_exit! # lexical scope dependant -- should not recognize + ::RubyVM::ZJIT.induce_side_exit! + end + end + "); + assert_snapshot!(hir_string_proc("NonTopLexicalScope.instance_method(:test)"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_side_exit! # SendFallbackReason: Uncategorized(opt_send_without_block) + v16:BasicObject = GetConstantPath 0x1000 + SideExit DirectiveInduced + "); + } + + #[test] + fn test_induce_side_exit_sensitive_to_constant_state() { + eval(" + def test = ::RubyVM::ZJIT.induce_side_exit! + "); + assert!(hir_string("test").contains("SideExit DirectiveInduced")); + eval(" + class RubyVM + remove_const(:ZJIT) + end + "); + let hir_after_removal = hir_string("test"); + assert_eq!(false, hir_string("test").contains("SideExit DirectiveInduced"), "should not work when the constant lookup would fail"); + assert_snapshot!(hir_after_removal, @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_side_exit! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_side_exit_doesnt_work_when_method_after_undef() { + eval(" + class << RubyVM::ZJIT + undef :induce_side_exit! + end + def test = ::RubyVM::ZJIT.induce_side_exit! + "); + assert_eq!(false, hir_string("test").contains("SideExit DirectiveInduced"), "should not work after undef"); + } + + #[test] + fn test_induce_compile_failure_does_not_trigger_autoload() { + eval(" + class RubyVM + remove_const(:ZJIT) + autoload :ZJIT, 'a-file-that-does-not-exist-as-a-trap' + end + def test = ::RubyVM::ZJIT.induce_compile_failure! + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_compile_failure! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_compile_failure_checks_full_const_path() { + eval("def test = ::RubyVM::ZJIT::TooDeep.induce_compile_failure!"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_compile_failure! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_compile_failure() { + eval("def test = ::RubyVM::ZJIT.induce_compile_failure!"); + assert_compile_fails("test", ParseError::DirectiveInduced); + } + + #[test] + fn test_induce_breakpoint() { + eval("def test = ::RubyVM::ZJIT.induce_breakpoint!"); + assert!(hir_string("test").contains("BreakPoint")); + } + + #[test] + fn test_induce_breakpoint_returns_nil() { + eval(" + def test + x = ::RubyVM::ZJIT.induce_breakpoint! + x + end + "); + let hir = hir_string("test"); + assert!(hir.contains("BreakPoint")); + assert!(hir.contains("Return v")); + } + + #[test] + fn test_getspecialnumber() { + eval(" + def test(a) + a =~/(hello)/ + $1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:BasicObject = Send v10, :=~, v15 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + v22:StringExact|NilClass = GetSpecialNumber 2 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_getspecialsymbol() { + eval(" + def test(a) + a =~/(hello)/ + $& + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:BasicObject = Send v10, :=~, v15 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + v22:StringExact|NilClass = GetSpecialSymbol LastMatch + CheckInterrupts + Return v22 + "); + } +} + + /// Test successor and predecessor set computations. + #[cfg(test)] + mod control_flow_info_tests { + use super::*; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb1, bb2)); + assert!(cfi.is_succeeded_by(bb2, bb1)); + assert!(cfi.predecessors(bb3).eq([bb2])); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v1, if_true: edge(bb2), if_false: edge(bb1) }); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb2, bb3)); + assert!(cfi.is_preceded_by(bb1, bb3)); + assert!(!cfi.is_preceded_by(bb0, bb3)); + assert!(cfi.is_succeeded_by(bb1, bb0)); + assert!(cfi.is_succeeded_by(bb3, bb1)); + } + + #[test] + fn test_cfi_deduplicated_successors_and_predecessors() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + // Construct two separate jump instructions. + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v1, if_true: edge(bb1), if_false: edge(bb1)}); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert_eq!(cfi.predecessors(bb1).collect::<Vec<_>>().len(), 1); + assert_eq!(cfi.successors(bb0).collect::<Vec<_>>().len(), 1); + } + } + + /// Test dominator set computations. + #[cfg(test)] + mod dom_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + fn assert_dominators_contains_self(function: &Function, dominators: &Dominators) { + for (i, _) in function.blocks.iter().enumerate() { + // Ensure that each dominating set contains the block itself. + assert!(dominators.is_dominated_by(BlockId(i), BlockId(i))); + } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + bb4(): + v3:Any = Const CBool(true) + Return v3 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb1, bb2, bb3]); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb2) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb3() + bb2(): + Jump bb4() + bb3(): + Jump bb4() + bb4(): + v4:Any = Const CBool(true) + Return v4 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb3]); + } + + #[test] + fn test_complex_cfg() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + let bb7 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let v0 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::CondBranch { val: v0, if_true: edge(bb2), if_false: edge(bb4) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let v1 = function.push_insn(bb3, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb3, Insn::CondBranch { val: v1, if_true: edge(bb5), if_false: edge(bb7) }); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + function.push_insn(bb6, Insn::Jump(edge(bb7))); + + let retval = function.push_insn(bb7, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb7, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + v1:Any = Const Value(false) + CondBranch v1, bb3(), bb5() + bb3(): + Jump bb4() + bb4(): + v4:Any = Const Value(false) + CondBranch v4, bb6(), bb8() + bb5(): + Jump bb6() + bb6(): + Jump bb7() + bb7(): + Jump bb8() + bb8(): + v9:Any = Const CBool(true) + Return v9 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb1, bb2, bb3]); + assert_eq!(dominators.dominators(bb4), vec![entries, bb0, bb1, bb4]); + assert_eq!(dominators.dominators(bb5), vec![entries, bb0, bb1, bb5]); + assert_eq!(dominators.dominators(bb6), vec![entries, bb0, bb1, bb5, bb6]); + assert_eq!(dominators.dominators(bb7), vec![entries, bb0, bb1, bb7]); + } + + #[test] + fn test_back_edges() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let v0 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v0, if_true: edge(bb1), if_false: edge(bb4) }); + + let v1 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::CondBranch { val: v1, if_true: edge(bb2), if_false: edge(bb3) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let v2 = function.push_insn(bb5, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb5, Insn::CondBranch { val: v2, if_true: edge(bb3), if_false: edge(bb4) }); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb5() + bb2(): + v2:Any = Const Value(false) + CondBranch v2, bb3(), bb4() + bb3(): + Jump bb4() + bb5(): + Jump bb6() + bb6(): + v6:Any = Const Value(false) + CondBranch v6, bb4(), bb5() + bb4(): + v8:Any = Const CBool(true) + Return v8 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb3]); + assert_eq!(dominators.dominators(bb4), vec![entries, bb0, bb4]); + assert_eq!(dominators.dominators(bb5), vec![entries, bb0, bb4, bb5]); + } + + #[test] + fn test_multiple_entry_blocks() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + function.jit_entry_blocks.push(bb1); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb2, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb3() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const CBool(true) + Return v2 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb2]); + + assert!(!dominators.is_dominated_by(bb1, bb2)); + } + } + + /// Test loop information computation. +#[cfg(test)] +mod loop_info_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_loop_depth() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ ┌─────┐ + // ┌►│ bb2 ├─────►│ bb1 │ + // │ └──┬──┘ T └──┬──┘ + // │ F│ │ + // │ ┌──▼──┐ │ + // │ │ bb3 │ │ + // │ └─────┘ │ + // └─────────────────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb3) }); + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb3, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb3() + bb3(): + v1:Any = Const Value(false) + CondBranch v1, bb2(), bb4() + bb2(): + Jump bb3() + bb4(): + v3:Any = Const CBool(true) + Return v3 + "); + + assert!(loop_info.is_loop_header(bb2)); + assert!(loop_info.is_back_edge_source(bb1)); + assert_eq!(loop_info.loop_depth(bb1), 1); + } + + #[test] + fn test_nested_loops() { + // ┌─────┐ + // │ bb0 ◄─────┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄───┐ │ + // └──┬──┘ │ │ + // │ │ │ + // ┌──▼──┐ │ │ + // │ bb2 ┼───┘ │ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb3 ┼─────┘ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb4 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cond = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb3) }); + + let cond = function.push_insn(bb3, Insn::Const { val: Const::Value(Qtrue) }); + let _ = function.push_insn(bb3, Insn::CondBranch { val: cond, if_true: edge(bb0), if_false: edge(bb4) }); + + let retval = function.push_insn(bb4, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb4, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const Value(false) + CondBranch v2, bb2(), bb4() + bb4(): + v4:Any = Const Value(true) + CondBranch v4, bb1(), bb5() + bb5(): + v6:Any = Const CBool(true) + Return v6 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 1); + assert_eq!(loop_info.loop_depth(bb4), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + } + + #[test] + fn test_complex_loops() { + // ┌─────┐ + // ┌──────► bb0 │ + // │ └──┬──┘ + // │ ┌────┴────┐ + // │ ┌──▼──┐ ┌──▼──┐ + // │ │ bb1 ◄─┐ │ bb3 ◄─┐ + // │ └──┬──┘ │ └──┬──┘ │ + // │ │ │ │ │ + // │ ┌──▼──┐ │ ┌──▼──┐ │ + // │ │ bb2 ┼─┘ │ bb4 ┼─┘ + // │ └──┬──┘ └──┬──┘ + // │ └────┬────┘ + // │ ┌──▼──┐ + // └──────┼ bb5 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb6 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb3) }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let _ = function.push_insn(bb2, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb5) }); + + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let _ = function.push_insn(bb4, Insn::CondBranch { val: cond, if_true: edge(bb3), if_false: edge(bb5) }); + + let _ = function.push_insn(bb5, Insn::CondBranch { val: cond, if_true: edge(bb0), if_false: edge(bb6) }); + + let retval = function.push_insn(bb6, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb6, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb4() + bb2(): + Jump bb3() + bb3(): + CondBranch v0, bb2(), bb6() + bb4(): + Jump bb5() + bb5(): + CondBranch v0, bb4(), bb6() + bb6(): + CondBranch v0, bb1(), bb7() + bb7(): + v7:Any = Const CBool(true) + Return v7 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + assert!(loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb5)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb6)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 2); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + assert_eq!(loop_info.loop_depth(bb6), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + } + + #[test] + fn linked_list_non_loop() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb1 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb2 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const CBool(true) + Return v2 + "); + + assert!(!loop_info.is_loop_header(bb0)); + assert!(!loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + + assert_eq!(loop_info.loop_depth(bb0), 0); + assert_eq!(loop_info.loop_depth(bb1), 0); + assert_eq!(loop_info.loop_depth(bb2), 0); + } + + #[test] + fn triple_nested_loop() { + // ┌─────┐ + // │ bb0 ◄──┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄─┐│ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb2 ◄┐││ + // └──┬──┘│││ + // │ │││ + // ┌──▼──┐│││ + // │ bb3 ┼┘││ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb4 ┼─┘│ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb5 ┼──┘ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + let _ = function.push_insn(bb2, Insn::Jump(edge(bb3))); + let _ = function.push_insn(bb3, Insn::CondBranch {val: cond, if_true: edge(bb2), if_false: edge(bb4) }); + let _ = function.push_insn(bb4, Insn::CondBranch {val: cond, if_true: edge(bb1), if_false: edge(bb5) }); + let _ = function.push_insn(bb5, Insn::CondBranch {val: cond, if_true: edge(bb0), if_false: edge(bb6) }); + function.push_insn(bb6, Insn::Unreachable); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + bb4(): + CondBranch v0, bb3(), bb5() + bb5(): + CondBranch v0, bb2(), bb6() + bb6(): + CondBranch v0, bb1(), bb7() + bb7(): + Unreachable + "); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 3); + assert_eq!(loop_info.loop_depth(bb3), 3); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(loop_info.is_loop_header(bb2)); + assert!(!loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb5)); + } + } + +/// Test dumping to iongraph format. +#[cfg(test)] +mod iongraph_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_simple_function() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let retval = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: retval }); + function.seal_entries(); + + let json = function.to_iongraph_pass("simple"); + assert_snapshot!(json.to_string(), @r#"{"name":"simple", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_two_blocks() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + function.seal_entries(); + let json = function.to_iongraph_pass("two_blocks"); + assert_snapshot!(json.to_string(), @r#"{"name":"two_blocks", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4099, "id":3, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4097, "id":1, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"Return v1", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_instructions() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let val1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: val1 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("multiple_instructions"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_instructions", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_conditional_branch() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb2) }); + + let retval1 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval2 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("conditional_branch"); + assert_snapshot!(json.to_string(), @r#"{"name":"conditional_branch", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2, 3], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"CondBranch v0, bb2(), bb3()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4098, "id":2, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_loop_structure() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb3) }); + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb3, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + function.seal_entries(); + let json = function.to_iongraph_pass("loop_structure"); + assert_snapshot!(json.to_string(), @r#"{"name":"loop_structure", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[3], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb3()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":1, "attributes":["loopheader"], "predecessors":[1, 2], "successors":[2, 4], "instructions":[{"ptr":4097, "id":1, "opcode":"Const Value(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"CondBranch v1, bb2(), bb4()", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":1, "attributes":["backedge"], "predecessors":[3], "successors":[3], "instructions":[{"ptr":4101, "id":5, "opcode":"Jump bb3()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4100, "id":4, "loopDepth":0, "attributes":[], "predecessors":[3], "successors":[], "instructions":[{"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_successors() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb2) }); + + let retval1 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval2 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("multiple_successors"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_successors", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2, 3], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"CondBranch v0, bb2(), bb3()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4098, "id":2, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + } diff --git a/zjit/src/hir_effect/gen_hir_effect.rb b/zjit/src/hir_effect/gen_hir_effect.rb new file mode 100644 index 0000000000..1cc552d1e6 --- /dev/null +++ b/zjit/src/hir_effect/gen_hir_effect.rb @@ -0,0 +1,126 @@ +# Generate hir_effect.inc.rs. To do this, we build up a DAG that +# represents the ZJIT effect hierarchy. + +require 'set' + +# Effect represents not just a Ruby class but a named union of other effects. +class Effect + attr_accessor :name, :subeffects + + def initialize name, subeffects=nil + @name = name + @subeffects = subeffects || [] + end + + def all_subeffects + subeffects.flat_map { |subeffect| subeffect.all_subeffects } + subeffects + end + + def subeffect name + result = Effect.new name + @subeffects << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec effect, f + effect.subeffects.each {|subeffect| + f.puts effect.name + "->" + subeffect.name + ";" + } + effect.subeffects.each {|subeffect| + to_graphviz_rec subeffect, f + } +end + +# Generate graphviz. +def to_graphviz effect, f + f.puts "digraph G {" + to_graphviz_rec effect, f + f.puts "}" +end + +# ===== Start generating the effect DAG ===== + +# Start at Any. All effects are subeffects of Any. +any = Effect.new 'Any' +# Build the effect universe. +allocator = any.subeffect 'Allocator' +control = any.subeffect 'Control' +memory = any.subeffect 'Memory' +patch_point = any.subeffect 'PatchPoint' +interrupt_flag = memory.subeffect 'InterruptFlag' +other = memory.subeffect 'Other' +frame = memory.subeffect 'Frame' +pc = frame.subeffect 'PC' +locals = frame.subeffect 'Locals' +stack = frame.subeffect 'Stack' + +# Use the smallest unsigned value needed to describe all effect bits +# If it becomes an issue, this can be generated but for now we do it manually +$int_label = 'u8' + +# Assign individual bits to effect leaves and union bit patterns to nodes with subeffects +num_bits = 0 +$bits = {"Empty" => ["0#{$int_label}"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subeffects].sort_by(&:name).each {|effect| + subeffects = effect.subeffects + if subeffects.empty? + # Assign bits for leaves + $bits[effect.name] = ["1#{$int_label} << #{num_bits}"] + $numeric_bits[effect.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[effect.name] = subeffects.map(&:name).sort + end +} +[*any.all_subeffects, any].each {|effect| + subeffects = effect.subeffects + unless subeffects.empty? + $numeric_bits[effect.name] = subeffects.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir/gen_hir_effect.rb." +puts "mod bits {" +$bits.keys.sort.map {|effect_name| + subeffects = $bits[effect_name].join(" | ") + puts " pub const #{effect_name}: #{$int_label} = #{subeffects};" +} +puts " pub const AllBitPatterns: [(&str, #{$int_label}); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of an Effect. For example, Frame instead of +# PC|Stack|Locals +$numeric_bits.sort_by {|key, val| -val}.each {|effect_name, _| + puts " (\"#{effect_name}\", #{effect_name})," +} +puts " ];" +puts " pub const NumEffectBits: #{$int_label} = #{num_bits}; +}" + +puts "pub mod effect_types {" +puts " pub type EffectBits = #{$int_label};" +puts "}" + +puts "pub mod abstract_heaps { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: AbstractHeap = AbstractHeap::from_bits(bits::#{effect_name});" +} +puts "}" + +puts "pub mod effects { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: Effect = Effect::promote(abstract_heaps::#{effect_name});" +} +puts "}" + +File.open("zjit_effects.dot", "w") do |f| + to_graphviz(any, f) +end + diff --git a/zjit/src/hir_effect/hir_effect.inc.rs b/zjit/src/hir_effect/hir_effect.inc.rs new file mode 100644 index 0000000000..75e3447da1 --- /dev/null +++ b/zjit/src/hir_effect/hir_effect.inc.rs @@ -0,0 +1,63 @@ +// This file is @generated by src/hir/gen_hir_effect.rb. +mod bits { + pub const Allocator: u8 = 1u8 << 0; + pub const Any: u8 = Allocator | Control | Memory | PatchPoint; + pub const Control: u8 = 1u8 << 1; + pub const Empty: u8 = 0u8; + pub const Frame: u8 = Locals | PC | Stack; + pub const InterruptFlag: u8 = 1u8 << 2; + pub const Locals: u8 = 1u8 << 3; + pub const Memory: u8 = Frame | InterruptFlag | Other; + pub const Other: u8 = 1u8 << 4; + pub const PC: u8 = 1u8 << 5; + pub const PatchPoint: u8 = 1u8 << 6; + pub const Stack: u8 = 1u8 << 7; + pub const AllBitPatterns: [(&str, u8); 12] = [ + ("Any", Any), + ("Memory", Memory), + ("Frame", Frame), + ("Stack", Stack), + ("PatchPoint", PatchPoint), + ("PC", PC), + ("Other", Other), + ("Locals", Locals), + ("InterruptFlag", InterruptFlag), + ("Control", Control), + ("Allocator", Allocator), + ("Empty", Empty), + ]; + pub const NumEffectBits: u8 = 8; +} +pub mod effect_types { + pub type EffectBits = u8; +} +pub mod abstract_heaps { + use super::*; + pub const Allocator: AbstractHeap = AbstractHeap::from_bits(bits::Allocator); + pub const Any: AbstractHeap = AbstractHeap::from_bits(bits::Any); + pub const Control: AbstractHeap = AbstractHeap::from_bits(bits::Control); + pub const Empty: AbstractHeap = AbstractHeap::from_bits(bits::Empty); + pub const Frame: AbstractHeap = AbstractHeap::from_bits(bits::Frame); + pub const InterruptFlag: AbstractHeap = AbstractHeap::from_bits(bits::InterruptFlag); + pub const Locals: AbstractHeap = AbstractHeap::from_bits(bits::Locals); + pub const Memory: AbstractHeap = AbstractHeap::from_bits(bits::Memory); + pub const Other: AbstractHeap = AbstractHeap::from_bits(bits::Other); + pub const PC: AbstractHeap = AbstractHeap::from_bits(bits::PC); + pub const PatchPoint: AbstractHeap = AbstractHeap::from_bits(bits::PatchPoint); + pub const Stack: AbstractHeap = AbstractHeap::from_bits(bits::Stack); +} +pub mod effects { + use super::*; + pub const Allocator: Effect = Effect::promote(abstract_heaps::Allocator); + pub const Any: Effect = Effect::promote(abstract_heaps::Any); + pub const Control: Effect = Effect::promote(abstract_heaps::Control); + pub const Empty: Effect = Effect::promote(abstract_heaps::Empty); + pub const Frame: Effect = Effect::promote(abstract_heaps::Frame); + pub const InterruptFlag: Effect = Effect::promote(abstract_heaps::InterruptFlag); + pub const Locals: Effect = Effect::promote(abstract_heaps::Locals); + pub const Memory: Effect = Effect::promote(abstract_heaps::Memory); + pub const Other: Effect = Effect::promote(abstract_heaps::Other); + pub const PC: Effect = Effect::promote(abstract_heaps::PC); + pub const PatchPoint: Effect = Effect::promote(abstract_heaps::PatchPoint); + pub const Stack: Effect = Effect::promote(abstract_heaps::Stack); +} diff --git a/zjit/src/hir_effect/mod.rs b/zjit/src/hir_effect/mod.rs new file mode 100644 index 0000000000..b1d7b27411 --- /dev/null +++ b/zjit/src/hir_effect/mod.rs @@ -0,0 +1,420 @@ +//! High-level intermediate representation effects. + +#![allow(non_upper_case_globals)] +use crate::hir::{PtrPrintMap}; +include!("hir_effect.inc.rs"); + +// NOTE: Effect very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +/// The AbstractHeap struct is the main work horse of effect inference and specialization. The main interfaces +/// will look like: +/// +/// * is AbstractHeap A a subset of AbstractHeap B +/// * union/meet AbstractHeap A and AbstractHeap B +/// +/// or +/// +/// * is Effect A a subset of Effect B +/// * union/meet Effect A and Effect B +/// +/// The AbstractHeap is the work horse because Effect is simply 2 AbstractHeaps; one for read, and one for write. +/// Currently, the abstract heap is implemented as a bitset. As we enrich our effect system, this will be updated +/// to match the name and use a heap implementation, roughly aligned with +/// <https://gist.github.com/pizlonator/cf1e72b8600b1437dda8153ea3fdb963>. +/// +/// Most questions can be rewritten in terms of these operations. +/// +/// Lattice Top corresponds to the "Any" effect. All bits are set and any effect is possible. +/// Lattice Bottom corresponds to the "None" effect. No bits are set and no effects are possible. +/// Elements between abstract_heaps have effects corresponding to the bits that are set. +/// This enables more complex analyses compared to prior ZJIT implementations such as "has_effect", +/// a function that returns a boolean value. Such functions impose an implicit single bit effect +/// system. This explicit design with a lattice allows us many bits for effects. +#[derive(Clone, Copy, Debug)] +pub struct AbstractHeap { + bits: effect_types::EffectBits +} + +#[derive(Clone, Copy, Debug)] +pub struct Effect { + /// Unlike ZJIT's type system, effects do not have a notion of subclasses. + /// Instead of specializations, the Effect struct contains two AbstractHeaps. + /// We distinguish between read effects and write effects. + /// Both use the same effects lattice, but splitting into two heaps allows + /// for finer grained optimization. + /// + /// For instance: + /// We can elide HIR instructions with no write effects, but the read effects are necessary for instruction + /// reordering optimizations. + /// + /// These fields should not be directly read or written except by internal `Effect` APIs. + read: AbstractHeap, + write: AbstractHeap +} + +/// Print adaptor for [`AbstractHeap`]. See [`PtrPrintMap`]. +pub struct AbstractHeapPrinter<'a> { + inner: AbstractHeap, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for AbstractHeapPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let effect = self.inner; + let mut bits = effect.bits; + let mut sep = ""; + // First, make sure patterns are sorted from higher order bits to lower order. + // For each match where `bits` contains the pattern, we mask off the matched bits + // and continue searching for matches until bits == 0. + // Our first match could be exact and may not require a separator, but all subsequent + // matches do. + debug_assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + for (name, pattern) in bits::AllBitPatterns { + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + // The `sep != ""` check allows us to handle the effects::None case gracefully. + if (bits == 0) & (sep != "") { break; } + } + debug_assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + Ok(()) + } +} + +impl std::fmt::Display for AbstractHeap { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +/// Print adaptor for [`Effect`]. See [`PtrPrintMap`]. +pub struct EffectPrinter<'a> { + inner: Effect, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for EffectPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}, {}", self.inner.read, self.inner.write) + } +} + +impl std::fmt::Display for Effect { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl AbstractHeap { + const fn from_bits(bits: effect_types::EffectBits) -> Self { + Self { bits } + } + + pub const fn union(&self, other: Self) -> Self { + Self::from_bits(self.bits | other.bits) + } + + pub const fn intersect(&self, other: Self) -> Self { + Self::from_bits(self.bits & other.bits) + } + + pub const fn exclude(&self, other: Self) -> Self { + Self::from_bits(self.bits - (self.bits & other.bits)) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Self) -> bool { + self.bits == other.bits + } + + pub const fn includes(&self, other: Self) -> bool { + self.bit_equal( + self.union(other) + ) + } + + pub const fn overlaps(&self, other: Self) -> bool { + !abstract_heaps::Empty.includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> AbstractHeapPrinter<'_> { + AbstractHeapPrinter { inner: self, ptr_map } + } +} + +impl Effect { + pub const fn read_write(read: AbstractHeap, write: AbstractHeap) -> Effect { + Effect { read, write } + } + + /// This function addresses the special case where the read and write heaps are the same + pub const fn promote(heap: AbstractHeap) -> Effect { + Effect {read: heap, write: heap } + } + + /// This function accepts write and heaps read to Any + pub const fn write(write: AbstractHeap) -> Effect { + Effect { read: abstract_heaps::Any, write } + } + + /// This function accepts read and heaps read to Any + pub const fn read(read: AbstractHeap) -> Effect { + Effect { read, write: abstract_heaps::Any } + } + + /// Method to access the private read field + pub const fn read_bits(&self) -> AbstractHeap { + self.read + } + + /// Method to access the private write field + pub const fn write_bits(&self) -> AbstractHeap { + self.write + } + + pub const fn union(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.union(other.read), + self.write.union(other.write) + ) + } + + pub const fn intersect(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.intersect(other.read), + self.write.intersect(other.write) + ) + } + + pub const fn exclude(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.exclude(other.read), + self.write.exclude(other.write) + ) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Effect) -> bool { + self.read.bit_equal(other.read) & self.write.bit_equal(other.write) + } + + pub const fn includes(&self, other: Effect) -> bool { + self.bit_equal(Effect::union(self, other)) + } + + pub const fn overlaps(&self, other: Effect) -> bool { + Effect::promote(abstract_heaps::Empty).includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> EffectPrinter<'_> { + EffectPrinter { inner: self, ptr_map } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[track_caller] + fn assert_heap_bit_equal(left: AbstractHeap, right: AbstractHeap) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(right.includes(left), "{left} is not a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_not_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(!right.includes(left), "{left} is a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Effect, right: Effect) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect(left: Effect, right: Effect) { + assert!(right.includes(left), "{left} is not a subeffect of {right}"); + } + + #[track_caller] + fn assert_not_subeffect(left: Effect, right: Effect) { + assert!(!right.includes(left), "{left} is a subeffect of {right}"); + } + + #[test] + fn effect_heap_none_is_subeffect_of_everything() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Control); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Stack); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Locals); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Allocator); + } + + #[test] + fn effect_heap_everything_is_subeffect_of_any() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Control, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Any); + } + + #[test] + fn effect_heap_union_never_shrinks() { + // iterate over all effect entries from bottom to top + for i in [0, 1, 4, 6, 10, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty.union(e)); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any.union(e)); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Frame.union(e)); + } + } + + #[test] + fn effect_heap_intersect_never_grows() { + // Randomly selected values from bottom to top + for i in [0, 3, 6, 8, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty.intersect(e), abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Any.intersect(e), abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame.intersect(e), abstract_heaps::Frame); + } + } + + #[test] + fn effect_heap_self_is_included() { + assert!(abstract_heaps::Stack.includes(abstract_heaps::Stack)); + assert!(abstract_heaps::Any.includes(abstract_heaps::Any)); + assert!(abstract_heaps::Empty.includes(abstract_heaps::Empty)); + } + + #[test] + fn effect_heap_frame_includes_stack_locals_and_pc() { + assert_subeffect_heap(abstract_heaps::Stack, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Frame); + } + + #[test] + fn effect_heap_frame_is_stack_locals_and_pc() { + let union = abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC)); + assert_heap_bit_equal(abstract_heaps::Frame, union); + } + + #[test] + fn effect_heap_any_includes_some_subeffects() { + assert_subeffect_heap(abstract_heaps::Allocator, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + } + + #[test] + fn effect_heap_display_exact_bits_match() { + assert_eq!(format!("{}", abstract_heaps::Empty), "Empty"); + assert_eq!(format!("{}", abstract_heaps::PC), "PC"); + assert_eq!(format!("{}", abstract_heaps::Any), "Any"); + assert_eq!(format!("{}", abstract_heaps::Frame), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + } + + #[test] + fn effect_heap_display_multiple_bits() { + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals)), "Stack|Locals"); + assert_eq!(format!("{}", abstract_heaps::PC.union(abstract_heaps::Allocator)), "PC|Allocator"); + } + + #[test] + fn effect_any_includes_everything() { + assert_subeffect(effects::Allocator, effects::Any); + assert_subeffect(effects::Frame, effects::Any); + assert_subeffect(effects::Memory, effects::Any); + // Let's do a less standard effect too + assert_subeffect( + Effect::read_write(abstract_heaps::Control, abstract_heaps::Any), + effects::Any + ); + } + + #[test] + fn effect_union_is_idempotent() { + assert_bit_equal( + Effect::read(abstract_heaps::Any) + .union(Effect::write(abstract_heaps::Any)), + effects::Any + ); + assert_bit_equal( + effects::Empty.union(effects::Empty), + effects::Empty + ); + } + + #[test] + fn effect_union_contains_and_excludes() { + assert_subeffect( + effects::Control.union(effects::Frame), + effects::Any + ); + assert_not_subeffect( + effects::Frame.union(effects::Locals), + effects::PC + ); + } + + #[test] + fn effect_intersect_is_empty() { + assert_subeffect(effects::Memory.intersect(effects::Control), effects::Empty); + assert_subeffect( + Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Other) + .intersect(Effect::read_write(abstract_heaps::Stack, abstract_heaps::PC)), + effects::Empty + ) + } + + #[test] + fn effect_intersect_exact_match() { + assert_subeffect(effects::Frame.intersect(effects::PC), effects::PC); + assert_subeffect(effects::Allocator.intersect(effects::Allocator), effects::Allocator); + } + + #[test] + fn effect_display_exact_bits_match() { + assert_eq!(format!("{}", effects::Empty), "Empty, Empty"); + assert_eq!(format!("{}", effects::PC), "PC, PC"); + assert_eq!(format!("{}", effects::Any), "Any, Any"); + assert_eq!(format!("{}", effects::Frame), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", Effect::write(abstract_heaps::Control)), "Any, Control"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Memory)), "Allocator, Memory"); + } + + #[test] + fn effect_display_multiple_bits() { + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals)), "Stack|Locals, Stack|Locals"); + assert_eq!(format!("{}", effects::PC.union(effects::Allocator)), "PC|Allocator, PC|Allocator"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Other, abstract_heaps::PC) + .union(Effect::read_write(abstract_heaps::Memory, abstract_heaps::Stack))), + "Memory, Stack|PC" + ); + } + +} diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb new file mode 100644 index 0000000000..2eb5ca1932 --- /dev/null +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -0,0 +1,251 @@ +# Generate hir_type.inc.rs. To do this, we build up a DAG that +# represents a slice of the Ruby type hierarchy that we care about optimizing. +# This also includes cvalue values such as C booleans, int32, and so on. + +require 'set' + +# Type represents not just a Ruby class but a named union of other types. +class Type + attr_accessor :name, :subtypes + + def initialize name, subtypes=nil + @name = name + @subtypes = subtypes || [] + end + + def all_subtypes + subtypes.flat_map { |subtype| subtype.all_subtypes } + subtypes + end + + def subtype name + result = Type.new name + @subtypes << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec type, f + type.subtypes.each {|subtype| + f.puts type.name + "->" + subtype.name + ";" + } + type.subtypes.each {|subtype| + to_graphviz_rec subtype, f + } +end + +# Generate graphviz. +def to_graphviz type, f + f.puts "digraph G {" + to_graphviz_rec type, f + f.puts "}" +end + +# ===== Start generating the type DAG ===== + +# Start at Any. All types are subtypes of Any. +any = Type.new "Any" +# Build the Ruby object universe. +value = any.subtype "RubyValue" +undef_ = value.subtype "Undef" +value.subtype "CallableMethodEntry" # rb_callable_method_entry_t* +basic_object = value.subtype "BasicObject" +basic_object_exact = basic_object.subtype "BasicObjectExact" +basic_object_subclass = basic_object.subtype "BasicObjectSubclass" +$object = basic_object.subtype "Object" +object_exact = $object.subtype "ObjectExact" +object_subclass = $object.subtype "ObjectSubclass" +$subclass = [basic_object_subclass.name, object_subclass.name] +$builtin_exact = [basic_object_exact.name, object_exact.name] + +$exact_c_names = { + "ObjectExact" => "rb_cObject", + "BasicObjectExact" => "rb_cBasicObject", +} + +$subclass_c_names = { + "ObjectSubclass" => "rb_cObject", + "BasicObjectSubclass" => "rb_cBasicObject", +} + +$inexact_c_names = { + "Object" => "rb_cObject", + "BasicObject" => "rb_cBasicObject", +} + +# Define a new type that can be subclassed (most of them). +# If c_name is given, mark the rb_cXYZ object as equivalent to this exact type. +def base_type name, base: $object, c_name: nil + type = base.subtype name + exact = type.subtype(name+"Exact") + subclass = type.subtype(name+"Subclass") + if c_name + $exact_c_names[exact.name] = c_name + $subclass_c_names[subclass.name] = c_name + $inexact_c_names[type.name] = c_name + end + $builtin_exact << exact.name + $subclass << subclass.name + [type, exact] +end + +# Define a new type that has no subclasses and cannot be subclassed. +# If c_name is given, mark the rb_cXYZ object as equivalent to this type. +def final_type name, base: $object, c_name: nil + if c_name + $exact_c_names[name] = c_name + $subclass_c_names[name] = c_name + $inexact_c_names[name] = c_name + end + type = base.subtype name + $builtin_exact << type.name + type +end + +base_type "String", c_name: "rb_cString" +base_type "Array", c_name: "rb_cArray" +base_type "Hash", c_name: "rb_cHash" +base_type "Range", c_name: "rb_cRange" +base_type "Set", c_name: "rb_cSet" +base_type "Regexp", c_name: "rb_cRegexp" +module_class, _ = base_type "Module", c_name: "rb_cModule" +# Class cannot be subclassed by doing `class Sub < Class`, +# but every metaclass is a subclass of `Class`. It's not final. +base_type "Class", base: module_class, c_name: "rb_cClass" + +numeric, _ = base_type "Numeric", c_name: "rb_cNumeric" + +integer_exact = final_type "Integer", base: numeric, c_name: "rb_cInteger" +# CRuby partitions Integer into immediate and non-immediate variants. +fixnum = integer_exact.subtype "Fixnum" +integer_exact.subtype "Bignum" + +float_exact = final_type "Float", base: numeric, c_name: "rb_cFloat" +# CRuby partitions Float into immediate and non-immediate variants. +flonum = float_exact.subtype "Flonum" +float_exact.subtype "HeapFloat" + +symbol_exact = final_type "Symbol", c_name: "rb_cSymbol" +# CRuby partitions Symbol into immediate and non-immediate variants. +static_sym = symbol_exact.subtype "StaticSymbol" +symbol_exact.subtype "DynamicSymbol" + +nil_exact = final_type "NilClass", c_name: "rb_cNilClass" +true_exact = final_type "TrueClass", c_name: "rb_cTrueClass" +false_exact = final_type "FalseClass", c_name: "rb_cFalseClass" + +# T_DATA objects have a distinct memory layout for field access and don't have a +# common class ancestor below BasicObject. +basic_object.subtype "TData" + +# Build the cvalue object universe. This is for C-level types that may be +# passed around when calling into the Ruby VM or after some strength reduction +# of HIR. +cvalue = any.subtype "CValue" +cvalue.subtype "CBool" +cvalue.subtype "CPtr" +cvalue.subtype "CDouble" +cvalue.subtype "CNull" +cvalue_int = cvalue.subtype "CInt" +signed = cvalue_int.subtype "CSigned" +unsigned = cvalue_int.subtype "CUnsigned" +[8, 16, 32, 64].each {|width| + signed.subtype "CInt#{width}" + unsigned.subtype "CUInt#{width}" +} +unsigned.subtype "CShape" +unsigned.subtype "CAttrIndex" + +# Assign individual bits to type leaves and union bit patterns to nodes with subtypes +num_bits = 0 +$bits = {"Empty" => ["0u64"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subtypes].sort_by(&:name).each {|type| + subtypes = type.subtypes + if subtypes.empty? + # Assign bits for leaves + $bits[type.name] = ["1u64 << #{num_bits}"] + $numeric_bits[type.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[type.name] = subtypes.map(&:name).sort + end +} +[*any.all_subtypes, any].each {|type| + subtypes = type.subtypes + unless subtypes.empty? + $numeric_bits[type.name] = subtypes.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# Unions are for names of groups of type bit patterns that don't fit neatly +# into the Ruby class hierarchy. For example, we might want to refer to a union +# of TrueClassExact|FalseClassExact by the name BoolExact even though a "bool" +# doesn't exist as a class in Ruby. +def add_union name, type_names + type_names = type_names.sort + $bits[name] = type_names + $numeric_bits[name] = type_names.map {|type_name| $numeric_bits[type_name]}.reduce(&:|) +end + +add_union "BuiltinExact", $builtin_exact +add_union "Subclass", $subclass +add_union "BoolExact", [true_exact.name, false_exact.name] +add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] +add_union "Falsy", [nil_exact.name, false_exact.name] +$bits["HeapBasicObject"] = ["BasicObject & !Immediate"] +$numeric_bits["HeapBasicObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] +$bits["HeapObject"] = ["Object & !Immediate"] +$numeric_bits["HeapObject"] = $numeric_bits["Object"] & ~$numeric_bits["Immediate"] +$bits["Truthy"] = ["BasicObject & !Falsy"] +$numeric_bits["Truthy"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Falsy"] +$bits["NotNil"] = ["BasicObject & !NilClass"] +$numeric_bits["NotNil"] = $numeric_bits["BasicObject"] & ~$numeric_bits["NilClass"] + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir_type/gen_hir_type.rb." +puts "mod bits {" +$bits.keys.sort.map {|type_name| + subtypes = $bits[type_name].join(" | ") + puts " pub const #{type_name}: u64 = #{subtypes};" +} +puts " pub const AllBitPatterns: [(&str, u64); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of a Type. For example, CSigned instead of +# CInt8|CInt16|... +$numeric_bits.sort_by {|key, val| -val}.each {|type_name, _| + puts " (\"#{type_name}\", #{type_name})," +} +puts " ];" +puts " pub const NumTypeBits: u64 = #{num_bits}; +}" + +puts "pub mod types { + use super::*;" +$bits.keys.sort.map {|type_name| + puts " pub const #{type_name}: Type = Type::from_bits(bits::#{type_name});" +} +puts " pub const ExactBitsAndClass: [(u64, *const VALUE); #{$exact_c_names.size}] = [" +$exact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$subclass_c_names = $subclass_c_names.to_a.sort_by {|name, _| $numeric_bits[name.sub("Subclass", "")]}.to_h +puts " pub const SubclassBitsAndClass: [(u64, *const VALUE); #{$subclass_c_names.size}] = [" +$subclass_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$inexact_c_names = $inexact_c_names.to_a.sort_by {|name, _| $numeric_bits[name]}.to_h +puts " pub const InexactBitsAndClass: [(u64, *const VALUE); #{$inexact_c_names.size}] = [" +$inexact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +puts "}" + +File.open("zjit_types.dot", "w") do |f| + to_graphviz(any, f) +end diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs new file mode 100644 index 0000000000..1dae344b36 --- /dev/null +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -0,0 +1,300 @@ +// This file is @generated by src/hir_type/gen_hir_type.rb. +mod bits { + pub const Any: u64 = CValue | RubyValue; + pub const Array: u64 = ArrayExact | ArraySubclass; + pub const ArrayExact: u64 = 1u64 << 0; + pub const ArraySubclass: u64 = 1u64 << 1; + pub const BasicObject: u64 = BasicObjectExact | BasicObjectSubclass | Object | TData; + pub const BasicObjectExact: u64 = 1u64 << 2; + pub const BasicObjectSubclass: u64 = 1u64 << 3; + pub const Bignum: u64 = 1u64 << 4; + pub const BoolExact: u64 = FalseClass | TrueClass; + pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | ClassExact | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | NumericExact | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; + pub const CAttrIndex: u64 = 1u64 << 5; + pub const CBool: u64 = 1u64 << 6; + pub const CDouble: u64 = 1u64 << 7; + pub const CInt: u64 = CSigned | CUnsigned; + pub const CInt16: u64 = 1u64 << 8; + pub const CInt32: u64 = 1u64 << 9; + pub const CInt64: u64 = 1u64 << 10; + pub const CInt8: u64 = 1u64 << 11; + pub const CNull: u64 = 1u64 << 12; + pub const CPtr: u64 = 1u64 << 13; + pub const CShape: u64 = 1u64 << 14; + pub const CSigned: u64 = CInt16 | CInt32 | CInt64 | CInt8; + pub const CUInt16: u64 = 1u64 << 15; + pub const CUInt32: u64 = 1u64 << 16; + pub const CUInt64: u64 = 1u64 << 17; + pub const CUInt8: u64 = 1u64 << 18; + pub const CUnsigned: u64 = CAttrIndex | CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; + pub const CValue: u64 = CBool | CDouble | CInt | CNull | CPtr; + pub const CallableMethodEntry: u64 = 1u64 << 19; + pub const Class: u64 = ClassExact | ClassSubclass; + pub const ClassExact: u64 = 1u64 << 20; + pub const ClassSubclass: u64 = 1u64 << 21; + pub const DynamicSymbol: u64 = 1u64 << 22; + pub const Empty: u64 = 0u64; + pub const FalseClass: u64 = 1u64 << 23; + pub const Falsy: u64 = FalseClass | NilClass; + pub const Fixnum: u64 = 1u64 << 24; + pub const Float: u64 = Flonum | HeapFloat; + pub const Flonum: u64 = 1u64 << 25; + pub const Hash: u64 = HashExact | HashSubclass; + pub const HashExact: u64 = 1u64 << 26; + pub const HashSubclass: u64 = 1u64 << 27; + pub const HeapBasicObject: u64 = BasicObject & !Immediate; + pub const HeapFloat: u64 = 1u64 << 28; + pub const HeapObject: u64 = Object & !Immediate; + pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; + pub const Integer: u64 = Bignum | Fixnum; + pub const Module: u64 = Class | ModuleExact | ModuleSubclass; + pub const ModuleExact: u64 = 1u64 << 29; + pub const ModuleSubclass: u64 = 1u64 << 30; + pub const NilClass: u64 = 1u64 << 31; + pub const NotNil: u64 = BasicObject & !NilClass; + pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; + pub const NumericExact: u64 = 1u64 << 32; + pub const NumericSubclass: u64 = 1u64 << 33; + pub const Object: u64 = Array | FalseClass | Hash | Module | NilClass | Numeric | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; + pub const ObjectExact: u64 = 1u64 << 34; + pub const ObjectSubclass: u64 = 1u64 << 35; + pub const Range: u64 = RangeExact | RangeSubclass; + pub const RangeExact: u64 = 1u64 << 36; + pub const RangeSubclass: u64 = 1u64 << 37; + pub const Regexp: u64 = RegexpExact | RegexpSubclass; + pub const RegexpExact: u64 = 1u64 << 38; + pub const RegexpSubclass: u64 = 1u64 << 39; + pub const RubyValue: u64 = BasicObject | CallableMethodEntry | Undef; + pub const Set: u64 = SetExact | SetSubclass; + pub const SetExact: u64 = 1u64 << 40; + pub const SetSubclass: u64 = 1u64 << 41; + pub const StaticSymbol: u64 = 1u64 << 42; + pub const String: u64 = StringExact | StringSubclass; + pub const StringExact: u64 = 1u64 << 43; + pub const StringSubclass: u64 = 1u64 << 44; + pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | ClassSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; + pub const Symbol: u64 = DynamicSymbol | StaticSymbol; + pub const TrueClass: u64 = 1u64 << 45; + pub const Truthy: u64 = BasicObject & !Falsy; + pub const TData: u64 = 1u64 << 46; + pub const Undef: u64 = 1u64 << 47; + pub const AllBitPatterns: [(&str, u64); 78] = [ + ("Any", Any), + ("RubyValue", RubyValue), + ("Immediate", Immediate), + ("Undef", Undef), + ("BasicObject", BasicObject), + ("NotNil", NotNil), + ("Truthy", Truthy), + ("HeapBasicObject", HeapBasicObject), + ("TData", TData), + ("Object", Object), + ("BuiltinExact", BuiltinExact), + ("BoolExact", BoolExact), + ("TrueClass", TrueClass), + ("HeapObject", HeapObject), + ("String", String), + ("Subclass", Subclass), + ("StringSubclass", StringSubclass), + ("StringExact", StringExact), + ("Symbol", Symbol), + ("StaticSymbol", StaticSymbol), + ("Set", Set), + ("SetSubclass", SetSubclass), + ("SetExact", SetExact), + ("Regexp", Regexp), + ("RegexpSubclass", RegexpSubclass), + ("RegexpExact", RegexpExact), + ("Range", Range), + ("RangeSubclass", RangeSubclass), + ("RangeExact", RangeExact), + ("ObjectSubclass", ObjectSubclass), + ("ObjectExact", ObjectExact), + ("Numeric", Numeric), + ("NumericSubclass", NumericSubclass), + ("NumericExact", NumericExact), + ("Falsy", Falsy), + ("NilClass", NilClass), + ("Module", Module), + ("ModuleSubclass", ModuleSubclass), + ("ModuleExact", ModuleExact), + ("Float", Float), + ("HeapFloat", HeapFloat), + ("Hash", Hash), + ("HashSubclass", HashSubclass), + ("HashExact", HashExact), + ("Flonum", Flonum), + ("Integer", Integer), + ("Fixnum", Fixnum), + ("FalseClass", FalseClass), + ("DynamicSymbol", DynamicSymbol), + ("Class", Class), + ("ClassSubclass", ClassSubclass), + ("ClassExact", ClassExact), + ("CallableMethodEntry", CallableMethodEntry), + ("CValue", CValue), + ("CInt", CInt), + ("CUnsigned", CUnsigned), + ("CUInt8", CUInt8), + ("CUInt64", CUInt64), + ("CUInt32", CUInt32), + ("CUInt16", CUInt16), + ("CShape", CShape), + ("CPtr", CPtr), + ("CNull", CNull), + ("CSigned", CSigned), + ("CInt8", CInt8), + ("CInt64", CInt64), + ("CInt32", CInt32), + ("CInt16", CInt16), + ("CDouble", CDouble), + ("CBool", CBool), + ("CAttrIndex", CAttrIndex), + ("Bignum", Bignum), + ("BasicObjectSubclass", BasicObjectSubclass), + ("BasicObjectExact", BasicObjectExact), + ("Array", Array), + ("ArraySubclass", ArraySubclass), + ("ArrayExact", ArrayExact), + ("Empty", Empty), + ]; + pub const NumTypeBits: u64 = 48; +} +pub mod types { + use super::*; + pub const Any: Type = Type::from_bits(bits::Any); + pub const Array: Type = Type::from_bits(bits::Array); + pub const ArrayExact: Type = Type::from_bits(bits::ArrayExact); + pub const ArraySubclass: Type = Type::from_bits(bits::ArraySubclass); + pub const BasicObject: Type = Type::from_bits(bits::BasicObject); + pub const BasicObjectExact: Type = Type::from_bits(bits::BasicObjectExact); + pub const BasicObjectSubclass: Type = Type::from_bits(bits::BasicObjectSubclass); + pub const Bignum: Type = Type::from_bits(bits::Bignum); + pub const BoolExact: Type = Type::from_bits(bits::BoolExact); + pub const BuiltinExact: Type = Type::from_bits(bits::BuiltinExact); + pub const CAttrIndex: Type = Type::from_bits(bits::CAttrIndex); + pub const CBool: Type = Type::from_bits(bits::CBool); + pub const CDouble: Type = Type::from_bits(bits::CDouble); + pub const CInt: Type = Type::from_bits(bits::CInt); + pub const CInt16: Type = Type::from_bits(bits::CInt16); + pub const CInt32: Type = Type::from_bits(bits::CInt32); + pub const CInt64: Type = Type::from_bits(bits::CInt64); + pub const CInt8: Type = Type::from_bits(bits::CInt8); + pub const CNull: Type = Type::from_bits(bits::CNull); + pub const CPtr: Type = Type::from_bits(bits::CPtr); + pub const CShape: Type = Type::from_bits(bits::CShape); + pub const CSigned: Type = Type::from_bits(bits::CSigned); + pub const CUInt16: Type = Type::from_bits(bits::CUInt16); + pub const CUInt32: Type = Type::from_bits(bits::CUInt32); + pub const CUInt64: Type = Type::from_bits(bits::CUInt64); + pub const CUInt8: Type = Type::from_bits(bits::CUInt8); + pub const CUnsigned: Type = Type::from_bits(bits::CUnsigned); + pub const CValue: Type = Type::from_bits(bits::CValue); + pub const CallableMethodEntry: Type = Type::from_bits(bits::CallableMethodEntry); + pub const Class: Type = Type::from_bits(bits::Class); + pub const ClassExact: Type = Type::from_bits(bits::ClassExact); + pub const ClassSubclass: Type = Type::from_bits(bits::ClassSubclass); + pub const DynamicSymbol: Type = Type::from_bits(bits::DynamicSymbol); + pub const Empty: Type = Type::from_bits(bits::Empty); + pub const FalseClass: Type = Type::from_bits(bits::FalseClass); + pub const Falsy: Type = Type::from_bits(bits::Falsy); + pub const Fixnum: Type = Type::from_bits(bits::Fixnum); + pub const Float: Type = Type::from_bits(bits::Float); + pub const Flonum: Type = Type::from_bits(bits::Flonum); + pub const Hash: Type = Type::from_bits(bits::Hash); + pub const HashExact: Type = Type::from_bits(bits::HashExact); + pub const HashSubclass: Type = Type::from_bits(bits::HashSubclass); + pub const HeapBasicObject: Type = Type::from_bits(bits::HeapBasicObject); + pub const HeapFloat: Type = Type::from_bits(bits::HeapFloat); + pub const HeapObject: Type = Type::from_bits(bits::HeapObject); + pub const Immediate: Type = Type::from_bits(bits::Immediate); + pub const Integer: Type = Type::from_bits(bits::Integer); + pub const Module: Type = Type::from_bits(bits::Module); + pub const ModuleExact: Type = Type::from_bits(bits::ModuleExact); + pub const ModuleSubclass: Type = Type::from_bits(bits::ModuleSubclass); + pub const NilClass: Type = Type::from_bits(bits::NilClass); + pub const NotNil: Type = Type::from_bits(bits::NotNil); + pub const Numeric: Type = Type::from_bits(bits::Numeric); + pub const NumericExact: Type = Type::from_bits(bits::NumericExact); + pub const NumericSubclass: Type = Type::from_bits(bits::NumericSubclass); + pub const Object: Type = Type::from_bits(bits::Object); + pub const ObjectExact: Type = Type::from_bits(bits::ObjectExact); + pub const ObjectSubclass: Type = Type::from_bits(bits::ObjectSubclass); + pub const Range: Type = Type::from_bits(bits::Range); + pub const RangeExact: Type = Type::from_bits(bits::RangeExact); + pub const RangeSubclass: Type = Type::from_bits(bits::RangeSubclass); + pub const Regexp: Type = Type::from_bits(bits::Regexp); + pub const RegexpExact: Type = Type::from_bits(bits::RegexpExact); + pub const RegexpSubclass: Type = Type::from_bits(bits::RegexpSubclass); + pub const RubyValue: Type = Type::from_bits(bits::RubyValue); + pub const Set: Type = Type::from_bits(bits::Set); + pub const SetExact: Type = Type::from_bits(bits::SetExact); + pub const SetSubclass: Type = Type::from_bits(bits::SetSubclass); + pub const StaticSymbol: Type = Type::from_bits(bits::StaticSymbol); + pub const String: Type = Type::from_bits(bits::String); + pub const StringExact: Type = Type::from_bits(bits::StringExact); + pub const StringSubclass: Type = Type::from_bits(bits::StringSubclass); + pub const Subclass: Type = Type::from_bits(bits::Subclass); + pub const Symbol: Type = Type::from_bits(bits::Symbol); + pub const TrueClass: Type = Type::from_bits(bits::TrueClass); + pub const Truthy: Type = Type::from_bits(bits::Truthy); + pub const TData: Type = Type::from_bits(bits::TData); + pub const Undef: Type = Type::from_bits(bits::Undef); + pub const ExactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ObjectExact, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectExact, &raw const crate::cruby::rb_cBasicObject), + (bits::StringExact, &raw const crate::cruby::rb_cString), + (bits::ArrayExact, &raw const crate::cruby::rb_cArray), + (bits::HashExact, &raw const crate::cruby::rb_cHash), + (bits::RangeExact, &raw const crate::cruby::rb_cRange), + (bits::SetExact, &raw const crate::cruby::rb_cSet), + (bits::RegexpExact, &raw const crate::cruby::rb_cRegexp), + (bits::ModuleExact, &raw const crate::cruby::rb_cModule), + (bits::ClassExact, &raw const crate::cruby::rb_cClass), + (bits::NumericExact, &raw const crate::cruby::rb_cNumeric), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + ]; + pub const SubclassBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ArraySubclass, &raw const crate::cruby::rb_cArray), + (bits::ClassSubclass, &raw const crate::cruby::rb_cClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::HashSubclass, &raw const crate::cruby::rb_cHash), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::ModuleSubclass, &raw const crate::cruby::rb_cModule), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::NumericSubclass, &raw const crate::cruby::rb_cNumeric), + (bits::RangeSubclass, &raw const crate::cruby::rb_cRange), + (bits::RegexpSubclass, &raw const crate::cruby::rb_cRegexp), + (bits::SetSubclass, &raw const crate::cruby::rb_cSet), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::StringSubclass, &raw const crate::cruby::rb_cString), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::ObjectSubclass, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectSubclass, &raw const crate::cruby::rb_cBasicObject), + ]; + pub const InexactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::Array, &raw const crate::cruby::rb_cArray), + (bits::Class, &raw const crate::cruby::rb_cClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Hash, &raw const crate::cruby::rb_cHash), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Module, &raw const crate::cruby::rb_cModule), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::Numeric, &raw const crate::cruby::rb_cNumeric), + (bits::Range, &raw const crate::cruby::rb_cRange), + (bits::Regexp, &raw const crate::cruby::rb_cRegexp), + (bits::Set, &raw const crate::cruby::rb_cSet), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::String, &raw const crate::cruby::rb_cString), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::Object, &raw const crate::cruby::rb_cObject), + (bits::BasicObject, &raw const crate::cruby::rb_cBasicObject), + ]; +} diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs new file mode 100644 index 0000000000..d7327975ce --- /dev/null +++ b/zjit/src/hir_type/mod.rs @@ -0,0 +1,1107 @@ +//! High-level intermediate representation types. + +#![allow(non_upper_case_globals)] +use crate::cruby; +use crate::cruby::{rb_block_param_proxy, Qfalse, Qnil, Qtrue, RUBY_T_ARRAY, RUBY_T_HASH, RUBY_T_STRING, VALUE}; +use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cRange, rb_zjit_singleton_class_p}; +use crate::cruby::ClassRelationship; +use crate::cruby::get_class_name; +use crate::cruby::get_module_name; +use crate::cruby::ruby_sym_to_rust_string; +use crate::cruby::rb_mRubyVMFrozenCore; +use crate::hir::{Const, PtrPrintMap}; +use crate::profile::ProfiledType; + +#[derive(Copy, Clone, Debug, PartialEq)] +/// Specialization of the type. If we know additional information about the object, we put it here. +/// This includes information about its value as a cvalue. For Ruby objects, type specialization +/// is split into three sub-cases: +/// +/// * Object, where we know exactly what object (pointer) the Type corresponds to +/// * Type exact, where we know exactly what class the Type represents (which could be because we +/// have an instance of it; includes Object specialization) +/// * Type, where we know that the Type could represent the given class or any of its subclasses +/// +/// It is also a lattice but a much shallower one. It is not meant to be used directly, just by +/// Type internals. +pub enum Specialization { + /// We know nothing about the specialization of this Type. + Any, + /// We know that this Type is an instance of the given Ruby class in the VALUE or any of its subclasses. + Type(VALUE), + /// We know that this Type is an instance of exactly the Ruby class in the VALUE. + TypeExact(VALUE), + /// We know that this Type is exactly the Ruby object in the VALUE. + Object(VALUE), + /// We know that this Type is exactly the given cvalue/C integer value (use the type bits to + /// inform how we should interpret the u64, e.g. as CBool or CInt32). + Int(u64), + /// We know that this Type is exactly the given cvalue/C double. + Double(f64), + /// We know that the Type is [`types::Empty`] and therefore the instruction that produces this + /// value never returns. + Empty, +} + +// NOTE: Type very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +#[derive(Copy, Clone, Debug)] +/// The main work horse of intraprocedural type inference and specialization. The main interfaces +/// will look like: +/// +/// * is type A a subset of type B +/// * union/meet type A and type B +/// +/// Most questions can be rewritten in terms of these operations. +pub struct Type { + /// A bitset representing type information about the object. Specific bits are assigned for + /// leaf types (for example, static symbols) and union-ing bitsets together represents + /// union-ing sets of types. These sets form a lattice (with Any as "could be anything" and + /// Empty as "can be nothing"). + /// + /// Capable of also representing cvalue types (bool, i32, etc). + /// + /// This field should not be directly read or written except by internal `Type` APIs. + bits: u64, + /// Specialization of the type. See [`Specialization`]. + /// + /// This field should not be directly read or written except by internal `Type` APIs. + spec: Specialization +} + +include!("hir_type.inc.rs"); + +fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::Result { + let ty = printer.inner; + match ty.spec { + Specialization::Any | Specialization::Empty => { Ok(()) }, + Specialization::Object(val) if val == unsafe { rb_mRubyVMFrozenCore } => write!(f, "[VMFrozenCore]"), + Specialization::Object(val) if val == unsafe { rb_block_param_proxy } => write!(f, "[BlockParamProxy]"), + Specialization::Object(val) if ty.is_subtype(types::Symbol) => write!(f, "[:{}]", ruby_sym_to_rust_string(val)), + Specialization::Object(val) if ty.is_subtype(types::Class) => + write!(f, "[{}@{:p}]", get_class_name(val), printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)), + Specialization::Object(val) if ty.is_subtype(types::Module) => { + if let Some(name) = get_module_name(val) { + write!(f, "[{}@{:p}]", name, printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)) + } else { + // Same as generic Specialization::Object + write!(f, "[{}]", val.print(printer.ptr_map)) + } + } + Specialization::Object(val) => write!(f, "[{}]", val.print(printer.ptr_map)), + // TODO(max): Ensure singleton classes never have Type specialization + Specialization::Type(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), + Specialization::Type(val) => write!(f, "[class:{}]", get_class_name(val)), + Specialization::TypeExact(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class_exact*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), + Specialization::TypeExact(val) => + write!(f, "[class_exact:{}]", get_class_name(val)), + Specialization::Int(val) if ty.is_subtype(types::CBool) => write!(f, "[{}]", val != 0), + Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val & u8::MAX as u64) as i8), + Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val & u16::MAX as u64) as i16), + Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val & u32::MAX as u64) as i32), + Specialization::Int(val) if ty.is_subtype(types::CShape) => + write!(f, "[{:p}]", printer.ptr_map.map_shape(crate::cruby::ShapeId((val & u32::MAX as u64) as u32))), + Specialization::Int(val) if ty.is_subtype(types::CInt64) => write!(f, "[{}]", val as i64), + Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val & u8::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val & u16::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val & u32::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt64) => { + // Print in hex if signed bit is set + if 0 != val & (1 << (u64::BITS - 1)) { + write!(f, "[0x{val:x}]") + } else { + write!(f, "[{val}]") + } + } + Specialization::Int(val) if ty.is_subtype(types::CPtr) => write!(f, "[{}]", Const::CPtr(val as *const u8).print(printer.ptr_map)), + Specialization::Int(val) => write!(f, "[{val}]"), + Specialization::Double(val) => write!(f, "[{val}]"), + } +} + +/// Print adaptor for [`Type`]. See [`PtrPrintMap`]. +pub struct TypePrinter<'a> { + inner: Type, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for TypePrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let ty = self.inner; + for (name, pattern) in bits::AllBitPatterns { + if ty.bits == pattern { + write!(f, "{name}")?; + return write_spec(f, self); + } + } + assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + let mut bits = ty.bits; + let mut sep = ""; + for (name, pattern) in bits::AllBitPatterns { + if bits == 0 { break; } + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + } + assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + write_spec(f, self) + } +} + +impl std::fmt::Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +fn is_array_exact(val: VALUE) -> bool { + // Prism hides array values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cArray } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_ARRAY) +} + +fn is_string_exact(val: VALUE) -> bool { + // Prism hides string values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cString } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_STRING) +} + +fn is_hash_exact(val: VALUE) -> bool { + // Prism hides hash values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cHash } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_HASH) +} + +fn is_range_exact(val: VALUE) -> bool { + val.class_of() == unsafe { rb_cRange } +} + +impl Type { + /// Create a `Type` from the given integer. + pub const fn fixnum(val: i64) -> Type { + Type { + bits: bits::Fixnum, + spec: Specialization::Object(VALUE::fixnum_from_usize(val as usize)), + } + } + + fn bits_from_exact_class(class: VALUE) -> Option<u64> { + types::ExactBitsAndClass + .iter() + .find(|&(_, class_object)| unsafe { **class_object } == class) + .map(|&(bits, _)| bits) + } + + fn bits_from_subclass(class: VALUE) -> Option<u64> { + types::SubclassBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + // Can't be an immediate if it's a subclass. + .map(|&(bits, _)| bits & !bits::Immediate) + } + + fn from_heap_object(val: VALUE) -> Type { + assert!(!val.special_const_p(), "val should be a heap object"); + let bits = + // GC-hidden types + if is_array_exact(val) { bits::ArrayExact } + else if is_hash_exact(val) { bits::HashExact } + else if is_string_exact(val) { bits::StringExact } + // Classes that have an immediate/heap split + else if val.class_of() == unsafe { rb_cInteger } { bits::Bignum } + else if val.class_of() == unsafe { rb_cFloat } { bits::HeapFloat } + else if val.class_of() == unsafe { rb_cSymbol } { bits::DynamicSymbol } + else if let Some(bits) = Self::bits_from_exact_class(val.class_of()) { bits } + else if let Some(bits) = Self::bits_from_subclass(val.class_of()) { bits } + else if val.data_p() { bits::TData } + else { + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(val.class_of())) + }; + let spec = Specialization::Object(val); + Type { bits, spec } + } + + /// Create a `Type` from a Ruby `VALUE`. The type is not guaranteed to have object + /// specialization in its `specialization` field (for example, `Qnil` will just be + /// `types::NilClass`), but will be available via `ruby_object()`. + pub fn from_value(val: VALUE) -> Type { + // Immediates + if val.fixnum_p() { + Type { bits: bits::Fixnum, spec: Specialization::Object(val) } + } + else if val.flonum_p() { + Type { bits: bits::Flonum, spec: Specialization::Object(val) } + } + else if val.static_sym_p() { + Type { bits: bits::StaticSymbol, spec: Specialization::Object(val) } + } + // Singleton objects; don't specialize + else if val == Qnil { types::NilClass } + else if val == Qtrue { types::TrueClass } + else if val == Qfalse { types::FalseClass } + else if val.cme_p() { + // NB: Checking for CME has to happen before looking at class_of because that's not + // valid on imemo. + Type { bits: bits::CallableMethodEntry, spec: Specialization::Object(val) } + } + else { + Self::from_heap_object(val) + } + } + + pub fn from_const(val: Const) -> Type { + match val { + Const::Value(v) => Self::from_value(v), + Const::CBool(v) => Self::from_cbool(v), + Const::CInt8(v) => Self::from_cint(types::CInt8, v as i64), + Const::CInt16(v) => Self::from_cint(types::CInt16, v as i64), + Const::CInt32(v) => Self::from_cint(types::CInt32, v as i64), + Const::CInt64(v) => Self::from_cint(types::CInt64, v), + Const::CUInt8(v) => Self::from_cint(types::CUInt8, v as i64), + Const::CUInt16(v) => Self::from_cint(types::CUInt16, v as i64), + Const::CUInt32(v) => Self::from_cint(types::CUInt32, v as i64), + Const::CAttrIndex(v) => Self::from_cint(types::CAttrIndex, v as i64), + Const::CShape(v) => Self::from_cint(types::CShape, v.0 as i64), + Const::CUInt64(v) => Self::from_cint(types::CUInt64, v as i64), + Const::CPtr(v) => Self::from_cptr(v), + Const::CDouble(v) => Self::from_double(v), + } + } + + pub fn from_profiled_type(val: ProfiledType) -> Type { + if val.is_fixnum() { types::Fixnum } + else if val.is_flonum() { types::Flonum } + else if val.is_static_symbol() { types::StaticSymbol } + else if val.is_nil() { types::NilClass } + else if val.is_true() { types::TrueClass } + else if val.is_false() { types::FalseClass } + // TODO(max): Revisit and maybe specialize to *not* an immediate + else { Self::from_class(val.class()).intersection(types::HeapBasicObject) } + } + + pub fn from_class(class: VALUE) -> Type { + if let Some(bits) = Self::bits_from_exact_class(class) { + return Type::from_bits(bits); + } + if let Some(bits) = Self::bits_from_subclass(class) { + return Type { bits, spec: Specialization::TypeExact(class) } + } + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(class)) + } + + pub fn from_class_inexact(class: VALUE) -> Type { + let bits = types::InexactBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + .unwrap_or_else(|| panic!("Class {} is not a subclass of BasicObject! Don't know what to do.", get_class_name(class))).0; + Type { bits, spec: Specialization::Type(class) } + } + + /// Private. Only for creating type globals. + const fn from_bits(bits: u64) -> Type { + Type { + bits, + spec: if bits == bits::Empty { + Specialization::Empty + } else { + Specialization::Any + }, + } + } + + /// Create a `Type` from a cvalue integer. Use the `ty` given to specify what size the + /// `specialization` represents. For example, `Type::from_cint(types::CBool, 1)` or + /// `Type::from_cint(types::CUInt16, 12)`. + pub fn from_cint(ty: Type, val: i64) -> Type { + assert_eq!(ty.spec, Specialization::Any); + assert!((ty.is_subtype(types::CUnsigned) || ty.is_subtype(types::CSigned)) && + ty.bits != types::CUnsigned.bits && ty.bits != types::CSigned.bits, + "ty must be a specific int size"); + Type { bits: ty.bits, spec: Specialization::Int(val as u64) } + } + + pub fn from_cptr(val: *const u8) -> Type { + Type { bits: bits::CPtr, spec: Specialization::Int(val as u64) } + } + + /// Create a `Type` (a `CDouble` with double specialization) from a f64. + pub fn from_double(val: f64) -> Type { + Type { bits: bits::CDouble, spec: Specialization::Double(val) } + } + + /// Create a `Type` from a cvalue boolean. + pub fn from_cbool(val: bool) -> Type { + Type { bits: bits::CBool, spec: Specialization::Int(val as u64) } + } + + /// Return true if the value with this type is definitely truthy. + pub fn is_known_truthy(&self) -> bool { + !self.could_be(types::NilClass) && !self.could_be(types::FalseClass) + } + + /// Return true if the value with this type is definitely falsy. + pub fn is_known_falsy(&self) -> bool { + self.is_subtype(types::NilClass) || self.is_subtype(types::FalseClass) + } + + pub fn has_value(&self, val: Const) -> bool { + match (self.spec, val) { + (Specialization::Object(v1), Const::Value(v2)) => v1 == v2, + (Specialization::Int(v1), Const::CBool(v2)) if self.is_subtype(types::CBool) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt8(v2)) if self.is_subtype(types::CInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt16(v2)) if self.is_subtype(types::CInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt32(v2)) if self.is_subtype(types::CInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt64(v2)) if self.is_subtype(types::CInt64) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt8(v2)) if self.is_subtype(types::CUInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt16(v2)) if self.is_subtype(types::CUInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt32(v2)) if self.is_subtype(types::CUInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CShape(v2)) if self.is_subtype(types::CShape) => v1 == (v2.0 as u64), + (Specialization::Int(v1), Const::CUInt64(v2)) if self.is_subtype(types::CUInt64) => v1 == v2, + (Specialization::Int(v1), Const::CPtr(v2)) if self.is_subtype(types::CPtr) => v1 == (v2 as u64), + (Specialization::Double(v1), Const::CDouble(v2)) => v1.to_bits() == v2.to_bits(), + _ => false, + } + } + + /// Return the object specialization, if any. + pub fn ruby_object(&self) -> Option<VALUE> { + match self.spec { + Specialization::Object(val) => Some(val), + _ => None, + } + } + + /// Return a Ruby object that needs to be marked on GC. + /// This covers Type and TypeExact unlike ruby_object(). + pub fn gc_object(&self) -> Option<VALUE> { + match self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + + /// Mutable version of gc_object(). + pub fn gc_object_mut(&mut self) -> Option<&mut VALUE> { + match &mut self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + + pub fn unspecialized(&self) -> Self { + Type { spec: Specialization::Any, ..*self } + } + + pub fn fixnum_value(&self) -> Option<i64> { + if self.is_subtype(types::Fixnum) { + self.ruby_object().map(|val| val.as_fixnum()) + } else { + None + } + } + + pub fn cint64_value(&self) -> Option<i64> { + match (self.is_subtype(types::CInt64), &self.spec) { + (true, Specialization::Int(val)) => Some(*val as i64), + _ => None, + } + } + + fn int_spec_signed(&self) -> Option<i64> { + assert!(self.is_subtype(types::CSigned), "int_spec_signed() only makes sense for signed integer types"); + match self.spec { + Specialization::Int(val) => Some(val as i64), + _ => None, + } + } + + pub fn known_nonnegative(&self) -> bool { + assert!(self.is_subtype(types::CSigned), "nonnegative() only makes sense for signed integer types"); + self.int_spec_signed().map_or(false, |val| val >= 0) + } + + /// Return true if the Type has object specialization and false otherwise. + pub fn ruby_object_known(&self) -> bool { + matches!(self.spec, Specialization::Object(_)) + } + + /// Find a `T_*` type that is exactly as wide as `self`. + pub fn builtin_type_equivalent(&self) -> Option<cruby::ruby_value_type> { + if self.bit_equal(types::Array) { + Some(cruby::RUBY_T_ARRAY) + } else if self.bit_equal(types::Class) { + Some(cruby::RUBY_T_CLASS) + } else if self.bit_equal(types::Module) { + Some(cruby::RUBY_T_MODULE) + } else if self.bit_equal(types::String) { + Some(cruby::RUBY_T_STRING) + } else if self.bit_equal(types::Hash) { + Some(cruby::RUBY_T_HASH) + } else if self.bit_equal(types::TData) { + Some(cruby::RUBY_T_DATA) + } else { + None + } + } + + fn is_builtin(class: VALUE) -> bool { + types::ExactBitsAndClass + .iter() + .any(|&(_, class_object)| unsafe { *class_object } == class) + } + + /// Union both types together, preserving specialization if possible. + pub fn union(&self, other: Type) -> Type { + // Easy cases first + if self.is_subtype(other) { return other; } + if other.is_subtype(*self) { return *self; } + let bits = self.bits | other.bits; + let result = Type::from_bits(bits); + // If one type isn't type specialized, we can't return a specialized Type + if !self.type_known() || !other.type_known() { return result; } + let self_class = self.inexact_ruby_class().unwrap(); + let other_class = other.inexact_ruby_class().unwrap(); + // Pick one of self/other as the least upper bound. This is not the most specific (there + // could be intermediate classes in the inheritance hierarchy) but it is fast to compute. + let super_class = match self_class.is_subclass_of(other_class) { + ClassRelationship::Subclass => other_class, + ClassRelationship::Superclass => self_class, + ClassRelationship::NoRelation => return result, + }; + // Don't specialize built-in types; we can represent them perfectly with type bits. + if Type::is_builtin(super_class) { return result; } + // Supertype specialization can be exact only if the exact type specializations are identical + if let Some(self_class) = self.exact_ruby_class() { + if let Some(other_class) = other.exact_ruby_class() { + if self_class == other_class { + return Type { bits, spec: Specialization::TypeExact(self_class) }; + } + } + } + Type { bits, spec: Specialization::Type(super_class) } + } + + /// Intersect both types, preserving specialization if possible. + pub fn intersection(&self, other: Type) -> Type { + let bits = self.bits & other.bits; + if bits == bits::Empty { return types::Empty; } + if self.spec_is_subtype_of(other) { return Type { bits, spec: self.spec }; } + if other.spec_is_subtype_of(*self) { return Type { bits, spec: other.spec }; } + types::Empty + } + + pub fn could_be(&self, other: Type) -> bool { + !self.intersection(other).bit_equal(types::Empty) + } + + /// Check if the type field of `self` is a subtype of the type field of `other` and also check + /// if the specialization of `self` is a subtype of the specialization of `other`. + pub fn is_subtype(&self, other: Type) -> bool { + (self.bits & other.bits) == self.bits && self.spec_is_subtype_of(other) + } + + /// Return the type specialization, if any. Type specialization asks if we know the Ruby type + /// (including potentially its subclasses) corresponding to a `Type`, including knowing exactly + /// what object is is. + pub fn type_known(&self) -> bool { + matches!(self.spec, Specialization::TypeExact(_) | Specialization::Type(_) | Specialization::Object(_)) + } + + /// Return the exact type specialization, if any. Type specialization asks if we know the + /// *exact* Ruby type corresponding to a `Type`, including knowing exactly what object is is. + pub fn exact_class_known(&self) -> bool { + matches!(self.spec, Specialization::TypeExact(_) | Specialization::Object(_)) + } + + /// Return the exact type specialization, if any. Type specialization asks if we know the exact + /// Ruby type corresponding to a `Type` (no subclasses), including knowing exactly what object + /// it is. + pub fn exact_ruby_class(&self) -> Option<VALUE> { + match self.spec { + // If we're looking at a precise object, we can pull out its class. + Specialization::Object(val) => Some(val.class_of()), + Specialization::TypeExact(val) => Some(val), + _ => None, + } + } + + /// Return the type specialization, if any. Type specialization asks if we know the inexact + /// Ruby type corresponding to a `Type`, including knowing exactly what object is is. + pub fn inexact_ruby_class(&self) -> Option<VALUE> { + match self.spec { + // If we're looking at a precise object, we can pull out its class. + Specialization::Object(val) => Some(val.class_of()), + Specialization::TypeExact(val) | Specialization::Type(val) => Some(val), + _ => None, + } + } + + /// Return a pointer to the Ruby class that an object of this Type would have at run-time, if + /// known. This includes classes for HIR types such as ArrayExact or NilClass, which have + /// canonical Type representations that lack an explicit specialization in their `spec` fields. + pub fn runtime_exact_ruby_class(&self) -> Option<VALUE> { + if let Some(val) = self.exact_ruby_class() { + return Some(val); + } + types::ExactBitsAndClass + .iter() + .find(|&(bits, _)| self.is_subtype(Type::from_bits(*bits))) + .map(|&(_, class_object)| unsafe { *class_object }) + } + + /// Check bit equality of two `Type`s. Do not use! You are probably looking for [`Type::is_subtype`]. + pub fn bit_equal(&self, other: Type) -> bool { + self.bits == other.bits && self.spec == other.spec + } + + /// Check *only* if `self`'s specialization is a subtype of `other`'s specialization. Private. + /// You probably want [`Type::is_subtype`] instead. + fn spec_is_subtype_of(&self, other: Type) -> bool { + match (self.spec, other.spec) { + // Empty is a subtype of everything; Any is a supertype of everything + (Specialization::Empty, _) | (_, Specialization::Any) => true, + // Other is not Any from the previous case, so Any is definitely not a subtype + (Specialization::Any, _) | (_, Specialization::Empty) => false, + // Int and double specialization requires exact equality + (Specialization::Int(_), _) | (_, Specialization::Int(_)) | + (Specialization::Double(_), _) | (_, Specialization::Double(_)) => + self.bits == other.bits && self.spec == other.spec, + // Check other's specialization type in decreasing order of specificity + (_, Specialization::Object(_)) => + self.ruby_object_known() && self.ruby_object() == other.ruby_object(), + (_, Specialization::TypeExact(_)) => + self.exact_class_known() && self.inexact_ruby_class() == other.inexact_ruby_class(), + (_, Specialization::Type(other_class)) => + self.inexact_ruby_class().unwrap().is_subclass_of(other_class) == ClassRelationship::Subclass, + } + } + + pub fn is_immediate(&self) -> bool { + self.is_subtype(types::Immediate) + } + + pub fn print(self, ptr_map: &PtrPrintMap) -> TypePrinter<'_> { + TypePrinter { inner: self, ptr_map } + } + + pub fn num_bits(&self) -> u8 { + self.num_bytes() * crate::cruby::BITS_PER_BYTE as u8 + } + + pub fn num_bytes(&self) -> u8 { + if self.is_subtype(types::CUInt8) || self.is_subtype(types::CInt8) { return 1; } + if self.is_subtype(types::CUInt16) || self.is_subtype(types::CInt16) { return 2; } + if self.is_subtype(types::CUInt32) || self.is_subtype(types::CInt32) { return 4; } + if self.is_subtype(types::CShape) { + use crate::cruby::{SHAPE_ID_NUM_BITS, BITS_PER_BYTE}; + return (SHAPE_ID_NUM_BITS as usize / BITS_PER_BYTE).try_into().unwrap(); + } + // CUInt64, CInt64, CPtr, CNull, CDouble, or anything else defaults to 8 bytes + crate::cruby::SIZEOF_VALUE as u8 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cruby::rust_str_to_ruby; + use crate::cruby::rust_str_to_sym; + use crate::cruby::rb_ary_new_capa; + use crate::cruby::rb_hash_new; + use crate::cruby::rb_float_new; + use crate::cruby::define_class; + use crate::cruby::rb_cObject; + use crate::cruby::rb_cSet; + use crate::cruby::rb_cTrueClass; + use crate::cruby::rb_cFalseClass; + use crate::cruby::rb_cNilClass; + + #[track_caller] + fn assert_bit_equal(left: Type, right: Type) { + assert_eq!(left.bits, right.bits, "{left} bits are not equal to {right} bits"); + assert_eq!(left.spec, right.spec, "{left} spec is not equal to {right} spec"); + } + + #[track_caller] + fn assert_subtype(left: Type, right: Type) { + assert!(left.is_subtype(right), "{left} is not a subtype of {right}"); + } + + #[track_caller] + fn assert_not_subtype(left: Type, right: Type) { + assert!(!left.is_subtype(right), "{left} is a subtype of {right}"); + } + + #[test] + fn empty_is_subtype_of_everything() { + // Spot check a few cases + assert_subtype(types::Empty, types::NilClass); + assert_subtype(types::Empty, types::Array); + assert_subtype(types::Empty, types::Object); + assert_subtype(types::Empty, types::CUInt16); + assert_subtype(types::Empty, Type::from_cint(types::CInt32, 10)); + assert_subtype(types::Empty, types::Any); + assert_subtype(types::Empty, types::Empty); + } + + #[test] + fn everything_is_a_subtype_of_any() { + // Spot check a few cases + assert_subtype(types::NilClass, types::Any); + assert_subtype(types::Array, types::Any); + assert_subtype(types::Object, types::Any); + assert_subtype(types::CUInt16, types::Any); + assert_subtype(Type::from_cint(types::CInt32, 10), types::Any); + assert_subtype(types::Empty, types::Any); + assert_subtype(types::Any, types::Any); + } + + #[test] + fn from_const() { + let cint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int(12)); + assert_eq!(format!("{}", cint32), "CInt32[12]"); + + let cint32 = Type::from_const(Const::CInt32(-12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int((-12i64) as u64)); + assert_eq!(format!("{}", cint32), "CInt32[-12]"); + + let cuint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cuint32, types::CInt32); + assert_eq!(cuint32.spec, Specialization::Int(12)); + + let cuint32 = Type::from_const(Const::CUInt32(0xffffffff)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xffffffff)); + assert_eq!(format!("{}", cuint32), "CUInt32[4294967295]"); + + let cuint32 = Type::from_const(Const::CUInt32(0xc00087)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xc00087)); + assert_eq!(format!("{}", cuint32), "CUInt32[12583047]"); + } + + #[test] + fn integer() { + assert_subtype(Type::fixnum(123), types::Fixnum); + assert_subtype(Type::fixnum(123), Type::fixnum(123)); + assert_not_subtype(Type::fixnum(123), Type::fixnum(200)); + assert_subtype(Type::from_value(VALUE::fixnum_from_usize(123)), types::Fixnum); + assert_subtype(types::Fixnum, types::Integer); + assert_subtype(types::Bignum, types::Integer); + } + + #[test] + fn float() { + assert_subtype(types::Flonum, types::Float); + assert_subtype(types::HeapFloat, types::Float); + } + + #[test] + fn numeric() { + assert_subtype(types::Integer, types::Numeric); + assert_subtype(types::Float, types::Numeric); + assert_subtype(types::Float.union(types::Integer), types::Numeric); + assert_bit_equal(types::Float + .union(types::Integer) + .union(types::NumericExact) + .union(types::NumericSubclass), types::Numeric); + } + + #[test] + fn symbol() { + assert_subtype(types::StaticSymbol, types::Symbol); + assert_subtype(types::DynamicSymbol, types::Symbol); + } + + #[test] + fn immediate() { + assert_subtype(Type::fixnum(123), types::Immediate); + assert_subtype(types::Fixnum, types::Immediate); + assert_not_subtype(types::Bignum, types::Immediate); + assert_not_subtype(types::Integer, types::Immediate); + assert_subtype(types::NilClass, types::Immediate); + assert_subtype(types::TrueClass, types::Immediate); + assert_subtype(types::FalseClass, types::Immediate); + assert_subtype(types::StaticSymbol, types::Immediate); + assert_not_subtype(types::DynamicSymbol, types::Immediate); + assert_subtype(types::Flonum, types::Immediate); + assert_not_subtype(types::HeapFloat, types::Immediate); + } + + #[test] + fn heap_basic_object() { + assert_not_subtype(Type::fixnum(123), types::HeapBasicObject); + assert_not_subtype(types::Fixnum, types::HeapBasicObject); + assert_subtype(types::Bignum, types::HeapBasicObject); + assert_not_subtype(types::Integer, types::HeapBasicObject); + assert_not_subtype(types::NilClass, types::HeapBasicObject); + assert_not_subtype(types::TrueClass, types::HeapBasicObject); + assert_not_subtype(types::FalseClass, types::HeapBasicObject); + assert_not_subtype(types::StaticSymbol, types::HeapBasicObject); + assert_subtype(types::DynamicSymbol, types::HeapBasicObject); + assert_not_subtype(types::Flonum, types::HeapBasicObject); + assert_subtype(types::HeapFloat, types::HeapBasicObject); + assert_not_subtype(types::BasicObject, types::HeapBasicObject); + assert_not_subtype(types::Object, types::HeapBasicObject); + assert_not_subtype(types::Immediate, types::HeapBasicObject); + assert_not_subtype(types::HeapBasicObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapBasicObject); + assert_subtype(right, types::HeapBasicObject); + assert_subtype(left.union(right), types::HeapBasicObject); + }); + } + + #[test] + fn heap_object() { + assert_not_subtype(Type::fixnum(123), types::HeapObject); + assert_not_subtype(types::Fixnum, types::HeapObject); + assert_subtype(types::Bignum, types::HeapObject); + assert_not_subtype(types::Integer, types::HeapObject); + assert_not_subtype(types::NilClass, types::HeapObject); + assert_not_subtype(types::TrueClass, types::HeapObject); + assert_not_subtype(types::FalseClass, types::HeapObject); + assert_not_subtype(types::StaticSymbol, types::HeapObject); + assert_subtype(types::DynamicSymbol, types::HeapObject); + assert_not_subtype(types::Flonum, types::HeapObject); + assert_subtype(types::HeapFloat, types::HeapObject); + assert_not_subtype(types::BasicObject, types::HeapObject); + assert_not_subtype(types::Object, types::HeapObject); + assert_not_subtype(types::Immediate, types::HeapObject); + assert_not_subtype(types::HeapObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapObject); + assert_subtype(right, types::HeapObject); + assert_subtype(left.union(right), types::HeapObject); + }); + } + + #[test] + fn fixnum_has_ruby_object() { + assert_eq!(Type::fixnum(3).ruby_object(), Some(VALUE::fixnum_from_usize(3))); + assert_eq!(types::Fixnum.ruby_object(), None); + assert_eq!(types::Integer.ruby_object(), None); + } + + #[test] + fn singletons_do_not_have_ruby_object() { + assert_eq!(Type::from_value(Qnil).ruby_object(), None); + assert_eq!(types::NilClass.ruby_object(), None); + assert_eq!(Type::from_value(Qtrue).ruby_object(), None); + assert_eq!(types::TrueClass.ruby_object(), None); + assert_eq!(Type::from_value(Qfalse).ruby_object(), None); + assert_eq!(types::FalseClass.ruby_object(), None); + } + + #[test] + fn integer_has_exact_ruby_class() { + assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.exact_ruby_class(), None); + assert_eq!(types::Integer.exact_ruby_class(), None); + } + + #[test] + fn singletons_do_not_have_exact_ruby_class() { + assert_eq!(Type::from_value(Qnil).exact_ruby_class(), None); + assert_eq!(types::NilClass.exact_ruby_class(), None); + assert_eq!(Type::from_value(Qtrue).exact_ruby_class(), None); + assert_eq!(types::TrueClass.exact_ruby_class(), None); + assert_eq!(Type::from_value(Qfalse).exact_ruby_class(), None); + assert_eq!(types::FalseClass.exact_ruby_class(), None); + } + + #[test] + fn singletons_do_not_have_ruby_class() { + assert_eq!(Type::from_value(Qnil).inexact_ruby_class(), None); + assert_eq!(types::NilClass.inexact_ruby_class(), None); + assert_eq!(Type::from_value(Qtrue).inexact_ruby_class(), None); + assert_eq!(types::TrueClass.inexact_ruby_class(), None); + assert_eq!(Type::from_value(Qfalse).inexact_ruby_class(), None); + assert_eq!(types::FalseClass.inexact_ruby_class(), None); + } + + #[test] + fn from_class() { + crate::cruby::with_rubyvm(|| { + assert_bit_equal(Type::from_class(unsafe { rb_cInteger }), types::Integer); + assert_bit_equal(Type::from_class(unsafe { rb_cString }), types::StringExact); + assert_bit_equal(Type::from_class(unsafe { rb_cArray }), types::ArrayExact); + assert_bit_equal(Type::from_class(unsafe { rb_cHash }), types::HashExact); + assert_bit_equal(Type::from_class(unsafe { rb_cNilClass }), types::NilClass); + assert_bit_equal(Type::from_class(unsafe { rb_cTrueClass }), types::TrueClass); + assert_bit_equal(Type::from_class(unsafe { rb_cFalseClass }), types::FalseClass); + let c_class = define_class("C", unsafe { rb_cObject }); + assert_bit_equal(Type::from_class(c_class), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }); + }); + } + + #[test] + fn integer_has_ruby_class() { + crate::cruby::with_rubyvm(|| { + assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.inexact_ruby_class(), None); + assert_eq!(types::Integer.inexact_ruby_class(), None); + }); + } + + #[test] + fn set() { + assert_subtype(types::SetExact, types::Set); + assert_subtype(types::SetSubclass, types::Set); + } + + #[test] + fn set_has_ruby_class() { + crate::cruby::with_rubyvm(|| { + assert_eq!(types::SetExact.runtime_exact_ruby_class(), Some(unsafe { rb_cSet })); + assert_eq!(types::Set.runtime_exact_ruby_class(), None); + assert_eq!(types::SetSubclass.runtime_exact_ruby_class(), None); + }); + } + + #[test] + fn display_exact_bits_match() { + assert_eq!(format!("{}", Type::fixnum(4)), "Fixnum[4]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt8, -1)), "CInt8[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt8, -1)), "CUInt8[255]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt16, -1)), "CInt16[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt16, -1)), "CUInt16[65535]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt32, -1)), "CInt32[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt32, -1)), "CUInt32[4294967295]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt64, -1)), "CInt64[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt64, -1)), "CUInt64[0xffffffffffffffff]"); + assert_eq!(format!("{}", Type::from_cbool(true)), "CBool[true]"); + assert_eq!(format!("{}", Type::from_cbool(false)), "CBool[false]"); + assert_eq!(format!("{}", types::Fixnum), "Fixnum"); + assert_eq!(format!("{}", types::Integer), "Integer"); + } + + #[test] + fn display_multiple_bits() { + assert_eq!(format!("{}", types::CSigned), "CSigned"); + assert_eq!(format!("{}", types::CUInt8.union(types::CInt32)), "CUInt8|CInt32"); + assert_eq!(format!("{}", types::HashExact.union(types::HashSubclass)), "Hash"); + } + + #[test] + fn union_equal() { + assert_bit_equal(types::Fixnum.union(types::Fixnum), types::Fixnum); + assert_bit_equal(Type::fixnum(3).union(Type::fixnum(3)), Type::fixnum(3)); + } + + #[test] + fn union_bits_subtype() { + assert_bit_equal(types::Fixnum.union(types::Integer), types::Integer); + assert_bit_equal(types::Fixnum.union(types::Object), types::Object); + assert_bit_equal(Type::fixnum(3).union(types::Fixnum), types::Fixnum); + + assert_bit_equal(types::Integer.union(types::Fixnum), types::Integer); + assert_bit_equal(types::Object.union(types::Fixnum), types::Object); + assert_bit_equal(types::Fixnum.union(Type::fixnum(3)), types::Fixnum); + } + + #[test] + fn union_bits_unions_bits() { + assert_bit_equal(types::Fixnum.union(types::StaticSymbol), Type { bits: bits::Fixnum | bits::StaticSymbol, spec: Specialization::Any }); + } + + #[test] + fn union_int_specialized() { + assert_bit_equal(Type::from_cbool(true).union(Type::from_cbool(true)), Type::from_cbool(true)); + assert_bit_equal(Type::from_cbool(true).union(Type::from_cbool(false)), types::CBool); + assert_bit_equal(Type::from_cbool(true).union(types::CBool), types::CBool); + + assert_bit_equal(Type::from_cbool(false).union(Type::from_cbool(true)), types::CBool); + assert_bit_equal(types::CBool.union(Type::from_cbool(true)), types::CBool); + } + + #[test] + fn union_one_type_specialized_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let specialized = Type::from_value(unsafe { rb_ary_new_capa(0) }); + let unspecialized = types::StringExact; + assert_bit_equal(specialized.union(unspecialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + assert_bit_equal(unspecialized.union(specialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + }); + } + + #[test] + fn union_specialized_builtin_subtype_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let hello = Type::from_value(rust_str_to_ruby("hello")); + let world = Type::from_value(rust_str_to_ruby("world")); + assert_bit_equal(hello.union(world), types::StringExact); + }); + crate::cruby::with_rubyvm(|| { + let hello = Type::from_value(rust_str_to_sym("hello")); + let world = Type::from_value(rust_str_to_sym("world")); + assert_bit_equal(hello.union(world), types::StaticSymbol); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("hello")); + assert_bit_equal(left.union(right), types::StringExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_sym("hello")); + let right = Type::from_value(rust_str_to_sym("hello")); + assert_bit_equal(left.union(right), left); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_ary_new_capa(0) }); + let right = Type::from_value(unsafe { rb_ary_new_capa(0) }); + assert_bit_equal(left.union(right), types::ArrayExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_hash_new() }); + let right = Type::from_value(unsafe { rb_hash_new() }); + assert_bit_equal(left.union(right), types::HashExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_float_new(1.0) }); + let right = Type::from_value(unsafe { rb_float_new(2.0) }); + assert_bit_equal(left.union(right), types::Flonum); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_float_new(1.7976931348623157e+308) }); + let right = Type::from_value(unsafe { rb_float_new(1.7976931348623157e+308) }); + assert_bit_equal(left.union(right), types::HeapFloat); + }); + } + + #[test] + fn cme() { + use crate::cruby::{rb_callable_method_entry, ID}; + crate::cruby::with_rubyvm(|| { + let cme = unsafe { rb_callable_method_entry(rb_cInteger, ID!(to_s)) }; + assert!(!cme.is_null()); + let cme_value: VALUE = cme.into(); + let ty = Type::from_value(cme_value); + assert_subtype(ty, types::CallableMethodEntry); + assert!(ty.ruby_object_known()); + }); + } + + #[test] + fn string_subclass_is_string_subtype() { + crate::cruby::with_rubyvm(|| { + assert_subtype(types::StringExact, types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::StringExact); + let c_class = define_class("C", unsafe { rb_cString }); + assert_subtype(Type::from_class(c_class), types::String); + }); + } + + #[test] + fn union_specialized_with_no_relation_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let string = Type::from_value(rust_str_to_ruby("hello")); + let array = Type::from_value(unsafe { rb_ary_new_capa(0) }); + assert_bit_equal(string.union(array), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + }); + } + + #[test] + fn union_specialized_with_subclass_relationship_returns_superclass() { + crate::cruby::with_rubyvm(|| { + let c_class = define_class("C", unsafe { rb_cObject }); + let d_class = define_class("D", c_class); + let c_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }; + let d_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(d_class) }; + assert_bit_equal(c_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class)}); + assert_bit_equal(c_instance.union(d_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); + assert_bit_equal(d_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); + }); + } + + #[test] + fn has_value() { + // With known values + crate::cruby::with_rubyvm(|| { + let a = rust_str_to_sym("a"); + let b = rust_str_to_sym("b"); + let ty = Type::from_value(a); + assert!(ty.has_value(Const::Value(a))); + assert!(!ty.has_value(Const::Value(b))); + }); + + let true_ty = Type::from_cbool(true); + assert!(true_ty.has_value(Const::CBool(true))); + assert!(!true_ty.has_value(Const::CBool(false))); + + let int8_ty = Type::from_cint(types::CInt8, 42); + assert!(int8_ty.has_value(Const::CInt8(42))); + assert!(!int8_ty.has_value(Const::CInt8(-1))); + let neg_int8_ty = Type::from_cint(types::CInt8, -1); + assert!(neg_int8_ty.has_value(Const::CInt8(-1))); + + let int16_ty = Type::from_cint(types::CInt16, 1000); + assert!(int16_ty.has_value(Const::CInt16(1000))); + assert!(!int16_ty.has_value(Const::CInt16(2000))); + + let int32_ty = Type::from_cint(types::CInt32, 100000); + assert!(int32_ty.has_value(Const::CInt32(100000))); + assert!(!int32_ty.has_value(Const::CInt32(-100000))); + + let int64_ty = Type::from_cint(types::CInt64, i64::MAX); + assert!(int64_ty.has_value(Const::CInt64(i64::MAX))); + assert!(!int64_ty.has_value(Const::CInt64(0))); + + let uint8_ty = Type::from_cint(types::CUInt8, u8::MAX as i64); + assert!(uint8_ty.has_value(Const::CUInt8(u8::MAX))); + assert!(!uint8_ty.has_value(Const::CUInt8(0))); + + let uint16_ty = Type::from_cint(types::CUInt16, u16::MAX as i64); + assert!(uint16_ty.has_value(Const::CUInt16(u16::MAX))); + assert!(!uint16_ty.has_value(Const::CUInt16(1))); + + let uint32_ty = Type::from_cint(types::CUInt32, u32::MAX as i64); + assert!(uint32_ty.has_value(Const::CUInt32(u32::MAX))); + assert!(!uint32_ty.has_value(Const::CUInt32(42))); + + let uint64_ty = Type::from_cint(types::CUInt64, i64::MAX); + assert!(uint64_ty.has_value(Const::CUInt64(i64::MAX as u64))); + assert!(!uint64_ty.has_value(Const::CUInt64(123))); + + let shape_ty = Type::from_cint(types::CShape, 0x1234); + assert!(shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + assert!(!shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x5678)))); + + let ptr = 0x1000 as *const u8; + let ptr_ty = Type::from_cptr(ptr); + assert!(ptr_ty.has_value(Const::CPtr(ptr))); + assert!(!ptr_ty.has_value(Const::CPtr(0x2000 as *const u8))); + + let double_ty = Type::from_double(std::f64::consts::PI); + assert!(double_ty.has_value(Const::CDouble(std::f64::consts::PI))); + assert!(!double_ty.has_value(Const::CDouble(3.123))); + + let nan_ty = Type::from_double(f64::NAN); + assert!(nan_ty.has_value(Const::CDouble(f64::NAN))); + + // Mismatched types + assert!(!int8_ty.has_value(Const::CInt16(42))); + assert!(!int16_ty.has_value(Const::CInt32(1000))); + assert!(!uint8_ty.has_value(Const::CInt8(-1i8))); + + // Wrong specialization (unknown value) + assert!(!types::CInt8.has_value(Const::CInt8(42))); + assert!(!types::CBool.has_value(Const::CBool(true))); + assert!(!types::CShape.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + } +} diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs new file mode 100644 index 0000000000..0fa800755d --- /dev/null +++ b/zjit/src/invariants.rs @@ -0,0 +1,543 @@ +//! Code invalidation and patching for speculative optimizations. + +use std::{collections::{HashMap, HashSet}, mem}; + +use crate::{backend::lir::{Assembler, asm_comment}, cruby::{ID, IseqPtr, RedefinitionFlag, VALUE, iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock}, hir::Invariant, options::debug, state::{ZJITState, zjit_enabled_p, trace_invalidation}, virtualmem::CodePtr}; +use crate::payload::{IseqVersionRef, get_or_create_iseq_payload}; +use crate::codegen::invalidate_iseq_version; +use crate::cruby::rb_iseq_reset_jit_func; +use crate::stats::with_time_stat; +use crate::stats::Counter::invalidation_time_ns; +use crate::gc::remove_gc_offsets; + +macro_rules! compile_patch_points { + ($cb:expr, $patch_points:expr, $cause:ident, $($comment_args:tt)*) => { + trace_invalidation(&format!($($comment_args)*), || with_time_stat(invalidation_time_ns, || { + for patch_point in $patch_points { + let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { + let mut asm = Assembler::new(); + asm.new_block_without_id("invalidation"); + asm_comment!(asm, $($comment_args)*); + asm.jmp(patch_point.side_exit_ptr.into()); + asm.compile(cb).expect("can write existing code"); + }); + // Stop marking GC offsets corrupted by the jump instruction + remove_gc_offsets(patch_point.version, &written_range); + + let mut version = patch_point.version; + let iseq = unsafe { version.as_ref() }.iseq; + if !iseq.is_null() { + invalidate_iseq_version($cb, iseq, &mut version); + // Remember NoSingletonClass busts on the payload + if is_no_singleton_class!($cause) { + let payload = get_or_create_iseq_payload(iseq); + payload.was_invalidated_for_singleton_class_creation = true; + } + } + } + })); + }; +} + +macro_rules! is_no_singleton_class { + (NoSingletonClass) => { true }; + ($_:ident) => { false }; +} + +/// When a PatchPoint is invalidated, it generates a jump instruction from `from` to `to`. +#[derive(Debug, Eq, Hash, PartialEq)] +struct PatchPoint { + /// Code pointer to be invalidated + patch_point_ptr: CodePtr, + /// Code pointer to a side exit + side_exit_ptr: CodePtr, + /// ISEQ version to be invalidated + version: IseqVersionRef, +} + +impl PatchPoint { + /// PatchPointer constructor + fn new(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, version: IseqVersionRef) -> PatchPoint { + Self { + patch_point_ptr, + side_exit_ptr, + version, + } + } +} + +/// Used to track all of the various block references that contain assumptions +/// about the state of the virtual machine. +#[derive(Default)] +pub struct Invariants { + /// Set of ISEQs that are known to escape EP + ep_escape_iseqs: HashSet<IseqPtr>, + + /// Map from ISEQ that's assumed to not escape EP to a set of patch points + no_ep_escape_iseq_patch_points: HashMap<IseqPtr, HashSet<PatchPoint>>, + + /// Map from a class and its associated basic operator to a set of patch points + bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet<PatchPoint>>, + + /// Map from CME to patch points that assume the method hasn't been redefined + cme_patch_points: HashMap<*const rb_callable_method_entry_t, HashSet<PatchPoint>>, + + /// Map from constant ID to patch points that assume the constant hasn't been redefined + constant_state_patch_points: HashMap<ID, HashSet<PatchPoint>>, + + /// Set of patch points that assume that the TracePoint is not enabled + no_trace_point_patch_points: HashSet<PatchPoint>, + + /// Set of patch points that assume that the interpreter is running with only one ractor + single_ractor_patch_points: HashSet<PatchPoint>, + + /// Map from a class to a set of patch points that assume objects of the class + /// will have no singleton class. + no_singleton_class_patch_points: HashMap<VALUE, HashSet<PatchPoint>>, + + /// Set of patch points that assume only the root box is active + root_box_patch_points: HashSet<PatchPoint>, + + /// Whether a non-root box has ever been created + non_root_box_created: bool, +} + +impl Invariants { + /// Update object references in Invariants + pub fn update_references(&mut self) { + self.update_ep_escape_iseqs(); + self.update_no_ep_escape_iseq_patch_points(); + self.update_cme_patch_points(); + self.update_no_singleton_class_patch_points(); + } + + /// Forget an ISEQ when freeing it. We need to because a) if the address is reused, we'd be + /// tracking the wrong object b) dead VALUEs in the table can means we risk passing invalid + /// VALUEs to `rb_gc_location()`. + pub fn forget_iseq(&mut self, iseq: IseqPtr) { + // Why not patch the patch points? If the ISEQ is dead then the GC also proved that all + // generated code referencing the ISEQ are unreachable. We mark the ISEQs baked into + // generated code. + self.ep_escape_iseqs.remove(&iseq); + self.no_ep_escape_iseq_patch_points.remove(&iseq); + } + + /// Forget a CME when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_cme(&mut self, cme: *const rb_callable_method_entry_t) { + self.cme_patch_points.remove(&cme); + } + + /// Forget a class when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_klass(&mut self, klass: VALUE) { + self.no_singleton_class_patch_points.remove(&klass); + } + + /// Update ISEQ references in Invariants::ep_escape_iseqs + fn update_ep_escape_iseqs(&mut self) { + let updated = std::mem::take(&mut self.ep_escape_iseqs) + .into_iter() + .map(|iseq| unsafe { rb_gc_location(iseq.into()) }.as_iseq()) + .collect(); + self.ep_escape_iseqs = updated; + } + + /// Update ISEQ references in Invariants::no_ep_escape_iseq_patch_points + fn update_no_ep_escape_iseq_patch_points(&mut self) { + let updated = std::mem::take(&mut self.no_ep_escape_iseq_patch_points) + .into_iter() + .map(|(iseq, patch_points)| { + let new_iseq = unsafe { rb_gc_location(iseq.into()) }; + (new_iseq.as_iseq(), patch_points) + }) + .collect(); + self.no_ep_escape_iseq_patch_points = updated; + } + + fn update_cme_patch_points(&mut self) { + let updated_cme_patch_points = std::mem::take(&mut self.cme_patch_points) + .into_iter() + .map(|(cme, patch_points)| { + let new_cme = unsafe { rb_gc_location(cme.into()) }; + (new_cme.as_cme(), patch_points) + }) + .collect(); + self.cme_patch_points = updated_cme_patch_points; + } + + fn update_no_singleton_class_patch_points(&mut self) { + let updated_no_singleton_class_patch_points = std::mem::take(&mut self.no_singleton_class_patch_points) + .into_iter() + .map(|(klass, patch_points)| { + let new_klass = unsafe { rb_gc_location(klass) }; + (new_klass, patch_points) + }) + .collect(); + self.no_singleton_class_patch_points = updated_no_singleton_class_patch_points; + } +} + +/// Called when a basic operator is redefined. Note that all the blocks assuming +/// the stability of different operators are invalidated together and we don't +/// do fine-grained tracking. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic_operators) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.bop_patch_points.remove(&(klass, bop)) { + let cb = ZJITState::get_code_block(); + let bop = Invariant::BOPRedefined { klass, bop }; + debug!("BOP is redefined: {}", bop); + + // Invalidate all patch points for this BOP + compile_patch_points!(cb, patch_points, BOP, "BOP is redefined: {}", bop); + + cb.mark_all_executable(); + } + }); +} + +/// Invalidate blocks for a given ISEQ that assumes environment pointer is +/// equal to base pointer. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_ep_escape(iseq: IseqPtr) { + // Skip tracking EP escapes on boot. We don't need to invalidate anything during boot. + if !ZJITState::has_instance() { + return; + } + + with_vm_lock(src_loc!(), || { + // Remember that this ISEQ may escape EP + let invariants = ZJITState::get_invariants(); + invariants.ep_escape_iseqs.insert(iseq); + + // If the ISEQ has been compiled assuming it doesn't escape EP, invalidate the JIT code. + if let Some(patch_points) = invariants.no_ep_escape_iseq_patch_points.remove(&iseq) { + debug!("EP is escaped: {}", iseq_name(iseq)); + + // Invalidate the patch points for this ISEQ + let cb = ZJITState::get_code_block(); + compile_patch_points!(cb, patch_points, EP, "EP is escaped: {}", iseq_name(iseq)); + + // Also invalidate the ISEQ version so the method falls back to the + // interpreter on the next call. NoEPEscape PatchPoint side exits use + // without_locals() and don't save locals to the frame. If a PatchPoint + // fires on a later call (where EP hasn't escaped), the interpreter would + // read stale locals (e.g., nil instead of [] for keyword defaults). + // + // We can't use invalidate_iseq_version() here because it skips when + // at MAX_ISEQ_VERSIONS (to prevent unbounded recompilation). Instead, + // directly mark the version as invalidated and reset jit_func so the + // interpreter takes over permanently. + let payload = crate::payload::get_or_create_iseq_payload(iseq); + if let Some(version) = payload.versions.last_mut() { + use crate::payload::IseqStatus; + if unsafe { version.as_ref() }.status != IseqStatus::Invalidated { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(iseq) }; + } + } + + cb.mark_all_executable(); + } + }); +} + +/// Track that JIT code for a ISEQ will assume that base pointer is equal to environment pointer. +pub fn track_no_ep_escape_assumption( + iseq: IseqPtr, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_ep_escape_iseq_patch_points.entry(iseq).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Returns true if a given ISEQ has previously escaped environment pointer. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + ZJITState::get_invariants().ep_escape_iseqs.contains(&iseq) +} + +/// Track a patch point for a basic operator in a given class. +pub fn track_bop_assumption( + klass: RedefinitionFlag, + bop: ruby_basic_operators, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for a callable method entry (CME). +pub fn track_cme_assumption( + cme: *const rb_callable_method_entry_t, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for each constant name in a constant path assumption. +pub fn track_stable_constant_names_assumption( + idlist: *const ID, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + + let mut idx = 0; + loop { + let id = unsafe { *idlist.wrapping_add(idx) }; + if id.0 == 0 { + break; + } + + invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); + + idx += 1; + } +} + +/// Track a patch point for objects of a given class will have no singleton class. +pub fn track_no_singleton_class_assumption( + klass: VALUE, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_singleton_class_patch_points.entry(klass).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Called when a method is redefined. Invalidates all JIT code that depends on the CME. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + // Get the CMD's jumps and remove the entry from the map as it has been invalidated + if let Some(patch_points) = invariants.cme_patch_points.remove(&cme) { + let cb = ZJITState::get_code_block(); + debug!("CME is invalidated: {:?}", cme); + + // Invalidate all patch points for this CME + compile_patch_points!(cb, patch_points, CME, "CME is invalidated: {:?}", cme); + + cb.mark_all_executable(); + } + }); +} + +/// Called when a constant is redefined. Invalidates all JIT code that depends on the constant. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.constant_state_patch_points.remove(&id) { + let cb = ZJITState::get_code_block(); + debug!("Constant state changed: {:?}", id); + + // Invalidate all patch points for this constant ID + compile_patch_points!(cb, patch_points, Const, "Constant state changed: {:?}", id); + + cb.mark_all_executable(); + } + }); +} + +/// Track the JIT code that assumes that the interpreter is running with only one ractor +pub fn track_single_ractor_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.single_ractor_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Callback for when Ruby is about to spawn a ractor. In that case we need to +/// invalidate every block that is assuming single ractor mode. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_before_ractor_spawn() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); + + // Invalidate all patch points for single ractor mode + compile_patch_points!(cb, patch_points, Ractor, "Another ractor spawned, invalidating single ractor mode assumption"); + + cb.mark_all_executable(); + }); +} + +pub fn track_no_trace_point_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_trace_point_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_tracing_invalidate_all() { + use crate::payload::{get_or_create_iseq_payload, IseqStatus}; + use crate::cruby::for_each_iseq; + + if !zjit_enabled_p() { + return; + } + + // Stop other ractors since we are going to patch machine code. + with_vm_lock(src_loc!(), || { + debug!("Invalidating all ZJIT compiled code due to TracePoint"); + + for_each_iseq(|iseq| { + let payload = get_or_create_iseq_payload(iseq); + + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + } + unsafe { rb_iseq_reset_jit_func(iseq) }; + }); + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().no_trace_point_patch_points); + + compile_patch_points!(cb, patch_points, TracePoint, "TracePoint is enabled, invalidating no TracePoint assumption"); + + cb.mark_all_executable(); + }); +} + +/// Track the JIT code that assumes only the root box is active +pub fn track_root_box_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.root_box_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Returns true if a non-root box has ever been created. +pub fn non_root_box_created() -> bool { + ZJITState::get_invariants().non_root_box_created +} + +/// Callback for when a non-root box is created. In that case we need to +/// invalidate every block that assumes root-box-only mode. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_root_box() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + invariants.non_root_box_created = true; + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut invariants.root_box_patch_points); + + // Invalidate all patch points for root box mode + compile_patch_points!(cb, patch_points, Box, "Non-root box created, invalidating root box assumption"); + + cb.mark_all_executable(); + }); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +/// This is used to avoid an invalidation loop where we repeatedly compile code +/// that assumes no singleton class, only to have it invalidated. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + ZJITState::get_invariants() + .no_singleton_class_patch_points + .get(&klass) + .map_or(false, |patch_points| patch_points.is_empty()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_singleton_class(klass: VALUE) { + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + match invariants.no_singleton_class_patch_points.get_mut(&klass) { + Some(patch_points) => { + // Invalidate existing patch points and let has_singleton_class_of() + // return true when they are compiled again + let patch_points = mem::take(patch_points); + if !patch_points.is_empty() { + let cb = ZJITState::get_code_block(); + debug!("Singleton class created for {:?}", klass); + compile_patch_points!(cb, patch_points, NoSingletonClass, "Singleton class created for {:?}", klass); + cb.mark_all_executable(); + } + } + None => { + // Let has_singleton_class_of() return true for this class + invariants.no_singleton_class_patch_points.insert(klass, HashSet::new()); + } + } + }); +} diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs new file mode 100644 index 0000000000..b434d0a8ed --- /dev/null +++ b/zjit/src/jit_frame.rs @@ -0,0 +1,313 @@ +use crate::cruby::{IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; +use crate::cruby::zjit_jit_frame; +use crate::state::ZJITState; + +/// JITFrame struct is defined in zjit.h (the single source of truth) and +/// imported into Rust via bindgen. See zjit.h for field documentation. +pub type JITFrame = zjit_jit_frame; + +impl JITFrame { + /// Allocate a JITFrame on the heap, register it with ZJITState, and return + /// a raw pointer that remains valid for the lifetime of the process. + fn alloc(jit_frame: JITFrame) -> *const Self { + let raw_ptr = Box::into_raw(Box::new(jit_frame)); + ZJITState::get_jit_frames().push(raw_ptr); + raw_ptr as *const _ + } + + /// Create a JITFrame for an ISEQ frame. + pub fn new_iseq(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { + Self::alloc(JITFrame { pc, iseq, materialize_block_code }) + } + + /// Mark the iseq pointer for GC. Called from rb_zjit_root_mark. + pub fn mark(&self) { + if !self.iseq.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from(self.iseq)); } + } + } + + /// Update the iseq pointer after GC compaction. + pub fn update_references(&mut self) { + if !self.iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(self.iseq)) }.as_iseq(); + if self.iseq != new_iseq { + self.iseq = new_iseq; + } + } + } +} + +/// Update the iseq pointer in an on-stack JITFrame during GC compaction. +/// Called from rb_execution_context_update in vm.c. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_frame_update_references(jit_frame: *mut JITFrame) { + unsafe { &mut *jit_frame }.update_references(); +} + +#[cfg(test)] +mod tests { + use crate::cruby::{eval, inspect}; + use insta::assert_snapshot; + + #[test] + fn test_jit_frame_entry_first() { + eval(r#" + def test + itself + callee + end + + def callee + caller + end + + test + "#); + assert_snapshot!(inspect("test.first"), @r#""<compiled>:4:in 'Object#test'""#); + } + + #[test] + fn test_materialize_one_frame() { + assert_snapshot!(inspect(" + def jit_entry + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + #[test] + fn test_materialize_two_frames() { // materialize caller frames on raise + // At the point of `resuce`, there are two lightweight frames on stack and both need to be + // materialized before passing control to interpreter. + assert_snapshot!(inspect(" + def jit_entry = raise_and_rescue + def raise_and_rescue + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + // Materialize frames on side exit: a type guard triggers a side exit with + // multiple JIT frames on the stack. All frames must be materialized before + // the interpreter resumes. + #[test] + fn test_side_exit_materialize_frames() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); + } + + // BOP invalidation must not overwrite the top-most frame's PC with + // jit_frame's PC. After invalidation the interpreter resumes at a new + // PC, so a stale jit_frame PC would cause wrong execution. + #[test] + fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); + } + + // Side exit at the very start of a method, before any jit_return has been + // written by gen_save_pc_for_gc. The jit_return field should be 0 (from + // vm_push_frame), so materialization should be a no-op for that frame. + #[test] + fn test_side_exit_before_jit_return_write() { + assert_snapshot!(inspect(" + def entry(n) = n + 1 + entry(1) + [entry(1), entry(1.0)] + "), @"[2, 2.0]"); + } + + #[test] + fn test_caller_iseq() { + assert_snapshot!(inspect(r#" + def callee = call_caller + def test = callee + + def callee2 = call_caller + def test2 = callee2 + + def call_caller = caller + + test + test2 + test.first + "#), @r#""<compiled>:2:in 'Object#callee'""#); + } + + // ISEQ must be readable during exception handling so the interpreter + // can look up rescue/ensure tables. + #[test] + fn test_iseq_on_raise() { + assert_snapshot!(inspect(r#" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "#), @"(0/1)..1"); + } + + // Multiple exception raises during keyword argument evaluation: each + // raise needs correct ISEQ for catch table lookup. + #[test] + fn test_iseq_on_raise_on_ensure() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); + } + + // Send fallback (e.g. method_missing) calls into the interpreter, which + // reads cfp->iseq via GET_ISEQ(). gen_prepare_non_leaf_call writes the + // iseq to JITFrame, but GET_ISEQ reads cfp->iseq directly. This test + // ensures the interpreter can resolve the caller iseq for backtraces. + #[test] + fn test_send_fallback_caller_location() { + assert_snapshot!(inspect(r#" + def callee = caller_locations(1, 1)[0].label + def test = callee + test + test + "#), @r#""Object#test""#); + } + + // A send fallback may throw (e.g. via method_missing raising). The + // interpreter must be able to find the correct rescue handler in the + // caller's ISEQ catch table. This exercises throw through send fallback. + #[test] + fn test_send_fallback_throw() { + assert_snapshot!(inspect(r#" + class Foo + def method_missing(name, *) = raise("no #{name}") + end + def test + Foo.new.bar + rescue RuntimeError => e + e.message + end + test + test + "#), @r#""no bar""#); + } + + // Proc.new inside a block passed via invokeblock captures the caller's + // block_code. When the JIT compiles the caller, block_code must be + // correctly available for the proc to work. + #[test] + fn test_proc_from_invokeblock() { + assert_snapshot!(inspect(" + def capture_block(&blk) = blk + def test = capture_block { 42 } + test + test.call + "), @"42"); + } + + // binding() called from a JIT-compiled callee must see the correct + // source location (iseq + pc) of the caller frame. + #[test] + fn test_binding_source_location() { + assert_snapshot!(inspect(r#" + def callee = binding + def test = callee + test + b = test + b.source_location[1] > 0 + "#), @"true"); + } + + // $~ (Regexp special variable) is stored via svar which walks the EP + // chain to find the LEP. rb_vm_svar_lep uses rb_zjit_cfp_has_iseq to + // skip C frames, so it must work correctly with JITFrame. + #[test] + fn test_svar_regexp_match() { + assert_snapshot!(inspect(r#" + def test(s) + s =~ /hello/ + $~ + end + test("hello world") + test("hello world").to_s + "#), @r#""hello""#); + } + + // C function calls with rb_block_call (like Array#each, Enumerable#map) + // write an ifunc to cfp->block_code after the JIT pushes the C frame. + // GC must mark and relocate this ifunc. This test exercises the code + // path fixed by "Fix ZJIT segfault: write block_code for C frames and + // fix GC marking". + #[test] + fn test_cfunc_block_code_gc() { + assert_snapshot!(inspect(" + def test + # Use a cfunc that calls back into Ruby with a block (rb_block_call) + [1, 2, 3].map { |x| x.to_s } + end + test + test + "), @r#"["1", "2", "3"]"#); + } + + // Multiple levels of cfunc-with-block: a JIT-compiled method calls a + // cfunc that yields, and the block itself calls another cfunc that + // yields. Each C frame's block_code must be properly initialized. + #[test] + fn test_nested_cfunc_with_block() { + assert_snapshot!(inspect(" + def test + [1, 2].flat_map { |x| [x, x + 10].map { |y| y * 2 } } + end + test + test + "), @"[2, 22, 4, 24]"); + } +} diff --git a/zjit/src/json.rs b/zjit/src/json.rs new file mode 100644 index 0000000000..fa4b216821 --- /dev/null +++ b/zjit/src/json.rs @@ -0,0 +1,700 @@ +//! Single file JSON serializer for iongraph output of ZJIT HIR. + +use std::{ + fmt, + io::{self, Write}, +}; + +pub trait Jsonable { + fn to_json(&self) -> Json; +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Json { + Null, + Bool(bool), + Integer(isize), + UnsignedInteger(usize), + Floating(f64), + String(String), + Array(Vec<Json>), + Object(Vec<(String, Json)>), +} + +impl Json { + /// Convenience method for constructing a JSON array. + pub fn array<I, T>(iter: I) -> Self + where + I: IntoIterator<Item = T>, + T: Into<Json>, + { + Json::Array(iter.into_iter().map(Into::into).collect()) + } + + pub fn empty_array() -> Self { + Json::Array(Vec::new()) + } + + pub fn object() -> JsonObjectBuilder { + JsonObjectBuilder::new() + } + + pub fn marshal<W: Write>(&self, writer: &mut W) -> JsonResult<()> { + match self { + Json::Null => writer.write_all(b"null"), + Json::Bool(b) => writer.write_all(if *b { b"true" } else { b"false" }), + Json::Integer(i) => write!(writer, "{i}"), + Json::UnsignedInteger(u) => write!(writer, "{u}"), + Json::Floating(f) => write!(writer, "{f}"), + Json::String(s) => return Self::write_str(writer, s), + Json::Array(jsons) => return Self::write_array(writer, jsons), + Json::Object(map) => return Self::write_object(writer, map), + }?; + Ok(()) + } + + pub fn write_str<W: Write>(writer: &mut W, s: &str) -> JsonResult<()> { + writer.write_all(b"\"")?; + + for ch in s.chars() { + match ch { + '"' => write!(writer, "\\\"")?, + '\\' => write!(writer, "\\\\")?, + // The following characters are control, but have a canonical representation. + // https://datatracker.ietf.org/doc/html/rfc8259#section-7 + '\n' => write!(writer, "\\n")?, + '\r' => write!(writer, "\\r")?, + '\t' => write!(writer, "\\t")?, + '\x08' => write!(writer, "\\b")?, + '\x0C' => write!(writer, "\\f")?, + ch if ch.is_control() => { + let code_point = ch as u32; + write!(writer, "\\u{code_point:04X}")? + } + _ => write!(writer, "{ch}")?, + }; + } + + writer.write_all(b"\"")?; + Ok(()) + } + + pub fn write_array<W: Write>(writer: &mut W, jsons: &[Json]) -> JsonResult<()> { + writer.write_all(b"[")?; + let mut prefix = ""; + for item in jsons { + write!(writer, "{prefix}")?; + item.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"]")?; + Ok(()) + } + + pub fn write_object<W: Write>(writer: &mut W, pairs: &[(String, Json)]) -> JsonResult<()> { + writer.write_all(b"{")?; + let mut prefix = ""; + for (k, v) in pairs { + // Escape the keys, despite not being `Json::String` objects. + write!(writer, "{prefix}")?; + Self::write_str(writer, k)?; + writer.write_all(b":")?; + v.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"}")?; + Ok(()) + } +} + +impl std::fmt::Display for Json { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut buf = Vec::new(); + self.marshal(&mut buf).map_err(|_| std::fmt::Error)?; + let s = String::from_utf8(buf).map_err(|_| std::fmt::Error)?; + write!(f, "{s}") + } +} + +pub struct JsonObjectBuilder { + pairs: Vec<(String, Json)>, +} + +impl JsonObjectBuilder { + pub fn new() -> Self { + Self { pairs: Vec::new() } + } + + pub fn insert<K, V>(mut self, key: K, value: V) -> Self + where + K: Into<String>, + V: Into<Json>, + { + self.pairs.push((key.into(), value.into())); + self + } + + pub fn build(self) -> Json { + Json::Object(self.pairs) + } +} + +impl From<&str> for Json { + fn from(s: &str) -> Json { + Json::String(s.to_string()) + } +} + +impl From<String> for Json { + fn from(s: String) -> Json { + Json::String(s) + } +} + +impl From<i32> for Json { + fn from(i: i32) -> Json { + Json::Integer(i as isize) + } +} + +impl From<i64> for Json { + fn from(i: i64) -> Json { + Json::Integer(i as isize) + } +} + +impl From<u32> for Json { + fn from(u: u32) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<u64> for Json { + fn from(u: u64) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<usize> for Json { + fn from(u: usize) -> Json { + Json::UnsignedInteger(u) + } +} + +impl From<bool> for Json { + fn from(b: bool) -> Json { + Json::Bool(b) + } +} + +impl TryFrom<f64> for Json { + type Error = JsonError; + fn try_from(f: f64) -> Result<Self, Self::Error> { + if f.is_finite() { + Ok(Json::Floating(f)) + } else { + Err(JsonError::FloatError(f)) + } + } +} + +impl<T: Into<Json>> From<Vec<T>> for Json { + fn from(v: Vec<T>) -> Json { + Json::Array(v.into_iter().map(|item| item.into()).collect()) + } +} + +/// Convenience type for a result in JSON serialization. +pub type JsonResult<W> = std::result::Result<W, JsonError>; + +#[derive(Debug)] +pub enum JsonError { + /// Wrapper for a standard `io::Error`. + IoError(io::Error), + /// On attempting to serialize an invalid `f32` or `f64`. + /// Stores invalid values as 64 bit float. + FloatError(f64), +} + +impl From<io::Error> for JsonError { + fn from(err: io::Error) -> Self { + JsonError::IoError(err) + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::FloatError(v) => write!(f, "Cannot serialize float {v}"), + JsonError::IoError(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for JsonError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JsonError::IoError(e) => Some(e), + JsonError::FloatError(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + + fn marshal_to_string(json: &Json) -> String { + let mut buf = Vec::new(); + json.marshal(&mut buf).unwrap(); + String::from_utf8(buf).unwrap() + } + + #[test] + fn test_null() { + let json = Json::Null; + assert_snapshot!(marshal_to_string(&json), @"null"); + } + + #[test] + fn test_bool() { + let json: Json = true.into(); + assert_snapshot!(marshal_to_string(&json), @"true"); + let json: Json = false.into(); + assert_snapshot!(marshal_to_string(&json), @"false"); + } + + #[test] + fn test_integer_positive() { + let json: Json = 42.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_integer_negative() { + let json: Json = (-123).into(); + assert_snapshot!(marshal_to_string(&json), @"-123"); + } + + #[test] + fn test_integer_zero() { + let json: Json = 0.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_floating() { + let json = 2.14159.try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"2.14159"); + } + + #[test] + fn test_floating_negative() { + let json = (-2.5).try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"-2.5"); + } + + #[test] + fn test_floating_error() { + let json: Result<Json, JsonError> = f64::NAN.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::NEG_INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + } + + #[test] + fn test_string_simple() { + let json: Json = "hello".into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello""#); + } + + #[test] + fn test_string_empty() { + let json: Json = "".into(); + assert_snapshot!(marshal_to_string(&json), @r#""""#); + } + + #[test] + fn test_string_with_quotes() { + let json: Json = r#"hello "world""#.into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello \"world\"""#); + } + + #[test] + fn test_string_with_backslash() { + let json: Json = r"path\to\file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path\\to\\file""#); + } + + #[test] + fn test_string_with_slash() { + let json: Json = "path/to/file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path/to/file""#); + } + + #[test] + fn test_string_with_newline() { + let json: Json = "line1\nline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\nline2""#); + } + + #[test] + fn test_string_with_carriage_return() { + let json: Json = "line1\rline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\rline2""#); + } + + #[test] + fn test_string_with_tab() { + let json: Json = "col1\tcol2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""col1\tcol2""#); + } + + #[test] + fn test_string_with_backspace() { + let json: Json = "text\x08back".into(); + assert_snapshot!(marshal_to_string(&json), @r#""text\bback""#); + } + + #[test] + fn test_string_with_form_feed() { + let json: Json = "page\x0Cnew".into(); + assert_snapshot!(marshal_to_string(&json), @r#""page\fnew""#); + } + + #[test] + fn test_string_with_control_chars() { + let json: Json = "test\x01\x02\x03".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test\u0001\u0002\u0003""#); + } + + #[test] + fn test_string_with_all_escapes() { + let json: Json = "\"\\/\n\r\t\x08\x0C".into(); + assert_snapshot!(marshal_to_string(&json), @r#""\"\\/\n\r\t\b\f""#); + } + + #[test] + fn test_array_empty() { + let json: Json = Vec::<i32>::new().into(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_array_single_element() { + let json: Json = vec![42].into(); + assert_snapshot!(marshal_to_string(&json), @"[42]"); + } + + #[test] + fn test_array_multiple_elements() { + let json: Json = vec![1, 2, 3].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_array_mixed_types() { + let json = Json::Array(vec![ + Json::Null, + true.into(), + 42.into(), + 3.134.try_into().unwrap(), + "hello".into(), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"[null, true, 42, 3.134, "hello"]"#); + } + + #[test] + fn test_array_nested() { + let json = Json::Array(vec![1.into(), vec![2, 3].into(), 4.into()]); + assert_snapshot!(marshal_to_string(&json), @"[1, [2, 3], 4]"); + } + + #[test] + fn test_object_empty() { + let json = Json::Object(vec![]); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_single_field() { + let json = Json::Object(vec![("key".to_string(), "value".into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_multiple_fields() { + let json = Json::Object(vec![ + ("name".to_string(), "Alice".into()), + ("age".to_string(), 30.into()), + ("active".to_string(), true.into()), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_with_escaped_key() { + let json = Json::Object(vec![("key\nwith\nnewlines".to_string(), 42.into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key\nwith\nnewlines":42}"#); + } + + #[test] + fn test_object_nested() { + let inner = Json::Object(vec![("inner_key".to_string(), "inner_value".into())]); + let json = Json::Object(vec![("outer_key".to_string(), inner)]); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_from_str() { + let json: Json = "test string".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test string""#); + } + + #[test] + fn test_from_i32() { + let json: Json = 42i32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_i64() { + let json: Json = 9223372036854775807i64.into(); + assert_snapshot!(marshal_to_string(&json), @"9223372036854775807"); + } + + #[test] + fn test_from_u32() { + let json: Json = 42u32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_u64() { + let json: Json = 18446744073709551615u64.into(); + assert_snapshot!(marshal_to_string(&json), @"18446744073709551615"); + } + + #[test] + fn test_unsigned_integer_zero() { + let json: Json = 0u64.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_from_bool() { + let json_true: Json = true.into(); + let json_false: Json = false.into(); + assert_snapshot!(marshal_to_string(&json_true), @"true"); + assert_snapshot!(marshal_to_string(&json_false), @"false"); + } + + #[test] + fn test_from_vec() { + let json: Json = vec![1i32, 2i32, 3i32].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_from_vec_strings() { + let json: Json = vec!["a", "b", "c"].into(); + assert_snapshot!(marshal_to_string(&json), @r#"["a", "b", "c"]"#); + } + + #[test] + fn test_complex_nested_structure() { + let settings = Json::Object(vec![ + ("notifications".to_string(), true.into()), + ("theme".to_string(), "dark".into()), + ]); + + let json = Json::Object(vec![ + ("id".to_string(), 1.into()), + ("name".to_string(), "Alice".into()), + ("tags".to_string(), vec!["admin", "user"].into()), + ("settings".to_string(), settings), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"id":1, "name":"Alice", "tags":["admin", "user"], "settings":{"notifications":true, "theme":"dark"}}"#); + } + + #[test] + fn test_deeply_nested_arrays() { + let json = Json::Array(vec![ + Json::Array(vec![vec![1, 2].into(), 3.into()]), + 4.into(), + ]); + assert_snapshot!(marshal_to_string(&json), @"[[[1, 2], 3], 4]"); + } + + #[test] + fn test_unicode_string() { + let json: Json = "兵马俑".into(); + assert_snapshot!(marshal_to_string(&json), @r#""兵马俑""#); + } + + #[test] + fn test_json_array_convenience() { + let json = Json::array(vec![1, 2, 3]); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_json_array_from_iterator() { + let json = Json::array([1, 2, 3].iter().map(|&x| x * 2)); + assert_snapshot!(marshal_to_string(&json), @"[2, 4, 6]"); + } + + #[test] + fn test_json_empty_array() { + let json = Json::empty_array(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_object_builder_empty() { + let json = Json::object().build(); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_builder_single_field() { + let json = Json::object().insert("key", "value").build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_builder_multiple_fields() { + let json = Json::object() + .insert("name", "Alice") + .insert("age", 30) + .insert("active", true) + .build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_builder_with_nested_objects() { + let inner = Json::object().insert("inner_key", "inner_value").build(); + let json = Json::object().insert("outer_key", inner).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_object_builder_with_array() { + let json = Json::object().insert("items", vec![1, 2, 3]).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"items":[1, 2, 3]}"#); + } + + #[test] + fn test_display_trait() { + let json = Json::object() + .insert("name", "Bob") + .insert("count", 42) + .build(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#"{"name":"Bob", "count":42}"#); + } + + #[test] + fn test_display_trait_array() { + let json: Json = vec![1, 2, 3].into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @"[1, 2, 3]"); + } + + #[test] + fn test_display_trait_string() { + let json: Json = "test".into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#""test""#); + } + + #[test] + fn test_from_usize() { + let json: Json = 123usize.into(); + assert_snapshot!(marshal_to_string(&json), @"123"); + } + + #[test] + fn test_from_usize_large() { + let json: Json = usize::MAX.into(); + let expected = format!("{}", usize::MAX); + assert_eq!(marshal_to_string(&json), expected); + } + + #[test] + fn test_json_error_float_display() { + let err = JsonError::FloatError(f64::NAN); + let display_output = format!("{}", err); + assert!(display_output.contains("Cannot serialize float")); + assert!(display_output.contains("NaN")); + } + + #[test] + fn test_json_error_float_display_infinity() { + let err = JsonError::FloatError(f64::INFINITY); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"Cannot serialize float inf"); + } + + #[test] + fn test_json_error_io_display() { + let io_err = io::Error::new(io::ErrorKind::WriteZero, "write error"); + let err = JsonError::IoError(io_err); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"write error"); + } + + #[test] + fn test_io_error_during_marshal() { + struct FailingWriter; + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result<usize> { + Err(io::Error::other("simulated write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let json: Json = "test".into(); + let mut writer = FailingWriter; + let result = json.marshal(&mut writer); + assert!(result.is_err()); + assert!(matches!(result, Err(JsonError::IoError(_)))); + } + + #[test] + fn test_clone_json() { + let json1: Json = vec![1, 2, 3].into(); + let json2 = json1.clone(); + assert_eq!(json1, json2); + } + + #[test] + fn test_debug_json() { + let json: Json = "test".into(); + let debug_output = format!("{:?}", json); + assert!(debug_output.contains("String")); + assert!(debug_output.contains("test")); + } + + #[test] + fn test_partial_eq_json() { + let json1: Json = 42.into(); + let json2: Json = 42.into(); + let json3: Json = 43.into(); + assert_eq!(json1, json2); + assert_ne!(json1, json3); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs new file mode 100644 index 0000000000..1440b6ff69 --- /dev/null +++ b/zjit/src/lib.rs @@ -0,0 +1,46 @@ +#![allow(dead_code)] +#![allow(static_mut_refs)] + +#![allow(clippy::enum_variant_names)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::needless_bool)] + +// Add std docs to cargo doc. +#[doc(inline)] +pub use std; + +mod state; +mod distribution; +mod cruby; +mod cruby_methods; +mod hir; +mod hir_type; +mod hir_effect; +mod codegen; +mod stats; +mod cast; +mod virtualmem; +mod asm; +mod backend; +#[cfg(feature = "disasm")] +mod disasm; +mod options; +mod profile; +mod invariants; +mod bitset; +mod gc; +mod jit_frame; +mod payload; +mod json; +mod ttycolors; + +/// Pull in YJIT's symbols for linking the test binary in `make zjit-test`. The test binary builds +/// ZJIT symbols and they should take precendence over the ones built for miniruby, so libminiruby +/// doesn't include any ZJIT code. But, in removing from libminiruby the object which contains all +/// rust code, including ZJIT code, we also remove all YJIT symbols which the rest of libminiruby +/// might request in YJIT+ZJIT configurations. We add back the YJIT symbols here. +/// +/// Only relevant for YJIT+ZJIT configurations, but building YJIT is fast, so always do it for the +/// test binary for simplicity. +#[cfg(test)] +use yjit as _; diff --git a/zjit/src/options.rs b/zjit/src/options.rs new file mode 100644 index 0000000000..5ddaee1951 --- /dev/null +++ b/zjit/src/options.rs @@ -0,0 +1,631 @@ +//! Configurable options for ZJIT. + +use std::{ffi::{CStr, CString}, fs::File, ptr::null}; +use std::os::raw::{c_char, c_int, c_uint}; +use crate::cruby::*; +use crate::stats::Counter; +use std::collections::HashSet; + +/// Type of symbols to dump into /tmp/perf-{pid}.map +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PerfMap { + /// Dump one symbol per ISEQ + ISEQ, + /// Dump one symbol per HIR instruction + HIR, +} + +/// Default --zjit-num-profiles +const DEFAULT_NUM_PROFILES: NumProfiles = 5; +pub type NumProfiles = u16; + +/// Default --zjit-call-threshold. This should be large enough to avoid compiling +/// warmup code, but small enough to perform well on micro-benchmarks. +pub const DEFAULT_CALL_THRESHOLD: CallThreshold = 30; +pub type CallThreshold = u64; + +/// Number of calls to start profiling YARV instructions. +/// They are profiled `rb_zjit_call_threshold - rb_zjit_profile_threshold` times, +/// which is equal to --zjit-num-profiles. +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +pub static mut rb_zjit_profile_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD - DEFAULT_NUM_PROFILES as CallThreshold; + +/// Number of calls to compile ISEQ with ZJIT at jit_compile() in vm.c. +/// --zjit-call-threshold=1 compiles on first execution without profiling information. +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +pub static mut rb_zjit_call_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD; + +/// ZJIT command-line options. This is set before rb_zjit_init() sets +/// ZJITState so that we can query some options while loading builtins. +pub static mut OPTIONS: Option<Options> = None; + +#[derive(Clone, Debug)] +pub struct Options { + /// Hard limit of the executable memory block to allocate in bytes. + /// Note that the command line argument is expressed in MiB and not bytes. + pub exec_mem_bytes: usize, + + /// Hard limit of ZJIT's total memory usage. + /// Note that the command line argument is expressed in MiB and not bytes. + pub mem_bytes: usize, + + /// Number of times YARV instructions should be profiled. + pub num_profiles: NumProfiles, + + /// Enable ZJIT statistics + pub stats: bool, + + /// Print stats on exit (when stats is also true) + pub print_stats: bool, + + /// Print stats to file on exit (when stats is also true) + pub print_stats_file: Option<std::path::PathBuf>, + + /// Enable debug logging + pub debug: bool, + + // Whether to enable JIT at boot. This option prevents other + // ZJIT tuning options from enabling ZJIT at boot. + pub disable: bool, + + /// Turn off the HIR optimizer + pub disable_hir_opt: bool, + + /// Dump initial High-level IR before optimization + pub dump_hir_init: Option<DumpHIR>, + + /// Dump High-level IR after optimization, right before codegen. + pub dump_hir_opt: Option<DumpHIR>, + + /// Dump High-level IR to the given file in Graphviz format after optimization + pub dump_hir_graphviz: Option<std::path::PathBuf>, + + /// Dump High-level IR in Iongraph JSON format after optimization to /tmp/zjit-iongraph-{$PID} + pub dump_hir_iongraph: bool, + + /// Dump low-level IR + pub dump_lir: Option<HashSet<DumpLIR>>, + + /// Dump all compiled machine code. + pub dump_disasm: Option<DumpDisasm>, + + /// Trace and write side exit source maps to /tmp for stackprof. + pub trace_side_exits: Option<TraceExits>, + + /// Frequency of tracing side exits. + pub trace_side_exits_sample_interval: usize, + + /// Trace compilation phases as Perfetto duration events. + pub trace_compiles: bool, + + /// Trace invalidation events as Perfetto duration events. + pub trace_invalidation: bool, + + /// Dump code map to /tmp for performance profilers. + pub perf: Option<PerfMap>, + + /// List of ISEQs that can be compiled, identified by their iseq_get_location() + pub allowed_iseqs: Option<HashSet<String>>, + + /// Path to a file where compiled ISEQs will be saved. + pub log_compiled_iseqs: Option<std::path::PathBuf>, + + /// Maximum number of versions per ISEQ + pub max_versions: usize, +} + +impl Default for Options { + fn default() -> Self { + Options { + exec_mem_bytes: 64 * 1024 * 1024, + mem_bytes: 128 * 1024 * 1024, + num_profiles: DEFAULT_NUM_PROFILES, + stats: false, + print_stats: false, + print_stats_file: None, + debug: false, + disable: false, + disable_hir_opt: false, + dump_hir_init: None, + dump_hir_opt: None, + dump_hir_graphviz: None, + dump_hir_iongraph: false, + dump_lir: None, + dump_disasm: None, + trace_side_exits: None, + trace_side_exits_sample_interval: 0, + trace_compiles: false, + trace_invalidation: false, + perf: None, + allowed_iseqs: None, + log_compiled_iseqs: None, + max_versions: 2, + } + } +} + +/// `ruby --help` descriptions for user-facing options. Do not add options for ZJIT developers. +/// Note that --help allows only 80 chars per line, including indentation, and it also puts the +/// description in a separate line if the option name is too long. 80-char limit --> | (any character beyond this `|` column fails the test) +pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ + ("--zjit-mem-size=num", + "Max amount of memory that ZJIT can use in MiB (default: 128)."), + ("--zjit-call-threshold=num", + "Number of calls to trigger JIT (default: 30)."), + ("--zjit-num-profiles=num", + "Number of profiled calls before JIT (default: 5)."), + ("--zjit-stats-quiet", + "Collect ZJIT stats and suppress output."), + ("--zjit-stats[=file]", + "Collect ZJIT stats (=file to write to a file)."), + ("--zjit-disable", + "Disable ZJIT for lazily enabling it with RubyVM::ZJIT.enable."), + ("--zjit-perf[=iseq|hir]", + "Dump symbols for Linux perf /tmp/perf-{}.map (default: iseq)."), + ("--zjit-log-compiled-iseqs=path", + "Log compiled ISEQs to the file. The file will be truncated."), + ("--zjit-trace-exits[=counter]", + "Record source on side-exit. `Counter` picks specific counter."), + ("--zjit-trace-exits-sample-rate=num", + "Frequency at which to record side exits. Must be `usize`."), + ("--zjit-trace-compiles", + "Record compilation phases as Perfetto trace events."), + ("--zjit-trace-invalidation", + "Record invalidation events as Perfetto trace events."), +]; + +#[derive(Copy, Clone, Debug)] +pub enum TraceExits { + // Trace all exits + All, + // Trace exits for a specific `Counter` + Counter(Counter), +} + +#[derive(Clone, Copy, Debug)] +pub enum DumpHIR { + // Dump High-level IR without Snapshot + WithoutSnapshot, + // Dump High-level IR with Snapshot + All, + // Pretty-print bare High-level IR structs + Debug, +} + +/// --zjit-dump-lir values. Using snake_case to stringify the exact filter value. +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum DumpLIR { + /// Dump the initial LIR + init, + /// Dump LIR after {arch}_split + split, + /// Dump LIR after alloc_regs + alloc_regs, + /// Dump LIR after compile_exits + compile_exits, + /// Dump LIR after resolve_parallel_mov + resolve_parallel_mov, + /// Dump LIR after {arch}_scratch_split + scratch_split, + /// Dump live intervals grid before alloc_regs + live_intervals, +} + +#[derive(Clone, Copy, Debug)] +pub enum DumpDisasm { + Stdout, + File(std::os::unix::io::RawFd), +} + +/// All compiler stages for --zjit-dump-lir=all. +const DUMP_LIR_ALL: &[DumpLIR] = &[ + DumpLIR::init, + DumpLIR::split, + DumpLIR::alloc_regs, + DumpLIR::compile_exits, + DumpLIR::resolve_parallel_mov, + DumpLIR::scratch_split, + DumpLIR::live_intervals, +]; + +/// Maximum value for --zjit-mem-size/--zjit-exec-mem-size in MiB. +/// We set 1TiB just to avoid overflow. We could make it smaller. +const MAX_MEM_MIB: usize = 1024 * 1024; + +/// Macro to dump LIR if --zjit-dump-lir is specified +macro_rules! asm_dump { + ($asm:expr, $target:ident) => { + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::$target) { + println!("LIR {}:\n{}", stringify!($target), $asm); + } + } + }; +} +pub(crate) use asm_dump; + +/// Macro to get an option value by name +macro_rules! get_option { + // Unsafe is ok here because options are initialized + // once before any Ruby code executes + ($option_name:ident) => { + unsafe { crate::options::OPTIONS.as_ref() }.unwrap().$option_name + }; +} +pub(crate) use get_option; + +/// Macro to reference an option value by name. +macro_rules! get_option_ref { + ($option_name:ident) => { + unsafe { crate::options::OPTIONS.as_ref() }.unwrap().$option_name.as_ref() + }; +} +pub(crate) use get_option_ref; + +/// Set default values to ZJIT options. Setting Some to OPTIONS will make `#with_jit` +/// enable the JIT hook while not enabling compilation yet. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_prepare_options() { + // rb_zjit_prepare_options() could be called for feature flags or $RUBY_ZJIT_ENABLE + // after rb_zjit_parse_option() is called, so we need to handle the already-initialized case. + if unsafe { OPTIONS.is_none() } { + unsafe { OPTIONS = Some(Options::default()); } + } +} + +/// Parse a --zjit* command-line flag +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_parse_option(str_ptr: *const c_char) -> bool { + parse_option(str_ptr).is_some() +} + +fn parse_jit_list(path_like: &str) -> HashSet<String> { + // Read lines from the file + let mut result = HashSet::new(); + if let Ok(lines) = std::fs::read_to_string(path_like) { + for line in lines.lines() { + let trimmed = line.trim(); + if !trimmed.is_empty() { + result.insert(trimmed.to_string()); + } + } + } else { + eprintln!("Failed to read JIT list from '{path_like}'"); + } + eprintln!("JIT list:"); + for item in &result { + eprintln!(" {item}"); + } + result +} + +/// Expected to receive what comes after the third dash in "--zjit-*". +/// Empty string means user passed only "--zjit". C code rejects when +/// they pass exact "--zjit-". +fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { + rb_zjit_prepare_options(); + let options = unsafe { OPTIONS.as_mut().unwrap() }; + + let c_str: &CStr = unsafe { CStr::from_ptr(str_ptr) }; + let opt_str: &str = c_str.to_str().ok()?; + + // Split the option name and value strings + // Note that some options do not contain an assignment + let parts = opt_str.split_once('='); + let (opt_name, opt_val) = match parts { + Some((before_eq, after_eq)) => (before_eq, after_eq), + None => (opt_str, ""), + }; + + // Match on the option name and value strings + match (opt_name, opt_val) { + ("", "") => {}, // Simply --zjit + + ("mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + + ("exec-mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.exec_mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + + ("call-threshold", _) => match opt_val.parse() { + Ok(n) => { + unsafe { rb_zjit_call_threshold = n; } + update_profile_threshold(); + }, + Err(_) => return None, + }, + + ("num-profiles", _) => match opt_val.parse() { + Ok(n) => { + options.num_profiles = n; + update_profile_threshold(); + }, + Err(_) => return None, + }, + + ("max-versions", _) => match opt_val.parse() { + Ok(n) => options.max_versions = n, + Err(_) => return None, + }, + + ("stats-quiet", _) => { + options.stats = true; + options.print_stats = false; + } + + ("stats", "") => { + options.stats = true; + options.print_stats = true; + } + ("stats", path) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path) + .map_err(|e| eprintln!("Failed to open file '{}': {}", path, e)) + .ok(); + let canonical_path = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.stats = true; + options.print_stats_file = Some(canonical_path); + } + + ("trace-exits", exits) => { + options.trace_side_exits = match exits { + "" => Some(TraceExits::All), + name => Some(Counter::get(name).map(TraceExits::Counter)?), + } + } + + ("trace-exits-sample-rate", sample_interval) => { + // If not already set, then set it to `TraceExits::All` by default. + if options.trace_side_exits.is_none() { + options.trace_side_exits = Some(TraceExits::All); + } + // `sample_interval ` must provide a string that can be validly parsed to a `usize`. + options.trace_side_exits_sample_interval = sample_interval.parse::<usize>().ok()?; + } + + ("trace-compiles", "") => options.trace_compiles = true, + + ("trace-invalidation", "") => options.trace_invalidation = true, + + ("debug", "") => options.debug = true, + + ("disable", "") => options.disable = true, + + ("disable-hir-opt", "") => options.disable_hir_opt = true, + + // --zjit-dump-hir dumps the actual input to the codegen, which is currently the same as --zjit-dump-hir-opt. + ("dump-hir" | "dump-hir-opt", "") => options.dump_hir_opt = Some(DumpHIR::WithoutSnapshot), + ("dump-hir" | "dump-hir-opt", "all") => options.dump_hir_opt = Some(DumpHIR::All), + ("dump-hir" | "dump-hir-opt", "debug") => options.dump_hir_opt = Some(DumpHIR::Debug), + + ("dump-hir-init", "") => options.dump_hir_init = Some(DumpHIR::WithoutSnapshot), + ("dump-hir-init", "all") => options.dump_hir_init = Some(DumpHIR::All), + ("dump-hir-init", "debug") => options.dump_hir_init = Some(DumpHIR::Debug), + + ("dump-hir-graphviz", "") => options.dump_hir_graphviz = Some("/dev/stderr".into()), + ("dump-hir-graphviz", _) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.dump_hir_graphviz = Some(opt_val); + } + + ("dump-hir-iongraph", "") => options.dump_hir_iongraph = true, + + ("dump-lir", "") => options.dump_lir = Some(HashSet::from([DumpLIR::init])), + ("dump-lir", filters) => { + let mut dump_lirs = HashSet::new(); + for filter in filters.split(',') { + let dump_lir = match filter { + "all" => { + for &dump_lir in DUMP_LIR_ALL { + dump_lirs.insert(dump_lir); + } + continue; + } + "init" => DumpLIR::init, + "split" => DumpLIR::split, + "alloc_regs" => DumpLIR::alloc_regs, + "compile_exits" => DumpLIR::compile_exits, + "resolve_parallel_mov" => DumpLIR::resolve_parallel_mov, + "scratch_split" => DumpLIR::scratch_split, + "live_intervals" => DumpLIR::live_intervals, + _ => { + let valid_options = DUMP_LIR_ALL.iter().map(|opt| format!("{opt:?}")).collect::<Vec<_>>().join(", "); + eprintln!("invalid --zjit-dump-lir option: '{filter}'"); + eprintln!("valid --zjit-dump-lir options: all, {valid_options}"); + return None; + } + }; + dump_lirs.insert(dump_lir); + } + options.dump_lir = Some(dump_lirs); + } + + ("dump-disasm", _) => { + if !cfg!(feature = "disasm") { + eprintln!("WARNING: the {opt_name} option works best when ZJIT is built in dev mode, i.e. ./configure --enable-zjit=dev"); + } + + match opt_val { + "" => options.dump_disasm = Some(DumpDisasm::Stdout), + directory => { + let path = format!("{directory}/zjit_{}.log", std::process::id()); + match File::options().create(true).append(true).open(&path) { + Ok(file) => { + use std::os::unix::io::IntoRawFd; + eprintln!("ZJIT disasm dump: {path}"); + options.dump_disasm = Some(DumpDisasm::File(file.into_raw_fd())); + } + Err(err) => eprintln!("Failed to create {path}: {err}"), + } + } + } + } + + ("perf", "" | "iseq") => options.perf = Some(PerfMap::ISEQ), + ("perf", "hir") => options.perf = Some(PerfMap::HIR), + + ("allowed-iseqs", _) if !opt_val.is_empty() => options.allowed_iseqs = Some(parse_jit_list(opt_val)), + ("log-compiled-iseqs", _) if !opt_val.is_empty() => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.log_compiled_iseqs = Some(opt_val); + } + + _ => return None, // Option name not recognized + } + + // Option successfully parsed + Some(()) +} + +/// Update rb_zjit_profile_threshold based on rb_zjit_call_threshold and options.num_profiles +fn update_profile_threshold() { + if unsafe { rb_zjit_call_threshold == 1 } { + // If --zjit-call-threshold=1, never rewrite ISEQs to profile instructions. + unsafe { rb_zjit_profile_threshold = 0; } + } else { + // Otherwise, profile instructions at least once. + let num_profiles = get_option!(num_profiles); + unsafe { rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(num_profiles.into()).max(1) }; + } +} + +/// Update --zjit-call-threshold for testing +#[cfg(test)] +pub fn set_call_threshold(call_threshold: CallThreshold) { + unsafe { rb_zjit_call_threshold = call_threshold; } + rb_zjit_prepare_options(); + update_profile_threshold(); +} + +/// Enable --zjit-stats for testing +#[cfg(test)] +pub fn enable_zjit_stats() { + rb_zjit_prepare_options(); + unsafe { OPTIONS.as_mut() }.unwrap().stats = true; +} + +/// Print ZJIT options for `ruby --help`. `width` is width of option parts, and +/// `columns` is indent width of descriptions. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_show_usage(help: c_int, highlight: c_int, width: c_uint, columns: c_int) { + for &(name, description) in ZJIT_OPTIONS.iter() { + unsafe extern "C" { + fn ruby_show_usage_line(name: *const c_char, secondary: *const c_char, description: *const c_char, + help: c_int, highlight: c_int, width: c_uint, columns: c_int); + } + let name = CString::new(name).unwrap(); + let description = CString::new(description).unwrap(); + unsafe { ruby_show_usage_line(name.as_ptr(), null(), description.as_ptr(), help, highlight, width, columns) } + } +} + +/// Macro to print a message only when --zjit-debug is given +macro_rules! debug { + ($($msg:tt)*) => { + if $crate::options::get_option!(debug) { + eprintln!($($msg)*); + } + }; +} +pub(crate) use debug; + +/// Return true if ZJIT should be enabled at boot. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_option_enable() -> bool { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| !opts.disable) { + true + } else { + false + } +} + +/// Return Qtrue if --zjit-stats has been specified. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return Qtrue if stats should be printed at exit. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_print_stats_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats && opts.print_stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return path if stats should be printed at exit to a specified file, else Qnil. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_stats_file_path_p(_ec: EcPtr, _self: VALUE) -> VALUE { + if let Some(opts) = unsafe { OPTIONS.as_ref() } { + if let Some(ref path) = opts.print_stats_file { + return rust_str_to_ruby(path.as_os_str().to_str().unwrap()); + } + } + Qnil +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_dump_disasm_path() { + unsafe { OPTIONS = Some(Options::default()); } + + let dir = std::env::temp_dir(); + let expected_path = dir.join(format!("zjit_{}.log", std::process::id())); + let option = CString::new(format!("dump-disasm={}", dir.display())).unwrap(); + + assert!(parse_option(option.as_ptr()).is_some()); + + let options = unsafe { OPTIONS.as_ref() }.unwrap(); + match options.dump_disasm { + Some(DumpDisasm::File(fd)) => assert!(fd >= 0), + _ => panic!("expected dump-disasm file output"), + } + assert!(expected_path.exists()); + + let _ = std::fs::remove_file(expected_path); + } +} diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs new file mode 100644 index 0000000000..010972bdae --- /dev/null +++ b/zjit/src/payload.rs @@ -0,0 +1,126 @@ +use std::ffi::c_void; +use std::ptr::NonNull; +use crate::codegen::IseqCallRef; +use crate::stats::CompileError; +use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; + +pub use crate::jit_frame::JITFrame; + +/// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. +#[derive(Debug)] +pub struct IseqPayload { + /// Type information of YARV instruction operands + pub profile: IseqProfile, + /// JIT code versions. Different versions should have different assumptions. + pub versions: Vec<IseqVersionRef>, + /// Whether a previous compilation of this ISEQ was invalidated due to + /// singleton class creation (violation of [`crate::hir::Invariant::NoSingletonClass`]). + pub was_invalidated_for_singleton_class_creation: bool, +} + +impl IseqPayload { + fn new() -> Self { + Self { + profile: IseqProfile::new(), + versions: vec![], + was_invalidated_for_singleton_class_creation: false, + } + } +} + +/// JIT code version. When the same ISEQ is compiled with a different assumption, a new version is created. +#[derive(Debug)] +pub struct IseqVersion { + /// ISEQ pointer. Stored here to minimize the size of PatchPoint. + pub iseq: IseqPtr, + + /// Compilation status of the ISEQ. It has the JIT code address of the first block if Compiled. + pub status: IseqStatus, + + /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. + pub gc_offsets: Vec<CodePtr>, + + /// JIT-to-JIT calls from the ISEQ. The IseqPayload's ISEQ is the caller of it. + pub outgoing: Vec<IseqCallRef>, + + /// JIT-to-JIT calls to the ISEQ. The IseqPayload's ISEQ is the callee of it. + pub incoming: Vec<IseqCallRef>, +} + +/// We use a raw pointer instead of Rc to save space for refcount +pub type IseqVersionRef = NonNull<IseqVersion>; + +impl IseqVersion { + /// Check if this version was invalidated + pub fn is_invalidated(&self) -> bool { + self.status == IseqStatus::Invalidated + } + + /// Allocate a new IseqVersion to be compiled + pub fn new(iseq: IseqPtr) -> IseqVersionRef { + let version = Self { + iseq, + status: IseqStatus::NotCompiled, + gc_offsets: vec![], + outgoing: vec![], + incoming: vec![], + }; + let version_ptr = Box::into_raw(Box::new(version)); + NonNull::new(version_ptr).expect("no null from Box") + } +} + +/// Set of CodePtrs for an ISEQ +#[derive(Clone, Debug, PartialEq)] +pub struct IseqCodePtrs { + /// Entry for the interpreter + pub start_ptr: CodePtr, + /// Entries for JIT-to-JIT calls + pub jit_entry_ptrs: Vec<CodePtr>, +} + +#[derive(Debug, PartialEq)] +pub enum IseqStatus { + Compiled(IseqCodePtrs), + CantCompile(CompileError), + NotCompiled, + Invalidated, +} + +/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { + type VoidPtr = *mut c_void; + + unsafe { + let payload = rb_iseq_get_zjit_payload(iseq); + if payload.is_null() { + // Allocate a new payload with Box and transfer ownership to the GC. + // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. + // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. + // We allocate in those cases anyways. + let new_payload = IseqPayload::new(); + let new_payload = Box::into_raw(Box::new(new_payload)); + rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); + + new_payload + } else { + payload as *mut IseqPayload + } + } +} + +/// Get the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { + let payload_non_null = get_or_create_iseq_payload_ptr(iseq); + payload_ptr_as_mut(payload_non_null) +} + +/// Convert an IseqPayload pointer to a mutable reference. Only one reference +/// should be kept at a time. +pub fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { + // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have + // exclusive mutable access. + // Hmm, nothing seems to stop calling this on the same + // iseq twice, though, which violates aliasing rules. + unsafe { payload_ptr.as_mut() }.unwrap() +} diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs new file mode 100644 index 0000000000..000c424da4 --- /dev/null +++ b/zjit/src/profile.rs @@ -0,0 +1,582 @@ +//! Profiler for runtime information. + +// We use the YARV bytecode constants which have a CRuby-style name +#![allow(non_upper_case_globals)] + +use std::collections::HashMap; +use crate::{cruby::*, payload::get_or_create_iseq_payload, options::{get_option, NumProfiles}}; +use crate::distribution::{Distribution, DistributionSummary}; +use crate::stats::Counter::profile_time_ns; +use crate::stats::with_time_stat; + +/// Ephemeral state for profiling runtime information +struct Profiler { + cfp: CfpPtr, + iseq: IseqPtr, + insn_idx: YarvInsnIdx, +} + +impl Profiler { + fn new(ec: EcPtr) -> Self { + let cfp = unsafe { get_ec_cfp(ec) }; + let iseq = unsafe { get_cfp_iseq(cfp) }; + Profiler { + cfp, + iseq, + insn_idx: unsafe { get_cfp_pc(cfp).offset_from(get_iseq_body_iseq_encoded(iseq)) as usize }, + } + } + + // Get an instruction operand that sits next to the opcode at PC. + fn insn_opnd(&self, idx: usize) -> VALUE { + unsafe { get_cfp_pc(self.cfp).add(1 + idx).read() } + } + + // Peek at the nth topmost value on the Ruby stack. + // Returns the topmost value when n == 0. + fn peek_at_stack(&self, n: isize) -> VALUE { + unsafe { + let sp: *mut VALUE = get_cfp_sp(self.cfp); + *(sp.offset(-1 - n)) + } + } + + fn peek_at_self(&self) -> VALUE { + unsafe { rb_get_cfp_self(self.cfp) } + } + + fn peek_at_block_handler(&self) -> VALUE { + unsafe { rb_vm_get_untagged_block_handler(self.cfp) } + } +} + +/// API called from zjit_* instruction. opcode is the bare (non-zjit_*) instruction. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_profile_insn(bare_opcode: u32, ec: EcPtr) { + with_vm_lock(src_loc!(), || { + with_time_stat(profile_time_ns, || profile_insn(bare_opcode as ruby_vminsn_type, ec)); + }); +} + +/// Profile a YARV instruction +fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { + let profiler = &mut Profiler::new(ec); + let profile = &mut get_or_create_iseq_payload(profiler.iseq).profile; + match bare_opcode { + YARVINSN_opt_nil_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_plus => profile_operands(profiler, profile, 2), + YARVINSN_opt_minus => profile_operands(profiler, profile, 2), + YARVINSN_opt_mult => profile_operands(profiler, profile, 2), + YARVINSN_opt_div => profile_operands(profiler, profile, 2), + YARVINSN_opt_mod => profile_operands(profiler, profile, 2), + YARVINSN_opt_eq => profile_operands(profiler, profile, 2), + YARVINSN_opt_neq => profile_operands(profiler, profile, 2), + YARVINSN_opt_lt => profile_operands(profiler, profile, 2), + YARVINSN_opt_le => profile_operands(profiler, profile, 2), + YARVINSN_opt_gt => profile_operands(profiler, profile, 2), + YARVINSN_opt_ge => profile_operands(profiler, profile, 2), + YARVINSN_opt_and => profile_operands(profiler, profile, 2), + YARVINSN_opt_or => profile_operands(profiler, profile, 2), + YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_aref => profile_operands(profiler, profile, 2), + YARVINSN_opt_ltlt => profile_operands(profiler, profile, 2), + YARVINSN_opt_aset => profile_operands(profiler, profile, 3), + YARVINSN_opt_not => profile_operands(profiler, profile, 1), + YARVINSN_getinstancevariable => profile_self(profiler, profile), + YARVINSN_setinstancevariable => profile_self(profiler, profile), + YARVINSN_definedivar => profile_self(profiler, profile), + YARVINSN_opt_regexpmatch2 => profile_operands(profiler, profile, 2), + YARVINSN_objtostring => profile_operands(profiler, profile, 1), + YARVINSN_opt_length => profile_operands(profiler, profile, 1), + YARVINSN_opt_size => profile_operands(profiler, profile, 1), + YARVINSN_opt_succ => profile_operands(profiler, profile, 1), + YARVINSN_invokeblock => profile_block_handler(profiler, profile), + YARVINSN_getblockparamproxy => profile_getblockparamproxy(profiler, profile), + YARVINSN_invokesuper => profile_invokesuper(profiler, profile), + YARVINSN_opt_send_without_block | YARVINSN_send => { + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = num_arguments_on_stack(cd); + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, argc + 1); + } + YARVINSN_splatkw => profile_operands(profiler, profile, 2), + _ => {} + } + + // Once we profile the instruction enough times, we stop profiling it. + let entry = profile.entry_mut(profiler.insn_idx); + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + if entry.profiles_remaining == 0 { + unsafe { rb_zjit_iseq_insn_set(profiler.iseq, profiler.insn_idx as u32, bare_opcode); } + } +} + +/// Return the argc as stated in the calldata plus: +/// * 1 if there is an explicit blockarg, since that will be passed on the stack +pub fn num_arguments_on_stack(cd: *const rb_call_data) -> usize { + let ci = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + let has_blockarg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + (unsafe { vm_ci_argc(ci) }) as usize + has_blockarg as usize +} + +const DISTRIBUTION_SIZE: usize = 4; + +pub type TypeDistribution = Distribution<ProfiledType, DISTRIBUTION_SIZE>; + +pub type TypeDistributionSummary = DistributionSummary<ProfiledType, DISTRIBUTION_SIZE>; + +/// Profile the Type of top-`n` stack operands +fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(n, TypeDistribution::new()); + } + + for (i, profile_type) in entry.opnd_types.iter_mut().enumerate() { + let obj = profiler.peek_at_stack((n - i - 1) as isize); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + profile_type.observe(ty); + } +} + +fn profile_self(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_self(); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_block_handler(); + let ty = ProfiledType::object(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_getblockparamproxy(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + + let level = profiler.insn_opnd(1).as_u32(); + let ep = unsafe { get_cfp_ep_level(profiler.cfp, level) }; + let block_handler = unsafe { *ep.offset(VM_ENV_DATA_INDEX_SPECVAL as isize) }; + let untagged = unsafe { rb_vm_untag_block_handler(block_handler) }; + + let ty = ProfiledType::object(untagged); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { + let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; + let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE + + profile.super_cme.entry(profiler.insn_idx) + .or_insert_with(|| TypeDistribution::new()).observe(ProfiledType::object(cme_value)); + + unsafe { rb_gc_writebarrier(profiler.iseq.into(), cme_value) }; + + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = num_arguments_on_stack(cd); + + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, (argc + 1) as usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Flags(u32); + +impl Flags { + const NONE: u32 = 0; + const IS_IMMEDIATE: u32 = 1 << 0; + /// Object is embedded and the ivar index lands within the object + const IS_EMBEDDED: u32 = 1 << 1; + /// Object is a T_OBJECT + const IS_T_OBJECT: u32 = 1 << 2; + /// Object is a struct with embedded fields + const IS_STRUCT_EMBEDDED: u32 = 1 << 3; + /// Set if the ProfiledType is used for profiling specific objects, not just classes/shapes + const IS_OBJECT_PROFILING: u32 = 1 << 4; + /// Class/module fields_obj is embedded (or absent) + const IS_FIELDS_EMBEDDED: u32 = 1 << 5; + /// Object is a T_CLASS + const IS_T_CLASS: u32 = 1 << 6; + /// Object is a T_MODULE + const IS_T_MODULE: u32 = 1 << 7; + /// Object is a T_DATA + const IS_T_DATA: u32 = 1 << 8; + + pub fn none() -> Self { Self(Self::NONE) } + + pub fn immediate() -> Self { Self(Self::IS_IMMEDIATE) } + pub fn is_immediate(self) -> bool { (self.0 & Self::IS_IMMEDIATE) != 0 } + pub fn is_embedded(self) -> bool { (self.0 & Self::IS_EMBEDDED) != 0 } + pub fn is_t_object(self) -> bool { (self.0 & Self::IS_T_OBJECT) != 0 } + pub fn is_struct_embedded(self) -> bool { (self.0 & Self::IS_STRUCT_EMBEDDED) != 0 } + pub fn is_object_profiling(self) -> bool { (self.0 & Self::IS_OBJECT_PROFILING) != 0 } + pub fn is_fields_embedded(self) -> bool { (self.0 & Self::IS_FIELDS_EMBEDDED) != 0 } + pub fn is_t_class(self) -> bool { (self.0 & Self::IS_T_CLASS) != 0 } + pub fn is_t_module(self) -> bool { (self.0 & Self::IS_T_MODULE) != 0 } + pub fn is_t_data(self) -> bool { (self.0 & Self::IS_T_DATA) != 0 } +} + +/// opt_send_without_block/opt_plus/... should store: +/// * the class of the receiver, so we can do method lookup +/// * the shape of the receiver, so we can optimize ivar lookup +/// +/// with those two, pieces of information, we can also determine when an object is an immediate: +/// * Integer + IS_IMMEDIATE == Fixnum +/// * Float + IS_IMMEDIATE == Flonum +/// * Symbol + IS_IMMEDIATE == StaticSymbol +/// * NilClass == Nil +/// * TrueClass == True +/// * FalseClass == False +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProfiledType { + class: VALUE, + shape: ShapeId, + flags: Flags, +} + +impl Default for ProfiledType { + fn default() -> Self { + Self::empty() + } +} + +impl ProfiledType { + /// Profile the object itself + fn object(obj: VALUE) -> Self { + let mut flags = Flags::none(); + flags.0 |= Flags::IS_OBJECT_PROFILING; + Self { class: obj, shape: INVALID_SHAPE_ID, flags } + } + + /// Profile the class and shape of the given object + fn new(obj: VALUE) -> Self { + if obj == Qfalse { + return Self { class: unsafe { rb_cFalseClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qtrue { + return Self { class: unsafe { rb_cTrueClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qnil { + return Self { class: unsafe { rb_cNilClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.fixnum_p() { + return Self { class: unsafe { rb_cInteger }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.flonum_p() { + return Self { class: unsafe { rb_cFloat }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.static_sym_p() { + return Self { class: unsafe { rb_cSymbol }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + let mut flags = Flags::none(); + if obj.embedded_p() { + flags.0 |= Flags::IS_EMBEDDED; + } + if obj.struct_embedded_p() { + flags.0 |= Flags::IS_STRUCT_EMBEDDED; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_OBJECT) } { + flags.0 |= Flags::IS_T_OBJECT; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_CLASS) } { + flags.0 |= Flags::IS_T_CLASS; + if obj.class_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + if unsafe { RB_TYPE_P(obj, RUBY_T_MODULE) } { + flags.0 |= Flags::IS_T_MODULE; + if obj.class_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + if obj.data_p() { + flags.0 |= Flags::IS_T_DATA; + if obj.data_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + Self { class: obj.class_of(), shape: obj.shape_id_of(), flags } + } + + pub fn empty() -> Self { + Self { class: VALUE(0), shape: INVALID_SHAPE_ID, flags: Flags::none() } + } + + pub fn is_empty(&self) -> bool { + self.class == VALUE(0) + } + + pub fn class(&self) -> VALUE { + self.class + } + + pub fn shape(&self) -> ShapeId { + self.shape + } + + pub fn flags(&self) -> Flags { + self.flags + } + + /// For ivar access, you need to know the index in the fields array (described by the shape) + /// and the way to get the fields array (described by the builtin type). Both pieces of + /// information are on the `RBasic::flags` field. This method returns expected masked flags + /// for guarding. + pub fn rbasic_flags_and_mask(&self) -> (u64, u64) { + let shape_flag_shift = u64::from(RB_SHAPE_FLAG_SHIFT); + let (shape, shape_mask) = (u64::from(self.shape().0) << shape_flag_shift, !0 << shape_flag_shift); + let (builtin_type, type_mask) = if self.flags().is_t_object() { + (RUBY_T_OBJECT, RUBY_T_MASK) + } else if self.flags().is_t_class() { + // Check class first since `Class < Module` + (RUBY_T_CLASS, RUBY_T_MASK) + } else if self.flags().is_t_module() { + (RUBY_T_MODULE, RUBY_T_MASK) + } else if self.flags().is_t_data() { + (RUBY_T_DATA, RUBY_T_MASK) + } else { + (0, 0) + }; + (shape | u64::from(builtin_type), shape_mask | u64::from(type_mask)) + } + + pub fn is_fixnum(&self) -> bool { + self.class == unsafe { rb_cInteger } && self.flags.is_immediate() + } + + pub fn is_string(&self) -> bool { + if self.flags.is_object_profiling() { + panic!("should not call is_string on object-profiled ProfiledType"); + } + // Fast paths for immediates and exact-class + if self.flags.is_immediate() { + return false; + } + + let string = unsafe { rb_cString }; + if self.class == string{ + return true; + } + + self.class.is_subclass_of(string) == ClassRelationship::Subclass + } + + pub fn is_flonum(&self) -> bool { + self.class == unsafe { rb_cFloat } && self.flags.is_immediate() + } + + pub fn is_static_symbol(&self) -> bool { + self.class == unsafe { rb_cSymbol } && self.flags.is_immediate() + } + + pub fn is_nil(&self) -> bool { + self.class == unsafe { rb_cNilClass } && self.flags.is_immediate() + } + + pub fn is_true(&self) -> bool { + self.class == unsafe { rb_cTrueClass } && self.flags.is_immediate() + } + + pub fn is_false(&self) -> bool { + self.class == unsafe { rb_cFalseClass } && self.flags.is_immediate() + } +} + +/// Per-instruction profile entry, stored sparsely in a sorted Vec. +#[derive(Debug)] +struct ProfileEntry { + /// YARV instruction index + insn_idx: u32, + /// Type information of YARV instruction operands + opnd_types: Vec<TypeDistribution>, + /// Number of profiles remaining before recompilation. Counts down from --zjit-num-profiles. + profiles_remaining: NumProfiles, +} + +#[derive(Debug)] +pub struct IseqProfile { + /// Sparse storage of per-instruction profile data, sorted by instruction index. + /// Only instructions that have actually been profiled have entries here. + entries: Vec<ProfileEntry>, + + /// Method entries for `super` calls (stored as VALUE to be GC-safe) + super_cme: HashMap<YarvInsnIdx, TypeDistribution> +} + +impl IseqProfile { + pub fn new() -> Self { + Self { + entries: Vec::new(), + super_cme: HashMap::new(), + } + } + + /// Get or create a mutable profile entry for the given instruction index. + fn entry_mut(&mut self, insn_idx: YarvInsnIdx) -> &mut ProfileEntry { + let idx = insn_idx as u32; + match self.entries.binary_search_by_key(&idx, |e| e.insn_idx) { + Ok(i) => &mut self.entries[i], + Err(i) => { + self.entries.insert(i, ProfileEntry { + insn_idx: idx, + opnd_types: Vec::new(), + profiles_remaining: get_option!(num_profiles), + }); + &mut self.entries[i] + } + } + } + + /// Get a profile entry for the given instruction index (read-only). + fn entry(&self, insn_idx: YarvInsnIdx) -> Option<&ProfileEntry> { + let idx = insn_idx as u32; + self.entries.binary_search_by_key(&idx, |e| e.insn_idx) + .ok().map(|i| &self.entries[i]) + } + + /// Check if enough profiles have been gathered for this instruction. + pub fn done_profiling_at(&self, insn_idx: YarvInsnIdx) -> bool { + self.entry(insn_idx).map_or(false, |e| e.profiles_remaining == 0) + } + + /// Profile send operands from the stack at runtime. + /// `sp` is the current stack pointer (after the args and receiver). + /// `argc` is the number of arguments (not counting receiver). + /// Returns true if enough profiles have been gathered and the ISEQ should be recompiled. + pub fn profile_send_at(&mut self, iseq: IseqPtr, insn_idx: YarvInsnIdx, sp: *const VALUE, argc: usize) -> bool { + let n = argc + 1; // args + receiver + let entry = self.entry_mut(insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(n, TypeDistribution::new()); + } + for i in 0..n { + let obj = unsafe { *sp.offset(i as isize - n as isize) }; + let ty = ProfiledType::new(obj); + VALUE::from(iseq).write_barrier(ty.class()); + entry.opnd_types[i].observe(ty); + } + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + entry.profiles_remaining == 0 + } + + /// Profile self for a shape guard exit at runtime. + /// This may be called on an instruction that was already profiled by YARV, + /// so we reset the counter to re-profile with the new shapes seen at runtime. + /// Returns true if enough profiles have been gathered and the ISEQ should be recompiled. + pub fn profile_self_at(&mut self, iseq: IseqPtr, insn_idx: YarvInsnIdx, self_val: VALUE) -> bool { + let entry = self.entry_mut(insn_idx); + // Reset profiling if the previous round already finished (stale YARV profiles). + // This ensures we collect num_profiles samples of the new shapes before recompiling. + if entry.profiles_remaining == 0 { + entry.profiles_remaining = get_option!(num_profiles); + } + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + let ty = ProfiledType::new(self_val); + VALUE::from(iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + entry.profiles_remaining == 0 + } + + /// Get profiled operand types for a given instruction index + pub fn get_operand_types(&self, insn_idx: YarvInsnIdx) -> Option<&[TypeDistribution]> { + self.entry(insn_idx).map(|e| e.opnd_types.as_slice()).filter(|s| !s.is_empty()) + } + + pub fn get_super_method_entry(&self, insn_idx: YarvInsnIdx) -> Option<*const rb_callable_method_entry_t> { + let Some(entry) = self.super_cme.get(&insn_idx) else { return None }; + let summary = TypeDistributionSummary::new(entry); + + if summary.is_monomorphic() { + Some(summary.bucket(0).class.0 as *const rb_callable_method_entry_t) + } else { + None + } + } + + /// Run a given callback with every object in IseqProfile + pub fn each_object(&self, callback: impl Fn(VALUE)) { + for entry in &self.entries { + for distribution in &entry.opnd_types { + for profiled_type in distribution.each_item() { + // If the type is a GC object, call the callback + callback(profiled_type.class); + } + } + } + + for super_cme_values in self.super_cme.values() { + for profiled_type in super_cme_values.each_item() { + callback(profiled_type.class) + } + } + } + + /// Run a given callback with a mutable reference to every object in IseqProfile. + pub fn each_object_mut(&mut self, callback: impl Fn(&mut VALUE)) { + for entry in &mut self.entries { + for distribution in &mut entry.opnd_types { + for ref mut profiled_type in distribution.each_item_mut() { + // If the type is a GC object, call the callback + callback(&mut profiled_type.class); + } + } + } + + // Update CME references if they move during compaction. + for super_cme_values in self.super_cme.values_mut() { + for ref mut profiled_type in super_cme_values.each_item_mut() { + callback(&mut profiled_type.class) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::cruby::*; + + #[test] + fn can_profile_block_handler() { + with_rubyvm(|| eval(" + def foo = yield + foo rescue 0 + foo rescue 0 + ")); + } +} diff --git a/zjit/src/state.rs b/zjit/src/state.rs new file mode 100644 index 0000000000..da09d09314 --- /dev/null +++ b/zjit/src/state.rs @@ -0,0 +1,541 @@ +//! Runtime state of ZJIT. + +use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_function_stub_hit_trampoline, gen_materialize_exit_trampoline, gen_materialize_exit_trampoline_with_counter}; +use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_profile_frames, rb_profile_frame_full_label, rb_profile_frame_absolute_path, rb_profile_frame_path, VALUE, VM_INSTRUCTION_SIZE, with_vm_lock, rust_str_to_id, rb_funcallv, rb_const_get, rb_cRubyVM}; +use crate::cruby_methods; +use cruby::{ID, rb_callable_method_entry, get_def_method_serial, rb_gc_register_mark_object, ruby_str_to_rust_string_result}; +use std::sync::atomic::Ordering; +use crate::invariants::Invariants; +use crate::asm::CodeBlock; +use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::jit_frame::JITFrame; +use crate::stats::{Counters, InsnCounters, PerfettoTracer}; +use crate::virtualmem::CodePtr; +use std::sync::atomic::AtomicUsize; +use std::collections::HashMap; +use std::ptr::null; + +/// Shared trampoline to enter ZJIT. Not null when ZJIT is enabled. +#[allow(non_upper_case_globals)] +#[unsafe(no_mangle)] +pub static mut rb_zjit_entry: *const u8 = null(); + +/// Like rb_zjit_enabled_p, but for Rust code. +pub fn zjit_enabled_p() -> bool { + unsafe { rb_zjit_entry != null() } +} + +/// Global state needed for code generation +pub struct ZJITState { + /// Inline code block (fast path) + code_block: CodeBlock, + + /// ZJIT statistics + counters: Counters, + + /// Side-exit counters + exit_counters: InsnCounters, + + /// Send fallback counters + send_fallback_counters: InsnCounters, + + /// Assumptions that require invalidation + invariants: Invariants, + + /// Assert successful compilation if set to true + assert_compiles: bool, + + /// Properties of core library methods + method_annotations: cruby_methods::Annotations, + + /// Trampoline to side-exit without restoring PC or the stack + exit_trampoline: CodePtr, + + /// Trampoline to materialize JIT frames before side-exiting + materialize_exit_trampoline: CodePtr, + + /// Trampoline to materialize JIT frames and increment exit_compilation_failure + materialize_exit_trampoline_with_counter: CodePtr, + + /// Trampoline to call function_stub_hit + function_stub_hit_trampoline: CodePtr, + + /// Counter pointers for full frame C functions + full_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for un-annotated C functions + not_annotated_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for all calls to any kind of C function from JIT code + ccall_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for access counts of ISEQs accessed by JIT code + iseq_calls_count_pointers: HashMap<String, Box<u64>>, + + /// Perfetto tracer for --zjit-trace-exits + perfetto_tracer: Option<PerfettoTracer>, + + /// Frame metadata for ISEQ and C calls that are known at compile time + jit_frames: Vec<*mut JITFrame>, +} + +/// Tracks the initialization progress +enum InitializationState { + Uninitialized, + + /// At boot time, rb_zjit_init will be called regardless of whether + /// ZJIT is enabled, in this phase we initialize any states that must + /// be captured at during boot. + Initialized(cruby_methods::Annotations), + + /// When ZJIT is enabled, either during boot with `--zjit`, or lazily + /// at a later time with `RubyVM::ZJIT.enable`, we perform the rest + /// of the initialization steps and produce the `ZJITState` instance. + Enabled(ZJITState), + + /// Indicates that ZJITState::init has panicked. Should never be + /// encountered in practice since we abort immediately when that + /// happens. + Panicked, +} + +/// Private singleton instance of the codegen globals +static mut ZJIT_STATE: InitializationState = InitializationState::Uninitialized; + +impl ZJITState { + /// Initialize the ZJIT globals. Return the address of the JIT entry trampoline. + pub fn init() -> *const u8 { + use InitializationState::*; + + let initialization_state = unsafe { + std::mem::replace(&mut ZJIT_STATE, Panicked) + }; + + let Initialized(method_annotations) = initialization_state else { + panic!("rb_zjit_init was never called"); + }; + + let mut cb = { + use crate::options::*; + use crate::virtualmem::*; + use std::rc::Rc; + use std::cell::RefCell; + + let mem_block = VirtualMem::alloc(get_option!(exec_mem_bytes), Some(get_option!(mem_bytes))); + let mem_block = Rc::new(RefCell::new(mem_block)); + + CodeBlock::new(mem_block.clone(), get_option_ref!(dump_disasm).is_some()) + }; + + let entry_trampoline = gen_entry_trampoline(&mut cb).unwrap().raw_ptr(&cb); + let exit_trampoline = gen_exit_trampoline(&mut cb).unwrap(); + let materialize_exit_trampoline = gen_materialize_exit_trampoline(&mut cb, exit_trampoline).unwrap(); + let function_stub_hit_trampoline = gen_function_stub_hit_trampoline(&mut cb).unwrap(); + + let perfetto_tracer = if get_option!(trace_side_exits).is_some() || get_option!(trace_compiles) || get_option!(trace_invalidation) { + Some(PerfettoTracer::new()) + } else { + None + }; + + // Initialize the codegen globals instance + let zjit_state = ZJITState { + code_block: cb, + counters: Counters::default(), + exit_counters: [0; VM_INSTRUCTION_SIZE as usize], + send_fallback_counters: [0; VM_INSTRUCTION_SIZE as usize], + invariants: Invariants::default(), + assert_compiles: false, + method_annotations, + exit_trampoline, + materialize_exit_trampoline, + materialize_exit_trampoline_with_counter: materialize_exit_trampoline, + function_stub_hit_trampoline, + full_frame_cfunc_counter_pointers: HashMap::new(), + not_annotated_frame_cfunc_counter_pointers: HashMap::new(), + ccall_counter_pointers: HashMap::new(), + iseq_calls_count_pointers: HashMap::new(), + perfetto_tracer, + jit_frames: vec![], + }; + unsafe { ZJIT_STATE = Enabled(zjit_state); } + + // With --zjit-stats, use a different trampoline on function stub exits + // to count exit_compilation_failure. Note that the trampoline code depends + // on the counter, so ZJIT_STATE needs to be initialized first. + if get_option!(stats) { + let cb = ZJITState::get_code_block(); + let code_ptr = gen_materialize_exit_trampoline_with_counter(cb, materialize_exit_trampoline).unwrap(); + ZJITState::get_instance().materialize_exit_trampoline_with_counter = code_ptr; + } + + entry_trampoline + } + + /// Return true if zjit_state has been initialized + pub fn has_instance() -> bool { + matches!(unsafe { &ZJIT_STATE }, InitializationState::Enabled(_)) + } + + /// Get a mutable reference to the codegen globals instance + fn get_instance() -> &'static mut ZJITState { + if let InitializationState::Enabled(instance) = unsafe { &mut ZJIT_STATE } { + instance + } else { + panic!("ZJITState::get_instance called when ZJIT is not enabled") + } + } + + /// Get a mutable reference to the inline code block + pub fn get_code_block() -> &'static mut CodeBlock { + &mut ZJITState::get_instance().code_block + } + + /// Get a mutable reference to the invariants + pub fn get_invariants() -> &'static mut Invariants { + &mut ZJITState::get_instance().invariants + } + + pub fn get_jit_frames() -> &'static mut Vec<*mut JITFrame> { + &mut ZJITState::get_instance().jit_frames + } + + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { + &ZJITState::get_instance().method_annotations + } + + /// Return true if successful compilation should be asserted + pub fn assert_compiles_enabled() -> bool { + ZJITState::get_instance().assert_compiles + } + + /// Start asserting successful compilation + pub fn enable_assert_compiles() { + let instance = ZJITState::get_instance(); + instance.assert_compiles = true; + } + + /// Stop asserting successful compilation + pub fn disable_assert_compiles() { + let instance = ZJITState::get_instance(); + instance.assert_compiles = false; + } + + /// Get a mutable reference to counters for ZJIT stats + pub fn get_counters() -> &'static mut Counters { + &mut ZJITState::get_instance().counters + } + + /// Get a mutable reference to side-exit counters + pub fn get_exit_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().exit_counters + } + + /// Get a mutable reference to fallback counters + pub fn get_send_fallback_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().send_fallback_counters + } + + /// Get a mutable reference to full frame cfunc counter pointers + pub fn get_not_inlined_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().full_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to non-annotated cfunc counter pointers + pub fn get_not_annotated_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().not_annotated_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to ccall counter pointers + pub fn get_ccall_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().ccall_counter_pointers + } + + /// Get a mutable reference to iseq access count pointers + pub fn get_iseq_calls_count_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().iseq_calls_count_pointers + } + + /// Was --zjit-save-compiled-iseqs specified? + pub fn should_log_compiled_iseqs() -> bool { + get_option!(log_compiled_iseqs).is_some() + } + + /// Log the name of a compiled ISEQ to the file specified in options.log_compiled_iseqs + pub fn log_compile(iseq_name: String) { + assert!(ZJITState::should_log_compiled_iseqs()); + let filename = get_option!(log_compiled_iseqs).as_ref().unwrap(); + use std::io::Write; + let mut file = match std::fs::OpenOptions::new().create(true).append(true).open(filename) { + Ok(f) => f, + Err(e) => { + eprintln!("ZJIT: Failed to create file '{}': {}", filename.display(), e); + return; + } + }; + if let Err(e) = writeln!(file, "{iseq_name}") { + eprintln!("ZJIT: Failed to write to file '{}': {}", filename.display(), e); + } + } + + /// Check if we are allowed to compile a given ISEQ based on --zjit-allowed-iseqs + pub fn can_compile_iseq(iseq: cruby::IseqPtr) -> bool { + if let Some(ref allowed_iseqs) = get_option!(allowed_iseqs) { + let name = cruby::iseq_get_location(iseq, 0); + allowed_iseqs.contains(&name) + } else { + true // If no restrictions, allow all ISEQs + } + } + + /// Return a code pointer to the side-exit trampoline + pub fn get_exit_trampoline() -> CodePtr { + ZJITState::get_instance().exit_trampoline + } + + /// Return a code pointer to the materialize_exit trampoline + pub fn get_materialize_exit_trampoline() -> CodePtr { + ZJITState::get_instance().materialize_exit_trampoline + } + + /// Return a code pointer to the materialize_exit trampoline for function stubs + pub fn get_materialize_exit_trampoline_with_counter() -> CodePtr { + ZJITState::get_instance().materialize_exit_trampoline_with_counter + } + + /// Return a code pointer to the function stub hit trampoline + pub fn get_function_stub_hit_trampoline() -> CodePtr { + ZJITState::get_instance().function_stub_hit_trampoline + } + + /// Get a mutable reference to the Perfetto tracer + pub fn get_tracer() -> Option<&'static mut PerfettoTracer> { + if !ZJITState::has_instance() { return None; } + ZJITState::get_instance().perfetto_tracer.as_mut() + } +} + +/// The `::RubyVM::ZJIT` module. +pub static ZJIT_MODULE: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_side_exit!` right after VM boot. +pub static INDUCE_SIDE_EXIT_SERIAL: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_compile_failure!` right after VM boot. +pub static INDUCE_COMPILE_FAILURE_SERIAL: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_breakpoint!` right after VM boot. +pub static INDUCE_BREAKPOINT_SERIAL: AtomicUsize = AtomicUsize::new(!0); + +/// Check if a method, `method_id`, currently exists on `ZJIT.singleton_class` and has the `expected_serial`. +pub fn zjit_module_method_match_serial(method_id: ID, expected_serial: &AtomicUsize) -> bool { + let zjit_module_singleton = VALUE(ZJIT_MODULE.load(Ordering::Relaxed)).class_of(); + let cme = unsafe { rb_callable_method_entry(zjit_module_singleton, method_id) }; + if cme.is_null() { + false + } else { + let serial = unsafe { get_def_method_serial((*cme).def) }; + serial == expected_serial.load(std::sync::atomic::Ordering::Relaxed) + } +} + +/// Initialize IDs and annotate builtin C method entries. +/// Must be called at boot before ruby_init_prelude() since the prelude +/// could redefine core methods (e.g. Kernel.prepend via bundler). +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_init_builtin_cmes() { + use InitializationState::*; + + debug_assert!( + matches!(unsafe { &ZJIT_STATE }, Uninitialized), + "rb_zjit_init_builtin_cmes should only be called once during boot", + ); + + cruby::ids::init(); + + let method_annotations = cruby_methods::init(); + + unsafe { ZJIT_STATE = Initialized(method_annotations); } + + // Boot time setup for compiler directives + unsafe { + let zjit_module = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_side_exit_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_side_exit! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_SIDE_EXIT_SERIAL.store(serial, Ordering::Relaxed); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_compile_failure_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_compile_failure! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_COMPILE_FAILURE_SERIAL.store(serial, Ordering::Relaxed); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_breakpoint_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_breakpoint! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_BREAKPOINT_SERIAL.store(serial, Ordering::Relaxed); + + // Root and pin the module since we'll be doing object identity comparisons. + ZJIT_MODULE.store(zjit_module.0, Ordering::Relaxed); + rb_gc_register_mark_object(zjit_module); + } +} + +/// Initialize ZJIT at boot. This is called even if ZJIT is disabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_init(zjit_enabled: bool) { + // If --zjit, enable ZJIT immediately + if zjit_enabled { + zjit_enable(); + } +} + +/// Enable ZJIT compilation. +fn zjit_enable() { + // Call ZJIT hooks before enabling ZJIT to avoid compiling the hooks themselves + unsafe { + let zjit = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + rb_funcallv(zjit, rust_str_to_id("call_jit_hooks"), 0, std::ptr::null()); + } + + // Catch panics to avoid UB for unwinding into C frames. + // See https://doc.rust-lang.org/nomicon/exception-safety.html + let result = std::panic::catch_unwind(|| { + // Initialize ZJIT states + let zjit_entry = ZJITState::init(); + + // Install a panic hook for ZJIT + rb_bug_panic_hook(); + + // Discard the instruction count for boot which we never compile + unsafe { rb_vm_insn_count = 0; } + + // ZJIT enabled and initialized successfully + assert!(unsafe{ rb_zjit_entry == null() }); + unsafe { rb_zjit_entry = zjit_entry; } + }); + + if result.is_err() { + println!("ZJIT: zjit_enable() panicked. Aborting."); + std::process::abort(); + } +} + +/// Enable ZJIT compilation, returning Qtrue if ZJIT was previously disabled +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_enable(_ec: EcPtr, _self: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || { + // Options would not have been initialized during boot if no flags were specified + rb_zjit_prepare_options(); + + // Initialize and enable ZJIT + zjit_enable(); + + // Add "+ZJIT" to RUBY_DESCRIPTION + unsafe { + unsafe extern "C" { + fn ruby_set_zjit_description(); + } + ruby_set_zjit_description(); + } + + Qtrue + }) +} + +/// Assert that any future ZJIT compilation will return a function pointer (not fail to compile) +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_assert_compiles(_ec: EcPtr, _self: VALUE) -> VALUE { + ZJITState::enable_assert_compiles(); + Qnil +} + +/// Resolve a profile frame VALUE to a human-readable "label (path)" string. +fn resolve_frame_label(frame: VALUE) -> String { + unsafe { + let label_str = ruby_str_to_rust_string_result(rb_profile_frame_full_label(frame)).unwrap_or("<unknown>".into()); + + let path = rb_profile_frame_absolute_path(frame); + let path = if path.nil_p() { rb_profile_frame_path(frame) } else { path }; + let path_str = ruby_str_to_rust_string_result(path).unwrap_or("<unknown>".into()); + + format!("{label_str} ({path_str})") + } +} + +/// Record a backtrace with ZJIT side exits as a Perfetto trace event +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_record_exit_stack(reason: *const std::ffi::c_char) { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + let tracer = match ZJITState::get_tracer() { + Some(t) => t, + None => return, + }; + + // When `trace_side_exits_sample_interval` is non-zero, apply sampling. + if get_option!(trace_side_exits_sample_interval) != 0 { + if tracer.skipped_samples < get_option!(trace_side_exits_sample_interval) { + tracer.skipped_samples += 1; + return; + } else { + tracer.skipped_samples = 0; + } + } + + // Collect profile frames + let frames = capture_ruby_frames(); + + // Get the reason string + let reason_str = if reason.is_null() { + "unknown" + } else { + unsafe { std::ffi::CStr::from_ptr(reason).to_str().unwrap_or("unknown") } + }; + + tracer.write_event("side_exit", reason_str, &frames); +} + +/// Wrap a closure in a Perfetto duration event with category "invalidation" +/// and a Ruby backtrace captured on the begin event. +pub fn trace_invalidation<F, R>(reason: &str, func: F) -> R where F: FnOnce() -> R { + if !get_option!(trace_invalidation) { + return func(); + } + + // Capture backtrace and emit begin event before patching + let frames = capture_ruby_frames(); + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_begin("invalidation", reason, ts, &frames); + } + + let result = func(); + + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_end("invalidation", reason, ts); + } + result +} + +/// Capture the current Ruby call stack as human-readable frame labels. +fn capture_ruby_frames() -> Vec<String> { + const BUFF_LEN: usize = 2048; + let mut frames_buffer = vec![VALUE(0_usize); BUFF_LEN]; + let mut lines_buffer = vec![0i32; BUFF_LEN]; + + let stack_length = unsafe { + rb_profile_frames( + 0, + BUFF_LEN as i32, + frames_buffer.as_mut_ptr(), + lines_buffer.as_mut_ptr(), + ) + }; + + // Resolve each frame to a human-readable string (top frame first) + (0..stack_length as usize) + .map(|i| resolve_frame_label(frames_buffer[i])) + .collect() +} diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs new file mode 100644 index 0000000000..57320a02e7 --- /dev/null +++ b/zjit/src/stats.rs @@ -0,0 +1,1280 @@ +//! Counters and associated methods for events when ZJIT is run. + +use std::time::Instant; +use std::sync::atomic::Ordering; +use crate::options::OPTIONS; + +// test binaries always bring it in as a cargo dependency +#[cfg(all(feature = "stats_allocator", not(test)))] +#[path = "../../jit/src/lib.rs"] +mod jit; + +use crate::{cruby::*, hir::ParseError, options::get_option, state::{zjit_enabled_p, ZJITState}}; + +macro_rules! make_counters { + ( + default { + $($default_counter_name:ident,)+ + } + exit { + $($exit_counter_name:ident,)+ + } + dynamic_send { + $($dynamic_send_counter_name:ident,)+ + } + optimized_send { + $($optimized_send_counter_name:ident,)+ + } + dynamic_setivar { + $($dynamic_setivar_counter_name:ident,)+ + } + dynamic_getivar { + $($dynamic_getivar_counter_name:ident,)+ + } + dynamic_definedivar { + $($dynamic_definedivar_counter_name:ident,)+ + } + $($counter_name:ident,)+ + ) => { + /// Struct containing the counter values + #[derive(Default, Debug)] + pub struct Counters { + $(pub $default_counter_name: u64,)+ + $(pub $exit_counter_name: u64,)+ + $(pub $dynamic_send_counter_name: u64,)+ + $(pub $optimized_send_counter_name: u64,)+ + $(pub $dynamic_setivar_counter_name: u64,)+ + $(pub $dynamic_getivar_counter_name: u64,)+ + $(pub $dynamic_definedivar_counter_name: u64,)+ + $(pub $counter_name: u64,)+ + } + + /// Enum to represent a counter + #[allow(non_camel_case_types)] + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub enum Counter { + $($default_counter_name,)+ + $($exit_counter_name,)+ + $($dynamic_send_counter_name,)+ + $($optimized_send_counter_name,)+ + $($dynamic_setivar_counter_name,)+ + $($dynamic_getivar_counter_name,)+ + $($dynamic_definedivar_counter_name,)+ + $($counter_name,)+ + } + + impl Counter { + pub fn name(&self) -> &'static str { + match self { + $( Counter::$default_counter_name => stringify!($default_counter_name), )+ + $( Counter::$exit_counter_name => stringify!($exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => stringify!($dynamic_send_counter_name), )+ + $( Counter::$optimized_send_counter_name => stringify!($optimized_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => stringify!($dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => stringify!($dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => stringify!($dynamic_definedivar_counter_name), )+ + $( Counter::$counter_name => stringify!($counter_name), )+ + } + } + + pub fn get(name: &str) -> Option<Counter> { + match name { + $( stringify!($default_counter_name) => Some(Counter::$default_counter_name), )+ + $( stringify!($exit_counter_name) => Some(Counter::$exit_counter_name), )+ + $( stringify!($dynamic_send_counter_name) => Some(Counter::$dynamic_send_counter_name), )+ + $( stringify!($optimized_send_counter_name) => Some(Counter::$optimized_send_counter_name), )+ + $( stringify!($dynamic_setivar_counter_name) => Some(Counter::$dynamic_setivar_counter_name), )+ + $( stringify!($dynamic_getivar_counter_name) => Some(Counter::$dynamic_getivar_counter_name), )+ + $( stringify!($dynamic_definedivar_counter_name) => Some(Counter::$dynamic_definedivar_counter_name), )+ + $( stringify!($counter_name) => Some(Counter::$counter_name), )+ + _ => None, + } + } + } + + /// Map a counter to a pointer + pub fn counter_ptr(counter: Counter) -> *mut u64 { + let counters = $crate::state::ZJITState::get_counters(); + match counter { + $( Counter::$default_counter_name => std::ptr::addr_of_mut!(counters.$default_counter_name), )+ + $( Counter::$exit_counter_name => std::ptr::addr_of_mut!(counters.$exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_definedivar_counter_name), )+ + $( Counter::$optimized_send_counter_name => std::ptr::addr_of_mut!(counters.$optimized_send_counter_name), )+ + $( Counter::$counter_name => std::ptr::addr_of_mut!(counters.$counter_name), )+ + } + } + + /// List of counters that are available without --zjit-stats. + /// They are incremented only by `incr_counter()` and don't use `gen_incr_counter()`. + pub const DEFAULT_COUNTERS: &'static [Counter] = &[ + $( Counter::$default_counter_name, )+ + ]; + + /// List of other counters that are summed as side_exit_count. + pub const EXIT_COUNTERS: &'static [Counter] = &[ + $( Counter::$exit_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_send_count. + pub const DYNAMIC_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_send_counter_name, )+ + ]; + + /// List of other counters that are summed as optimized_send_count. + pub const OPTIMIZED_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$optimized_send_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_setivar_count. + pub const DYNAMIC_SETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_setivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_getivar_count. + pub const DYNAMIC_GETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_getivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_definedivar_count. + pub const DYNAMIC_DEFINEDIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_definedivar_counter_name, )+ + ]; + + /// List of other counters that are available only for --zjit-stats. + pub const OTHER_COUNTERS: &'static [Counter] = &[ + $( Counter::$counter_name, )+ + ]; + } +} + +// Declare all the counters we track +make_counters! { + // Default counters that are available without --zjit-stats + default { + compiled_iseq_count, + failed_iseq_count, + skipped_native_stack_full, + + compile_time_ns, + profile_time_ns, + gc_time_ns, + invalidation_time_ns, + + compiled_side_exit_count, + side_exit_size, + compile_side_exit_time_ns, + + compile_hir_time_ns, + compile_hir_build_time_ns, + compile_hir_strength_reduce_time_ns, + compile_hir_optimize_load_store_time_ns, + compile_hir_canonicalize_time_ns, + compile_hir_fold_constants_time_ns, + compile_hir_clean_cfg_time_ns, + compile_hir_remove_redundant_patch_points_time_ns, + compile_hir_remove_duplicate_check_interrupts_time_ns, + compile_hir_eliminate_dead_code_time_ns, + compile_lir_time_ns, + } + + // Exit counters that are summed as side_exit_count + exit { + // exit_: Side exits reasons + exit_compile_error, + exit_unhandled_newarray_send_min, + exit_unhandled_newarray_send_hash, + exit_unhandled_newarray_send_pack, + exit_unhandled_newarray_send_pack_buffer, + exit_unhandled_newarray_send_unknown, + exit_unhandled_duparray_send, + exit_unhandled_tailcall, + exit_unhandled_splat, + exit_unhandled_kwarg, + exit_unhandled_block_arg, + exit_unknown_special_variable, + exit_unhandled_hir_insn, + exit_unhandled_yarv_insn, + exit_fixnum_add_overflow, + exit_fixnum_sub_overflow, + exit_fixnum_mult_overflow, + exit_fixnum_lshift_overflow, + exit_fixnum_mod_by_zero, + exit_fixnum_div_by_zero, + exit_box_fixnum_overflow, + exit_guard_type_failure, + exit_guard_bit_equals_failure, + exit_guard_int_equals_failure, + exit_guard_shape_failure, + exit_expandarray_failure, + exit_guard_not_frozen_failure, + exit_guard_not_shared_failure, + exit_guard_less_failure, + exit_guard_greater_eq_failure, + exit_guard_super_method_entry, + exit_patchpoint_bop_redefined, + exit_patchpoint_method_redefined, + exit_patchpoint_stable_constant_names, + exit_patchpoint_no_tracepoint, + exit_patchpoint_no_ep_escape, + exit_patchpoint_single_ractor_mode, + exit_patchpoint_no_singleton_class, + exit_patchpoint_root_box_only, + exit_callee_side_exit, + exit_obj_to_string_fallback, + exit_interrupt, + exit_stackoverflow, + exit_block_param_proxy_not_iseq_or_ifunc, + exit_block_param_proxy_not_nil, + exit_block_param_proxy_fallback_miss, + exit_block_param_proxy_profile_not_covered, + exit_block_param_wb_required, + exit_too_many_keyword_parameters, + exit_no_profile_send, + exit_splatkw_not_nil_or_hash, + exit_splatkw_polymorphic, + exit_splatkw_not_profiled, + exit_directive_induced, + exit_send_while_tracing, + exit_invokeblock_not_ifunc, + } + + // Send fallback counters that are summed as dynamic_send_count + dynamic_send { + // send_fallback_: Fallback reasons for send-ish instructions + send_fallback_send_without_block_polymorphic, + send_fallback_send_without_block_megamorphic, + send_fallback_send_without_block_no_profiles, + send_fallback_send_without_block_cfunc_not_variadic, + send_fallback_send_without_block_cfunc_array_variadic, + send_fallback_send_without_block_not_optimized_method_type, + send_fallback_send_without_block_not_optimized_method_type_optimized, + send_fallback_send_without_block_not_optimized_need_permission, + send_fallback_too_many_args_for_lir, + send_fallback_send_without_block_bop_redefined, + send_fallback_send_without_block_operands_not_fixnum, + send_fallback_send_without_block_polymorphic_fallback, + send_fallback_send_without_block_direct_keyword_mismatch, + send_fallback_send_without_block_direct_keyword_count_mismatch, + send_fallback_send_without_block_direct_missing_keyword, + send_fallback_send_without_block_direct_too_many_keywords, + send_fallback_send_polymorphic, + send_fallback_send_megamorphic, + send_fallback_send_no_profiles, + send_fallback_send_not_optimized_method_type, + send_fallback_send_not_optimized_need_permission, + send_fallback_ccall_with_frame_too_many_args, + send_fallback_argc_param_mismatch, + // The call has at least one feature on the caller or callee side + // that the optimizer does not support. + send_fallback_one_or_more_complex_arg_pass, + // Caller has keyword arguments but callee doesn't expect them. + send_fallback_unexpected_keyword_args, + // Singleton class previously created for receiver class. + send_fallback_singleton_class_seen, + send_fallback_bmethod_non_iseq_proc, + send_fallback_obj_to_string_not_string, + send_fallback_send_cfunc_variadic, + send_fallback_send_cfunc_array_variadic, + send_fallback_super_call_with_block, + send_fallback_super_from_block, + send_fallback_super_class_not_found, + send_fallback_super_complex_args_pass, + send_fallback_super_fallback_no_profile, + send_fallback_super_not_optimized_method_type, + send_fallback_super_polymorphic, + send_fallback_super_target_not_found, + send_fallback_super_target_complex_args_pass, + send_fallback_cannot_send_direct, + send_fallback_invokeblock_not_specialized, + send_fallback_sendforward_not_specialized, + send_fallback_invokesuperforward_not_specialized, + send_fallback_single_ractor_mode_required, + send_fallback_uncategorized, + } + + // Optimized send counters that are summed as optimized_send_count + optimized_send { + iseq_optimized_send_count, + inline_cfunc_optimized_send_count, + inline_iseq_optimized_send_count, + non_variadic_cfunc_optimized_send_count, + variadic_cfunc_optimized_send_count, + } + + // Ivar fallback counters that are summed as dynamic_setivar_count + dynamic_setivar { + // setivar_fallback_: Fallback reasons for dynamic setivar instructions + setivar_fallback_not_monomorphic, + setivar_fallback_immediate, + setivar_fallback_not_t_object, + setivar_fallback_complex, + setivar_fallback_frozen, + setivar_fallback_shape_transition, + setivar_fallback_new_shape_complex, + setivar_fallback_new_shape_needs_extension, + } + + // Ivar fallback counters that are summed as dynamic_getivar_count + dynamic_getivar { + // getivar_fallback_: Fallback reasons for dynamic getivar instructions + getivar_fallback_not_monomorphic, + getivar_fallback_immediate, + getivar_fallback_not_t_object, + getivar_fallback_complex, + } + + // Ivar fallback counters that are summed as dynamic_definedivar_count + dynamic_definedivar { + // definedivar_fallback_: Fallback reasons for dynamic definedivar instructions + definedivar_fallback_not_monomorphic, + definedivar_fallback_immediate, + definedivar_fallback_not_t_object, + definedivar_fallback_complex, + } + + // compile_error_: Compile error reasons + compile_error_iseq_version_limit_reached, + compile_error_iseq_stack_too_large, + compile_error_native_stack_too_large, + compile_error_exception_handler, + compile_error_out_of_memory, + compile_error_label_linking_failure, + compile_error_jit_to_jit_optional, + compile_error_register_spill_on_ccall, + compile_error_register_spill_on_alloc, + compile_error_parse_stack_underflow, + compile_error_parse_malformed_iseq, + compile_error_parse_not_allowed, + compile_error_parse_directive_induced, + compile_error_validation_block_has_no_terminator, + compile_error_validation_terminator_not_at_end, + compile_error_validation_mismatched_block_arity, + compile_error_validation_jump_target_not_in_rpo, + compile_error_validation_operand_not_defined, + compile_error_validation_duplicate_instruction, + compile_error_validation_type_check_failure, + compile_error_validation_misc_validation_error, + + // unhandled_hir_insn_: Unhandled HIR instructions + unhandled_hir_insn_array_max, + unhandled_hir_insn_fixnum_div, + unhandled_hir_insn_throw, + unhandled_hir_insn_invokebuiltin, + unhandled_hir_insn_unknown, + + // The number of times YARV instructions are executed on JIT code + zjit_insn_count, + + // Method call def_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_iseq, + unspecialized_send_without_block_def_type_cfunc, + unspecialized_send_without_block_def_type_attrset, + unspecialized_send_without_block_def_type_ivar, + unspecialized_send_without_block_def_type_bmethod, + unspecialized_send_without_block_def_type_zsuper, + unspecialized_send_without_block_def_type_alias, + unspecialized_send_without_block_def_type_undef, + unspecialized_send_without_block_def_type_not_implemented, + unspecialized_send_without_block_def_type_optimized, + unspecialized_send_without_block_def_type_missing, + unspecialized_send_without_block_def_type_refined, + unspecialized_send_without_block_def_type_null, + + // Method call optimized_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_optimized_send, + unspecialized_send_without_block_def_type_optimized_call, + unspecialized_send_without_block_def_type_optimized_block_call, + unspecialized_send_without_block_def_type_optimized_struct_aref, + unspecialized_send_without_block_def_type_optimized_struct_aset, + + // Method call def_type related to send fallback to dynamic dispatch + unspecialized_send_def_type_iseq, + unspecialized_send_def_type_cfunc, + unspecialized_send_def_type_attrset, + unspecialized_send_def_type_ivar, + unspecialized_send_def_type_bmethod, + unspecialized_send_def_type_zsuper, + unspecialized_send_def_type_alias, + unspecialized_send_def_type_undef, + unspecialized_send_def_type_not_implemented, + unspecialized_send_def_type_optimized, + unspecialized_send_def_type_missing, + unspecialized_send_def_type_refined, + unspecialized_send_def_type_null, + + // Super call def_type related to send fallback to dynamic dispatch + unspecialized_super_def_type_iseq, + unspecialized_super_def_type_cfunc, + unspecialized_super_def_type_attrset, + unspecialized_super_def_type_ivar, + unspecialized_super_def_type_bmethod, + unspecialized_super_def_type_zsuper, + unspecialized_super_def_type_alias, + unspecialized_super_def_type_undef, + unspecialized_super_def_type_not_implemented, + unspecialized_super_def_type_optimized, + unspecialized_super_def_type_missing, + unspecialized_super_def_type_refined, + unspecialized_super_def_type_null, + + // Unsupported parameter features + complex_arg_pass_param_rest, + complex_arg_pass_param_post, + complex_arg_pass_param_kwrest, + complex_arg_pass_param_block, + complex_arg_pass_param_forwardable, + complex_arg_pass_accepts_no_block, + complex_arg_pass_does_not_use_block, + + // Unsupported caller side features + complex_arg_pass_caller_splat, + complex_arg_pass_caller_blockarg, + complex_arg_pass_caller_kwarg, + complex_arg_pass_caller_kw_splat, + complex_arg_pass_caller_tailcall, + complex_arg_pass_caller_super, + complex_arg_pass_caller_zsuper, + complex_arg_pass_caller_forwarding, + + // Writes to the VM frame + vm_write_jit_frame_count, + vm_write_sp_count, + vm_write_locals_count, + vm_write_stack_count, + vm_write_to_parent_iseq_local_count, + // TODO(max): Implement + // vm_reify_stack_count, + + // The number of times we ran a dynamic check + guard_type_count, + guard_shape_count, + + load_field_count, + store_field_count, + + invokeblock_handler_monomorphic_iseq, + invokeblock_handler_monomorphic_ifunc, + invokeblock_handler_monomorphic_other, + invokeblock_handler_polymorphic, + invokeblock_handler_megamorphic, + invokeblock_handler_no_profiles, + + getblockparamproxy_handler_iseq, + getblockparamproxy_handler_ifunc, + getblockparamproxy_handler_symbol, + getblockparamproxy_handler_proc, + getblockparamproxy_handler_nil, + getblockparamproxy_handler_polymorphic, + getblockparamproxy_handler_megamorphic, + getblockparamproxy_handler_no_profiles, +} + +/// Increase a counter by a specified amount +pub fn incr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr += amount; } +} + +/// Decrease a counter by a specified amount +pub fn decr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr -= amount; } +} + +/// Increment a counter by its identifier +macro_rules! incr_counter { + ($counter_name:ident) => { + $crate::stats::incr_counter_by($crate::stats::Counter::$counter_name, 1) + } +} +pub(crate) use incr_counter; + +/// The number of side exits from each YARV instruction +pub type InsnCounters = [u64; VM_INSTRUCTION_SIZE as usize]; + +/// Return a raw pointer to the exit counter for a given YARV opcode +pub fn exit_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let exit_counters = ZJITState::get_exit_counters(); + unsafe { exit_counters.get_unchecked_mut(opcode as usize) } +} + +/// Return a raw pointer to the fallback counter for a given YARV opcode +pub fn send_fallback_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let fallback_counters = ZJITState::get_send_fallback_counters(); + unsafe { fallback_counters.get_unchecked_mut(opcode as usize) } +} + +/// Reason why ZJIT failed to produce any JIT code +#[derive(Clone, Debug, PartialEq)] +pub enum CompileError { + IseqVersionLimitReached, + IseqStackTooLarge, + NativeStackTooLarge, + ExceptionHandler, + OutOfMemory, + ParseError(ParseError), + /// When a ZJIT function is too large, the branches may have + /// offsets that don't fit in one instruction. We error in + /// error that case. + LabelLinkingFailure, +} + +/// Return a raw pointer to the exit counter for a given CompileError +pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { + use crate::hir::ParseError::*; + use crate::hir::ValidationError::*; + use crate::stats::CompileError::*; + use crate::stats::Counter::*; + match compile_error { + IseqVersionLimitReached => compile_error_iseq_version_limit_reached, + IseqStackTooLarge => compile_error_iseq_stack_too_large, + NativeStackTooLarge => compile_error_native_stack_too_large, + ExceptionHandler => compile_error_exception_handler, + OutOfMemory => compile_error_out_of_memory, + LabelLinkingFailure => compile_error_label_linking_failure, + ParseError(parse_error) => match parse_error { + StackUnderflow(_) => compile_error_parse_stack_underflow, + MalformedIseq(_) => compile_error_parse_malformed_iseq, + NotAllowed => compile_error_parse_not_allowed, + DirectiveInduced => compile_error_parse_directive_induced, + Validation(validation) => match validation { + BlockHasNoTerminator(_) => compile_error_validation_block_has_no_terminator, + TerminatorNotAtEnd(_, _, _) => compile_error_validation_terminator_not_at_end, + MismatchedBlockArity(_, _, _) => compile_error_validation_mismatched_block_arity, + JumpTargetNotInRPO(_) => compile_error_validation_jump_target_not_in_rpo, + OperandNotDefined(_, _, _) => compile_error_validation_operand_not_defined, + DuplicateInstruction(_, _) => compile_error_validation_duplicate_instruction, + MismatchedOperandType(..) => compile_error_validation_type_check_failure, + MiscValidationError(..) => compile_error_validation_misc_validation_error, + }, + } + } +} + +pub fn exit_counter_for_unhandled_hir_insn(insn: &crate::hir::Insn) -> Counter { + use crate::hir::Insn::*; + use crate::stats::Counter::*; + match insn { + ArrayMax { .. } => unhandled_hir_insn_array_max, + FixnumDiv { .. } => unhandled_hir_insn_fixnum_div, + Throw { .. } => unhandled_hir_insn_throw, + InvokeBuiltin { .. } => unhandled_hir_insn_invokebuiltin, + _ => unhandled_hir_insn_unknown, + } +} + +pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { + use crate::hir::SideExitReason::*; + use crate::hir::CallType::*; + use crate::hir::Invariant; + use crate::stats::Counter::*; + match reason { + UnhandledNewarraySend(send_type) => match send_type { + VM_OPT_NEWARRAY_SEND_MIN => exit_unhandled_newarray_send_min, + VM_OPT_NEWARRAY_SEND_HASH => exit_unhandled_newarray_send_hash, + VM_OPT_NEWARRAY_SEND_PACK => exit_unhandled_newarray_send_pack, + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => exit_unhandled_newarray_send_pack_buffer, + _ => exit_unhandled_newarray_send_unknown, + } + UnhandledDuparraySend(_) => exit_unhandled_duparray_send, + UnhandledCallType(Tailcall) => exit_unhandled_tailcall, + UnhandledCallType(Splat) => exit_unhandled_splat, + UnhandledCallType(Kwarg) => exit_unhandled_kwarg, + UnknownSpecialVariable(_) => exit_unknown_special_variable, + UnhandledHIRThrow => exit_unhandled_hir_insn, + UnhandledHIRInvokeBuiltin => exit_unhandled_hir_insn, + UnhandledHIRUnknown(_) => exit_unhandled_hir_insn, + UnhandledYARVInsn(_) => exit_unhandled_yarv_insn, + UnhandledBlockArg => exit_unhandled_block_arg, + FixnumAddOverflow => exit_fixnum_add_overflow, + FixnumSubOverflow => exit_fixnum_sub_overflow, + FixnumMultOverflow => exit_fixnum_mult_overflow, + FixnumLShiftOverflow => exit_fixnum_lshift_overflow, + FixnumModByZero => exit_fixnum_mod_by_zero, + FixnumDivByZero => exit_fixnum_div_by_zero, + BoxFixnumOverflow => exit_box_fixnum_overflow, + GuardType(_) => exit_guard_type_failure, + GuardShape(_) => exit_guard_shape_failure, + ExpandArray => exit_expandarray_failure, + GuardNotFrozen => exit_guard_not_frozen_failure, + GuardNotShared => exit_guard_not_shared_failure, + GuardLess => exit_guard_less_failure, + GuardGreaterEq => exit_guard_greater_eq_failure, + GuardSuperMethodEntry => exit_guard_super_method_entry, + CalleeSideExit => exit_callee_side_exit, + ObjToStringFallback => exit_obj_to_string_fallback, + Interrupt => exit_interrupt, + StackOverflow => exit_stackoverflow, + BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + BlockParamProxyNotNil => exit_block_param_proxy_not_nil, + BlockParamProxyFallbackMiss => exit_block_param_proxy_fallback_miss, + BlockParamProxyProfileNotCovered => exit_block_param_proxy_profile_not_covered, + BlockParamWbRequired => exit_block_param_wb_required, + TooManyKeywordParameters => exit_too_many_keyword_parameters, + SplatKwNotNilOrHash => exit_splatkw_not_nil_or_hash, + SplatKwPolymorphic => exit_splatkw_polymorphic, + SplatKwNotProfiled => exit_splatkw_not_profiled, + DirectiveInduced => exit_directive_induced, + PatchPoint(Invariant::BOPRedefined { .. }) + => exit_patchpoint_bop_redefined, + PatchPoint(Invariant::MethodRedefined { .. }) + => exit_patchpoint_method_redefined, + PatchPoint(Invariant::StableConstantNames { .. }) + => exit_patchpoint_stable_constant_names, + PatchPoint(Invariant::NoTracePoint) + => exit_patchpoint_no_tracepoint, + PatchPoint(Invariant::NoEPEscape(_)) + => exit_patchpoint_no_ep_escape, + PatchPoint(Invariant::SingleRactorMode) + => exit_patchpoint_single_ractor_mode, + PatchPoint(Invariant::NoSingletonClass { .. }) + => exit_patchpoint_no_singleton_class, + PatchPoint(Invariant::RootBoxOnly) + => exit_patchpoint_root_box_only, + SendWhileTracing => exit_send_while_tracing, + NoProfileSend => exit_no_profile_send, + InvokeBlockNotIfunc => exit_invokeblock_not_ifunc, + } +} + +pub fn exit_counter_ptr(reason: crate::hir::SideExitReason) -> *mut u64 { + let counter = side_exit_counter(reason); + counter_ptr(counter) +} + +pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter { + use crate::hir::SendFallbackReason::*; + use crate::stats::Counter::*; + match reason { + SendWithoutBlockPolymorphic => send_fallback_send_without_block_polymorphic, + SendWithoutBlockMegamorphic => send_fallback_send_without_block_megamorphic, + SendWithoutBlockNoProfiles => send_fallback_send_without_block_no_profiles, + SendWithoutBlockCfuncNotVariadic => send_fallback_send_without_block_cfunc_not_variadic, + SendWithoutBlockCfuncArrayVariadic => send_fallback_send_without_block_cfunc_array_variadic, + SendWithoutBlockNotOptimizedMethodType(_) => send_fallback_send_without_block_not_optimized_method_type, + SendWithoutBlockNotOptimizedMethodTypeOptimized(_) + => send_fallback_send_without_block_not_optimized_method_type_optimized, + SendWithoutBlockNotOptimizedNeedPermission + => send_fallback_send_without_block_not_optimized_need_permission, + TooManyArgsForLir => send_fallback_too_many_args_for_lir, + SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, + SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, + SendWithoutBlockPolymorphicFallback => send_fallback_send_without_block_polymorphic_fallback, + SendDirectKeywordMismatch => send_fallback_send_without_block_direct_keyword_mismatch, + SendDirectKeywordCountMismatch => send_fallback_send_without_block_direct_keyword_count_mismatch, + SendDirectMissingKeyword => send_fallback_send_without_block_direct_missing_keyword, + SendDirectTooManyKeywords => send_fallback_send_without_block_direct_too_many_keywords, + SendPolymorphic => send_fallback_send_polymorphic, + SendMegamorphic => send_fallback_send_megamorphic, + SendNoProfiles => send_fallback_send_no_profiles, + SendCfuncVariadic => send_fallback_send_cfunc_variadic, + SendCfuncArrayVariadic => send_fallback_send_cfunc_array_variadic, + ComplexArgPass => send_fallback_one_or_more_complex_arg_pass, + UnexpectedKeywordArgs => send_fallback_unexpected_keyword_args, + SingletonClassSeen => send_fallback_singleton_class_seen, + ArgcParamMismatch => send_fallback_argc_param_mismatch, + BmethodNonIseqProc => send_fallback_bmethod_non_iseq_proc, + SendNotOptimizedMethodType(_) => send_fallback_send_not_optimized_method_type, + SendNotOptimizedNeedPermission => send_fallback_send_not_optimized_need_permission, + CCallWithFrameTooManyArgs => send_fallback_ccall_with_frame_too_many_args, + ObjToStringNotString => send_fallback_obj_to_string_not_string, + SuperCallWithBlock => send_fallback_super_call_with_block, + SuperFromBlock => send_fallback_super_from_block, + SuperClassNotFound => send_fallback_super_class_not_found, + SuperComplexArgsPass => send_fallback_super_complex_args_pass, + SuperNoProfiles => send_fallback_super_fallback_no_profile, + SuperNotOptimizedMethodType(_) => send_fallback_super_not_optimized_method_type, + SuperPolymorphic => send_fallback_super_polymorphic, + SuperTargetNotFound => send_fallback_super_target_not_found, + SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + InvokeBlockNotSpecialized => send_fallback_invokeblock_not_specialized, + SendForwardNotSpecialized => send_fallback_sendforward_not_specialized, + InvokeSuperForwardNotSpecialized => send_fallback_invokesuperforward_not_specialized, + SingleRactorModeRequired => send_fallback_single_ractor_mode_required, + Uncategorized(_) => send_fallback_uncategorized, + } +} + +pub fn send_without_block_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_without_block_def_type_iseq, + Cfunc => unspecialized_send_without_block_def_type_cfunc, + Attrset => unspecialized_send_without_block_def_type_attrset, + Ivar => unspecialized_send_without_block_def_type_ivar, + Bmethod => unspecialized_send_without_block_def_type_bmethod, + Zsuper => unspecialized_send_without_block_def_type_zsuper, + Alias => unspecialized_send_without_block_def_type_alias, + Undefined => unspecialized_send_without_block_def_type_undef, + NotImplemented => unspecialized_send_without_block_def_type_not_implemented, + Optimized => unspecialized_send_without_block_def_type_optimized, + Missing => unspecialized_send_without_block_def_type_missing, + Refined => unspecialized_send_without_block_def_type_refined, + Null => unspecialized_send_without_block_def_type_null, + } +} + +pub fn send_without_block_fallback_counter_for_optimized_method_type(method_type: crate::hir::OptimizedMethodType) -> Counter { + use crate::hir::OptimizedMethodType::*; + use crate::stats::Counter::*; + + match method_type { + Send => unspecialized_send_without_block_def_type_optimized_send, + Call => unspecialized_send_without_block_def_type_optimized_call, + BlockCall => unspecialized_send_without_block_def_type_optimized_block_call, + StructAref => unspecialized_send_without_block_def_type_optimized_struct_aref, + StructAset => unspecialized_send_without_block_def_type_optimized_struct_aset, + } +} + +pub fn send_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_def_type_iseq, + Cfunc => unspecialized_send_def_type_cfunc, + Attrset => unspecialized_send_def_type_attrset, + Ivar => unspecialized_send_def_type_ivar, + Bmethod => unspecialized_send_def_type_bmethod, + Zsuper => unspecialized_send_def_type_zsuper, + Alias => unspecialized_send_def_type_alias, + Undefined => unspecialized_send_def_type_undef, + NotImplemented => unspecialized_send_def_type_not_implemented, + Optimized => unspecialized_send_def_type_optimized, + Missing => unspecialized_send_def_type_missing, + Refined => unspecialized_send_def_type_refined, + Null => unspecialized_send_def_type_null, + } +} + +pub fn send_fallback_counter_for_super_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_super_def_type_iseq, + Cfunc => unspecialized_super_def_type_cfunc, + Attrset => unspecialized_super_def_type_attrset, + Ivar => unspecialized_super_def_type_ivar, + Bmethod => unspecialized_super_def_type_bmethod, + Zsuper => unspecialized_super_def_type_zsuper, + Alias => unspecialized_super_def_type_alias, + Undefined => unspecialized_super_def_type_undef, + NotImplemented => unspecialized_super_def_type_not_implemented, + Optimized => unspecialized_super_def_type_optimized, + Missing => unspecialized_super_def_type_missing, + Refined => unspecialized_super_def_type_refined, + Null => unspecialized_super_def_type_null, + } +} + +/// Primitive called in zjit.rb. Zero out all the counters. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_reset_stats_bang(_ec: EcPtr, _self: VALUE) -> VALUE { + let counters = ZJITState::get_counters(); + let exit_counters = ZJITState::get_exit_counters(); + + // Reset all counters to zero + *counters = Counters::default(); + + // Reset exit counters for YARV instructions + exit_counters.as_mut_slice().fill(0); + + // Reset send fallback counters + ZJITState::get_send_fallback_counters().as_mut_slice().fill(0); + + // Reset not-inlined counters + ZJITState::get_not_inlined_cfunc_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset not-annotated counters + ZJITState::get_not_annotated_cfunc_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset ccall counters + ZJITState::get_ccall_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset iseq call counters + ZJITState::get_iseq_calls_count_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + Qnil +} + +/// Return a Hash object that contains ZJIT statistics +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE { + if !zjit_enabled_p() { + return Qnil; + } + + macro_rules! set_stat { + ($hash:ident, $key:expr, $value:expr) => { + let key = rust_str_to_sym($key); + if key == target_key { + return $value; + } else if $hash != Qnil { + #[allow(unused_unsafe)] + unsafe { rb_hash_aset($hash, key, $value); } + } + }; + } + + macro_rules! set_stat_usize { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, VALUE::fixnum_from_usize($value as usize)) + } + } + + macro_rules! set_stat_f64 { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, unsafe { rb_float_new($value) }) + } + } + + let hash = if target_key.nil_p() { + unsafe { rb_hash_new() } + } else { + Qnil + }; + + // Set default counters + for &counter in DEFAULT_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Memory usage stats + let code_region_bytes = ZJITState::get_code_block().mapped_region_size(); + set_stat_usize!(hash, "code_region_bytes", code_region_bytes); + set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes()); + set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes()); + + // End of default stats. Every counter beyond this is provided only for --zjit-stats. + if !get_option!(stats) { + return hash; + } + + // Set other stats-only counters + for &counter in OTHER_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Set side-exit counters for each SideExitReason + let mut side_exit_count = 0; + for &counter in EXIT_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + side_exit_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "side_exit_count", side_exit_count); + + // Set side-exit counters for UnhandledYARVInsn + let exit_counters = ZJITState::get_exit_counters(); + for (op_idx, count) in exit_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "unhandled_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Set send fallback counters for each DynamicSendReason + let mut dynamic_send_count = 0; + for &counter in DYNAMIC_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_send_count", dynamic_send_count); + + // Set optimized send counters + let mut optimized_send_count = 0; + for &counter in OPTIMIZED_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + optimized_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "optimized_send_count", optimized_send_count); + set_stat_usize!(hash, "send_count", dynamic_send_count + optimized_send_count); + + // Set send fallback counters for each setivar fallback reason + let mut dynamic_setivar_count = 0; + for &counter in DYNAMIC_SETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_setivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_setivar_count", dynamic_setivar_count); + + // Set send fallback counters for each getivar fallback reason + let mut dynamic_getivar_count = 0; + for &counter in DYNAMIC_GETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_getivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_getivar_count", dynamic_getivar_count); + + // Set send fallback counters for each definedivar fallback reason + let mut dynamic_definedivar_count = 0; + for &counter in DYNAMIC_DEFINEDIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_definedivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_definedivar_count", dynamic_definedivar_count); + + // Set send fallback counters for Uncategorized + let send_fallback_counters = ZJITState::get_send_fallback_counters(); + for (op_idx, count) in send_fallback_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "uncategorized_fallback_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Only ZJIT_STATS builds support rb_vm_insn_count + if unsafe { rb_vm_insn_count } > 0 { + let vm_insn_count = unsafe { rb_vm_insn_count }; + set_stat_usize!(hash, "vm_insn_count", vm_insn_count); + + let zjit_insn_count = ZJITState::get_counters().zjit_insn_count; + let total_insn_count = vm_insn_count + zjit_insn_count; + set_stat_usize!(hash, "total_insn_count", total_insn_count); + + set_stat_f64!(hash, "ratio_in_zjit", 100.0 * zjit_insn_count as f64 / total_insn_count as f64); + } + + // Set not inlined cfunc counters + let not_inlined_cfuncs = ZJITState::get_not_inlined_cfunc_counter_pointers(); + for (signature, counter) in not_inlined_cfuncs.iter() { + let key_string = format!("not_inlined_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set not annotated cfunc counters + let not_annotated_cfuncs = ZJITState::get_not_annotated_cfunc_counter_pointers(); + for (signature, counter) in not_annotated_cfuncs.iter() { + let key_string = format!("not_annotated_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set ccall counters + let ccall = ZJITState::get_ccall_counter_pointers(); + for (signature, counter) in ccall.iter() { + let key_string = format!("ccall_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set iseq access counters + let iseq_access_counts = ZJITState::get_iseq_calls_count_pointers(); + for (iseq_name, counter) in iseq_access_counts.iter() { + let key_string = format!("iseq_calls_count_{iseq_name}"); + set_stat_usize!(hash, &key_string, **counter); + } + + hash +} + +pub fn total_exit_count() -> u64 { + EXIT_COUNTERS.iter().fold(0, |sum, counter| sum + unsafe { *counter_ptr(*counter) }) +} + +/// Measure the time taken by func() and add that to zjit_compile_time. +pub fn with_time_stat<F, R>(counter: Counter, func: F) -> R where F: FnOnce() -> R { + let start = Instant::now(); + let ret = func(); + let nanos = Instant::now().duration_since(start).as_nanos(); + incr_counter_by(counter, nanos as u64); + ret +} + +/// The number of bytes ZJIT has allocated on the Rust heap. +pub fn zjit_alloc_bytes() -> usize { + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) +} + +/// Record a Perfetto duration event spanning the execution of `func`. +/// Uses Begin/End pairs so nested calls produce properly nested slices. +pub fn trace_compile_phase<F, R>(name: &str, func: F) -> R where F: FnOnce() -> R { + if !get_option!(trace_compiles) { + return func(); + } + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_begin("compile", name, ts, &[]); + } + let result = func(); + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_end("compile", name, ts); + } + result +} + +/// Fuchsia Trace Format (FXT) binary writer for --zjit-trace-exits. +/// Produces .fxt files that can be opened directly in Perfetto UI. +/// Uses the string table for deduplication of repeated reason/frame strings. +/// See: <https://fuchsia.dev/fuchsia-src/reference/tracing/trace-format> +pub struct PerfettoTracer { + writer: std::io::BufWriter<std::fs::File>, + start_time: std::time::Instant, + event_count: usize, + pub skipped_samples: usize, + /// String table: string content -> interned index (1..32767) + string_table: std::collections::HashMap<String, u16>, + next_string_index: u16, + pid: u32, +} + +impl PerfettoTracer { + /// Write a single 64-bit little-endian word. + fn write_word(&mut self, val: u64) { + use std::io::Write; + let _ = self.writer.write_all(&val.to_le_bytes()); + } + + /// Write bytes padded to 8-byte alignment. + fn write_padded_bytes(&mut self, bytes: &[u8]) { + use std::io::Write; + let _ = self.writer.write_all(bytes); + let remainder = bytes.len() % 8; + if remainder != 0 { + let _ = self.writer.write_all(&[0u8; 7][..8 - remainder]); + } + } + + /// Number of 8-byte words needed for `len` bytes (rounded up). + fn word_count(len: usize) -> u64 { + ((len + 7) / 8) as u64 + } + + pub fn new() -> Self { + let pid = std::process::id(); + let path = format!("/tmp/perfetto-{pid}.fxt"); + let tracer = Self::create(&path, pid); + eprintln!("ZJIT: writing trace exits to {path}"); + tracer + } + + fn create(path: &str, pid: u32) -> Self { + let file = std::fs::File::create(path) + .unwrap_or_else(|e| panic!("ZJIT: failed to create {path}: {e}")); + let mut tracer = PerfettoTracer { + writer: std::io::BufWriter::new(file), + start_time: std::time::Instant::now(), + event_count: 0, + skipped_samples: 0, + string_table: std::collections::HashMap::new(), + next_string_index: 1, // index 0 = empty string + pid, + }; + + // Magic number record: metadata type=4 (trace info), trace info type=0, + // magic=0x16547846 at bits [24..55] + tracer.write_word((1u64 << 4) | (4u64 << 16) | (0x16547846u64 << 24)); + + // Initialization record: 1 tick = 1 nanosecond + tracer.write_word(1u64 | (2u64 << 4)); + tracer.write_word(1_000_000_000u64); + + // Register thread at index 1: (process_koid=pid, thread_koid=1) + tracer.write_word(3u64 | (3u64 << 4) | (1u64 << 16)); + tracer.write_word(pid as u64); + tracer.write_word(1u64); + + // Kernel object record for process: type=7, obj_type=1 (ZX_OBJ_TYPE_PROCESS), no args + let process_name_ref = tracer.intern_string("ruby"); + let ko_process_header: u64 = 7u64 + | (2u64 << 4) // size = 2 words + | (1u64 << 16) // obj_type = ZX_OBJ_TYPE_PROCESS + | ((process_name_ref as u64) << 24); // name + tracer.write_word(ko_process_header); + tracer.write_word(pid as u64); // koid = process id + + // Kernel object record for thread: type=7, obj_type=2 (ZX_OBJ_TYPE_THREAD), 1 arg + let thread_name_ref = tracer.intern_string("main"); + let process_arg_name_ref = tracer.intern_string("process"); + let ko_thread_header: u64 = 7u64 + | (4u64 << 4) // size = 4 words (header + koid + 2-word arg) + | (2u64 << 16) // obj_type = ZX_OBJ_TYPE_THREAD + | ((thread_name_ref as u64) << 24) // name + | (1u64 << 40); // n_args = 1 + tracer.write_word(ko_thread_header); + tracer.write_word(1u64); // koid = thread id (matches thread record) + // Koid argument: type=8, size=2, name="process", value=pid + let arg_header: u64 = 8u64 | (2u64 << 4) | ((process_arg_name_ref as u64) << 16); + tracer.write_word(arg_header); + tracer.write_word(pid as u64); + + // Pre-intern common strings + tracer.intern_string("side_exit"); + tracer.intern_string("compile"); + tracer.intern_string("invalidation"); + // Pre-intern argument names "0".."14" for per-frame arguments + for i in 0..15u32 { + tracer.intern_string(&i.to_string()); + } + + // Flush header immediately so something is written even if process exits abruptly + { + use std::io::Write; + let _ = tracer.writer.flush(); + } + + tracer + } + + /// Intern a string into the string table, writing a string record if new. + /// Returns the string table index (1..32767). Returns 0 for empty strings + /// or if the table is full. + fn intern_string(&mut self, s: &str) -> u16 { + if s.is_empty() { + return 0; + } + if let Some(&idx) = self.string_table.get(s) { + return idx; + } + if self.next_string_index >= 0x8000 { + return 0; // table full + } + + let idx = self.next_string_index; + let bytes = s.as_bytes(); + let len = bytes.len().min(0x7FFF); // 15-bit max length + let record_words = 1 + Self::word_count(len); + + // String record: type=2, index in [16..30], length in [32..46] + let header: u64 = 2u64 + | (record_words << 4) + | ((idx as u64) << 16) + | ((len as u64) << 32); + self.write_word(header); + self.write_padded_bytes(&bytes[..len]); + + self.string_table.insert(s.to_string(), idx); + self.next_string_index += 1; + idx + } + + /// Return nanoseconds elapsed since tracer creation. + pub fn elapsed_ns(&self) -> u64 { + self.start_time.elapsed().as_nanos() as u64 + } + + /// Write a Duration Begin event (FXT event type 2) with optional frame arguments. + pub fn write_duration_begin(&mut self, category: &str, name: &str, ts_ns: u64, frames: &[String]) { + self.write_duration_event(2, category, name, ts_ns, frames); + } + + /// Write a Duration End event (FXT event type 3). + pub fn write_duration_end(&mut self, category: &str, name: &str, ts_ns: u64) { + self.write_duration_event(3, category, name, ts_ns, &[]); + } + + /// Write a Duration Begin or End event with optional frame arguments. + fn write_duration_event(&mut self, event_type: u64, category: &str, name: &str, ts_ns: u64, frames: &[String]) { + let category_ref = self.intern_string(category); + let name_ref = self.intern_string(name); + + let n_args = frames.len().min(15) as u64; + let mut frame_refs: Vec<(u16, u16)> = Vec::with_capacity(n_args as usize); + for (i, frame) in frames.iter().take(15).enumerate() { + let fname_ref = self.intern_string(&i.to_string()); + let value_ref = self.intern_string(frame); + frame_refs.push((fname_ref, value_ref)); + } + + let event_words: u64 = 2 + n_args; + let header: u64 = 4u64 // record type = event + | (event_words << 4) // record size + | (event_type << 16) // event type = begin or end + | (n_args << 20) // argument count + | (1u64 << 24) // thread_ref = 1 + | ((category_ref as u64) << 32) + | ((name_ref as u64) << 48); + self.write_word(header); + self.write_word(ts_ns); + + for (fname_ref, value_ref) in frame_refs { + let arg_header: u64 = 6u64 + | (1u64 << 4) + | ((fname_ref as u64) << 16) + | ((value_ref as u64) << 32); + self.write_word(arg_header); + } + + self.event_count += 1; + + use std::io::Write; + let _ = self.writer.flush(); + } + + pub fn write_event(&mut self, category: &str, reason: &str, frames: &[String]) { + let ts_nanos = self.start_time.elapsed().as_nanos() as u64; + + // Intern event metadata strings (may emit string records first) + let category_ref = self.intern_string(category); + let name_ref = self.intern_string(reason); + + // Intern each frame label and collect refs (max 15 due to 4-bit n_args) + let n_args = frames.len().min(15) as u64; + let mut frame_refs: Vec<(u16, u16)> = Vec::with_capacity(n_args as usize); + for (i, frame) in frames.iter().take(15).enumerate() { + let name_ref = self.intern_string(&i.to_string()); + let value_ref = self.intern_string(frame); + frame_refs.push((name_ref, value_ref)); + } + + // Each fully-interned string argument is exactly 1 word + let event_words = 2 + n_args; + let header: u64 = 4u64 + | (event_words << 4) + | (n_args << 20) // argument count + | (1u64 << 24) // thread_ref = 1 + | ((category_ref as u64) << 32) + | ((name_ref as u64) << 48); + self.write_word(header); + self.write_word(ts_nanos); + + // One 1-word string argument per frame: type=6, size=1, indexed name, indexed value + for (name_ref, value_ref) in frame_refs { + let arg_header: u64 = 6u64 + | (1u64 << 4) + | ((name_ref as u64) << 16) + | ((value_ref as u64) << 32); + self.write_word(arg_header); + } + + self.event_count += 1; + + // Flush to ensure data reaches disk. Static globals may not be + // dropped on process exit, so we can't rely on Drop for flushing. + use std::io::Write; + let _ = self.writer.flush(); + } +} + +impl Drop for PerfettoTracer { + fn drop(&mut self) { + use std::io::Write; + let _ = self.writer.flush(); + } +} + +/// Primitive called in zjit.rb +/// +/// Check if trace_exits generation is enabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_trace_exit_locations_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.trace_side_exits.is_some()) { + Qtrue + } else { + Qfalse + } +} diff --git a/zjit/src/ttycolors.rs b/zjit/src/ttycolors.rs new file mode 100644 index 0000000000..f325772431 --- /dev/null +++ b/zjit/src/ttycolors.rs @@ -0,0 +1,31 @@ +use std::io::IsTerminal; + +pub fn stdout_supports_colors() -> bool { + std::io::stdout().is_terminal() +} + +#[cfg_attr(not(feature = "disasm"), allow(dead_code))] +#[derive(Copy, Clone, Debug)] +pub struct TerminalColor { + pub bold_begin: &'static str, + pub bold_end: &'static str, +} + +pub static TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "\x1b[1m", + bold_end: "\x1b[22m", +}; + +pub static NON_TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "", + bold_end: "", +}; + +/// Terminal escape codes for colors, font weight, etc. Only enabled if stdout is a TTY. +pub fn get_colors() -> &'static TerminalColor { + if stdout_supports_colors() { + &TTY_TERMINAL_COLOR + } else { + &NON_TTY_TERMINAL_COLOR + } +} diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs new file mode 100644 index 0000000000..0088ef1a66 --- /dev/null +++ b/zjit/src/virtualmem.rs @@ -0,0 +1,504 @@ +//! Memory management stuff for ZJIT's code storage. Deals with virtual memory. +// I'm aware that there is an experiment in Rust Nightly right now for to see if banning +// usize->pointer casts is viable. It seems like a lot of work for us to participate for not much +// benefit. + +use std::ptr::NonNull; +use crate::cruby::*; +use crate::stats::zjit_alloc_bytes; + +pub type VirtualMem = VirtualMemory<sys::SystemAllocator>; + +/// Memory for generated executable machine code. When not testing, we reserve address space for +/// the entire region upfront and map physical memory into the reserved address space as needed. On +/// Linux, this is basically done using an `mmap` with `PROT_NONE` upfront and gradually using +/// `mprotect` with `PROT_READ|PROT_WRITE` as needed. The WIN32 equivalent seems to be +/// `VirtualAlloc` with `MEM_RESERVE` then later with `MEM_COMMIT`. +/// +/// This handles ["W^X"](https://en.wikipedia.org/wiki/W%5EX) semi-automatically. Writes +/// are always accepted and once writes are done a call to [Self::mark_all_executable] makes +/// the code in the region executable. +pub struct VirtualMemory<A: Allocator> { + /// Location of the virtual memory region. + region_start: NonNull<u8>, + + /// Size of this virtual memory region in bytes. + region_size_bytes: usize, + + /// mapped_region_bytes + zjit_alloc_size may not increase beyond this limit. + memory_limit_bytes: Option<usize>, + + /// Number of bytes per "page", memory protection permission can only be controlled at this + /// granularity. + page_size_bytes: usize, + + /// Number of bytes that have we have allocated physical memory for starting at + /// [Self::region_start]. + mapped_region_bytes: usize, + + /// Keep track of the address of the last written to page. + /// Used for changing protection to implement W^X. + current_write_page: Option<usize>, + + /// Zero size member for making syscalls to get physical memory during normal operation. + /// When testing this owns some memory. + allocator: A, +} + +/// Groups together the two syscalls to get get new physical memory and to change +/// memory protection. See [VirtualMemory] for details. +pub trait Allocator { + #[must_use] + fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool; + + fn mark_executable(&mut self, ptr: *const u8, size: u32); + + fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool; +} + +/// Pointer into a [VirtualMemory] represented as an offset from the base. +/// Note: there is no NULL constant for [CodePtr]. You should use `Option<CodePtr>` instead. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Debug)] +#[repr(C, packed)] +pub struct CodePtr(u32); + +impl CodePtr { + /// Advance the CodePtr. Can return a dangling pointer. + pub fn add_bytes(self, bytes: usize) -> Self { + let CodePtr(raw) = self; + let bytes: u32 = bytes.try_into().unwrap(); + CodePtr(raw + bytes) + } + + /// Subtract bytes from the CodePtr + pub fn sub_bytes(self, bytes: usize) -> Self { + let CodePtr(raw) = self; + let bytes: u32 = bytes.try_into().unwrap(); + CodePtr(raw.saturating_sub(bytes)) + } + + /// Note that the raw pointer might be dangling if there hasn't + /// been any writes to it through the [VirtualMemory] yet. + pub fn raw_ptr(self, base: &impl CodePtrBase) -> *const u8 { + let CodePtr(offset) = self; + base.base_ptr().as_ptr().wrapping_add(offset as usize) + } + + /// Get the address of the code pointer. + pub fn raw_addr(self, base: &impl CodePtrBase) -> usize { + self.raw_ptr(base).addr() + } + + /// Get the offset component for the code pointer. Useful finding the distance between two + /// code pointers that share the same [VirtualMem]. + pub fn as_offset(self) -> i64 { + let CodePtr(offset) = self; + offset.into() + } +} + +/// Errors that can happen when writing to [VirtualMemory] +#[derive(Debug, PartialEq)] +pub enum WriteError { + OutOfBounds, + FailedPageMapping, +} + +use WriteError::*; + +impl VirtualMem { + /// Allocate a VirtualMem insntace with a requested size + pub fn alloc(exec_mem_bytes: usize, mem_bytes: Option<usize>) -> Self { + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_bytes as u32) }; + + // Memory protection syscalls need page-aligned addresses, so check it here. Assuming + // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the + // page size in bytes is a power of two 2^19 or smaller. This is because the user + // requested size is half of mem_option * 2^20 as it's in MiB. + // + // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB + // (2^16 bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. + let page_size = unsafe { rb_jit_get_page_size() }; + assert_eq!( + virt_block as usize % page_size as usize, 0, + "Start of virtual address block should be page-aligned", + ); + + Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes) + } +} + +impl<A: Allocator> VirtualMemory<A> { + /// Bring a part of the address space under management. + pub fn new( + allocator: A, + page_size: u32, + virt_region_start: NonNull<u8>, + region_size_bytes: usize, + memory_limit_bytes: Option<usize>, + ) -> Self { + assert_ne!(0, page_size); + let page_size_bytes = page_size as usize; + + Self { + region_start: virt_region_start, + region_size_bytes, + memory_limit_bytes, + page_size_bytes, + mapped_region_bytes: 0, + current_write_page: None, + allocator, + } + } + + /// Return the start of the region as a raw pointer. Note that it could be a dangling + /// pointer so be careful dereferencing it. + pub fn start_ptr(&self) -> CodePtr { + CodePtr(0) + } + + pub fn mapped_end_ptr(&self) -> CodePtr { + self.start_ptr().add_bytes(self.mapped_region_bytes) + } + + pub fn virtual_end_ptr(&self) -> CodePtr { + self.start_ptr().add_bytes(self.region_size_bytes) + } + + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mapped_region_bytes + } + + /// Size of the region in bytes where writes could be attempted. + pub fn virtual_region_size(&self) -> usize { + self.region_size_bytes + } + + /// The granularity at which we can control memory permission. + /// On Linux, this is the page size that mmap(2) talks about. + pub fn system_page_size(&self) -> usize { + self.page_size_bytes + } + + /// Write a single byte. The first write to a page makes it readable. + pub fn write_byte(&mut self, write_ptr: CodePtr, byte: u8) -> Result<(), WriteError> { + let page_size = self.page_size_bytes; + let raw: *mut u8 = write_ptr.raw_ptr(self) as *mut u8; + let page_addr = (raw as usize / page_size) * page_size; + + if self.current_write_page == Some(page_addr) { + // Writing within the last written to page, nothing to do + } else { + // Switching to a different and potentially new page + let start = self.region_start.as_ptr(); + let mapped_region_end = start.wrapping_add(self.mapped_region_bytes); + let whole_region_end = start.wrapping_add(self.region_size_bytes); + let alloc = &mut self.allocator; + + // Ignore zjit_alloc_size() if self.memory_limit_bytes is None for testing + let mut required_region_bytes = page_addr + page_size - start as usize; + if self.memory_limit_bytes.is_some() { + required_region_bytes += zjit_alloc_bytes(); + } + + assert!((start..=whole_region_end).contains(&mapped_region_end)); + + if (start..mapped_region_end).contains(&raw) { + // Writing to a previously written to page. + // Need to make page writable, but no need to fill. + let page_size: u32 = page_size.try_into().unwrap(); + if !alloc.mark_writable(page_addr as *const _, page_size) { + return Err(FailedPageMapping); + } + + self.current_write_page = Some(page_addr); + } else if (start..whole_region_end).contains(&raw) && + required_region_bytes < self.memory_limit_bytes.unwrap_or(self.region_size_bytes) { + // Writing to a brand new page + let mapped_region_end_addr = mapped_region_end as usize; + let alloc_size = page_addr - mapped_region_end_addr + page_size; + + assert_eq!(0, alloc_size % page_size, "allocation size should be page aligned"); + assert_eq!(0, mapped_region_end_addr % page_size, "pointer should be page aligned"); + + if alloc_size > page_size { + // This is unusual for the current setup, so keep track of it. + //crate::stats::incr_counter!(exec_mem_non_bump_alloc); // TODO + } + + // Allocate new chunk + let alloc_size_u32: u32 = alloc_size.try_into().unwrap(); + unsafe { + if !alloc.mark_writable(mapped_region_end.cast(), alloc_size_u32) { + return Err(FailedPageMapping); + } + if cfg!(target_arch = "x86_64") { + // Fill new memory with PUSH DS (0x1E) so that executing uninitialized memory + // will fault with #UD in 64-bit mode. On Linux it becomes SIGILL and use the + // usual Ruby crash reporter. + std::slice::from_raw_parts_mut(mapped_region_end, alloc_size).fill(0x1E); + } else if cfg!(target_arch = "aarch64") { + // In aarch64, all zeros encodes UDF, so it's already what we want. + } else { + unreachable!("unknown arch"); + } + } + self.mapped_region_bytes += alloc_size; + + self.current_write_page = Some(page_addr); + } else { + return Err(OutOfBounds); + } + } + + // We have permission to write if we get here + unsafe { raw.write(byte) }; + + Ok(()) + } + + /// Return true if write_byte() can allocate a new page + pub fn can_allocate(&self) -> bool { + let memory_usage_bytes = self.mapped_region_bytes + zjit_alloc_bytes(); + let memory_limit_bytes = self.memory_limit_bytes.unwrap_or(self.region_size_bytes); + memory_usage_bytes + self.page_size_bytes < memory_limit_bytes + } + + /// Make all the code in the region writable. Call this before bulk writes (e.g. GC + /// reference updates). See [Self] for usual usage flow. + pub fn mark_all_writable(&mut self) { + self.current_write_page = None; + + let region_start = self.region_start; + let mapped_region_bytes: u32 = self.mapped_region_bytes.try_into().unwrap(); + + // Make mapped region writable + if mapped_region_bytes > 0 { + if !self.allocator.mark_writable(region_start.as_ptr(), mapped_region_bytes) { + panic!("Cannot make JIT memory region writable"); + } + } + } + + /// Make all the code in the region executable. Call this at the end of a write session. + /// See [Self] for usual usage flow. + pub fn mark_all_executable(&mut self) { + self.current_write_page = None; + + let region_start = self.region_start; + let mapped_region_bytes: u32 = self.mapped_region_bytes.try_into().unwrap(); + + // Make mapped region executable + self.allocator.mark_executable(region_start.as_ptr(), mapped_region_bytes); + } + + /// Free a range of bytes. start_ptr must be memory page-aligned. + pub fn free_bytes(&mut self, start_ptr: CodePtr, size: u32) { + assert_eq!(start_ptr.raw_ptr(self) as usize % self.page_size_bytes, 0); + + // Bounds check the request. We should only free memory we manage. + let mapped_region = self.start_ptr().raw_ptr(self)..self.mapped_end_ptr().raw_ptr(self); + let virtual_region = self.start_ptr().raw_ptr(self)..self.virtual_end_ptr().raw_ptr(self); + let last_byte_to_free = start_ptr.add_bytes(size.saturating_sub(1) as usize).raw_ptr(self); + assert!(mapped_region.contains(&start_ptr.raw_ptr(self))); + // On platforms where code page size != memory page size (e.g. Linux), we often need + // to free code pages that contain unmapped memory pages. When it happens on the last + // code page, it's more appropriate to check the last byte against the virtual region. + assert!(virtual_region.contains(&last_byte_to_free)); + + self.allocator.mark_unused(start_ptr.raw_ptr(self), size); + } +} + +/// Something that could provide a base pointer to compute a raw pointer from a [CodePtr]. +pub trait CodePtrBase { + fn base_ptr(&self) -> NonNull<u8>; +} + +impl<A: Allocator> CodePtrBase for VirtualMemory<A> { + fn base_ptr(&self) -> NonNull<u8> { + self.region_start + } +} + +/// Requires linking with CRuby to work +pub mod sys { + use crate::cruby::*; + + /// Zero size! This just groups together syscalls that require linking with CRuby. + pub struct SystemAllocator; + + type VoidPtr = *mut std::os::raw::c_void; + + impl super::Allocator for SystemAllocator { + fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } + } + + fn mark_executable(&mut self, ptr: *const u8, size: u32) { + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } + } + + fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } + } + } +} + + +#[cfg(test)] +pub mod tests { + use super::*; + + // Track allocation requests and owns some fixed size backing memory for requests. + // While testing we don't execute generated code. + pub struct TestingAllocator { + requests: Vec<AllocRequest>, + memory: Vec<u8>, + } + + #[derive(Debug)] + enum AllocRequest { + MarkWritable{ start_idx: usize, length: usize }, + MarkExecutable{ start_idx: usize, length: usize }, + MarkUnused, + } + use AllocRequest::*; + + impl TestingAllocator { + pub fn new(mem_size: usize) -> Self { + Self { requests: Vec::default(), memory: vec![0; mem_size] } + } + + pub fn mem_start(&self) -> *const u8 { + self.memory.as_ptr() + } + + // Verify that write_byte() bounds checks. Return `ptr` as an index. + fn bounds_check_request(&self, ptr: *const u8, size: u32) -> usize { + let mem_start = self.memory.as_ptr() as usize; + let index = ptr as usize - mem_start; + + assert!(index < self.memory.len()); + assert!(index + size as usize <= self.memory.len()); + + index + } + } + + // Bounds check and then record the request + impl super::Allocator for TestingAllocator { + fn mark_writable(&mut self, ptr: *const u8, length: u32) -> bool { + let index = self.bounds_check_request(ptr, length); + self.requests.push(MarkWritable { start_idx: index, length: length as usize }); + + true + } + + fn mark_executable(&mut self, ptr: *const u8, length: u32) { + let index = self.bounds_check_request(ptr, length); + self.requests.push(MarkExecutable { start_idx: index, length: length as usize }); + + // We don't try to execute generated code in cfg(test) + // so no need to actually request executable memory. + } + + fn mark_unused(&mut self, ptr: *const u8, length: u32) -> bool { + self.bounds_check_request(ptr, length); + self.requests.push(MarkUnused); + + true + } + } + + // Fictional architecture where each page is 4 bytes long + const PAGE_SIZE: usize = 4; + fn new_dummy_virt_mem() -> VirtualMemory<TestingAllocator> { + unsafe { + if crate::options::OPTIONS.is_none() { + crate::options::OPTIONS = Some(crate::options::Options::default()); + } + } + + let mem_size = PAGE_SIZE * 10; + let alloc = TestingAllocator::new(mem_size); + let mem_start: *const u8 = alloc.mem_start(); + + VirtualMemory::new( + alloc, + PAGE_SIZE.try_into().unwrap(), + NonNull::new(mem_start as *mut u8).unwrap(), + mem_size, + None, + ) + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn new_memory_is_initialized() { + let mut virt = new_dummy_virt_mem(); + + virt.write_byte(virt.start_ptr(), 1).unwrap(); + assert!( + virt.allocator.memory[..PAGE_SIZE].iter().all(|&byte| byte != 0), + "Entire page should be initialized", + ); + + // Skip a few page + let three_pages = 3 * PAGE_SIZE; + virt.write_byte(virt.start_ptr().add_bytes(three_pages), 1).unwrap(); + assert!( + virt.allocator.memory[..three_pages].iter().all(|&byte| byte != 0), + "Gaps between write requests should be filled", + ); + } + + #[test] + fn no_redundant_syscalls_when_writing_to_the_same_page() { + let mut virt = new_dummy_virt_mem(); + + virt.write_byte(virt.start_ptr(), 1).unwrap(); + virt.write_byte(virt.start_ptr(), 0).unwrap(); + + assert!( + matches!( + virt.allocator.requests[..], + [MarkWritable { start_idx: 0, length: PAGE_SIZE }], + ) + ); + } + + #[test] + fn bounds_checking() { + use super::WriteError::*; + let mut virt = new_dummy_virt_mem(); + + let one_past_end = virt.start_ptr().add_bytes(virt.virtual_region_size()); + assert_eq!(Err(OutOfBounds), virt.write_byte(one_past_end, 0)); + + let end_of_addr_space = CodePtr(u32::MAX); + assert_eq!(Err(OutOfBounds), virt.write_byte(end_of_addr_space, 0)); + } + + #[test] + fn only_written_to_regions_become_executable() { + // ... so we catch attempts to read/write/execute never-written-to regions + const THREE_PAGES: usize = PAGE_SIZE * 3; + let mut virt = new_dummy_virt_mem(); + let page_two_start = virt.start_ptr().add_bytes(PAGE_SIZE * 2); + virt.write_byte(page_two_start, 1).unwrap(); + virt.mark_all_executable(); + + assert!(virt.virtual_region_size() > THREE_PAGES); + assert!( + matches!( + virt.allocator.requests[..], + [ + MarkWritable { start_idx: 0, length: THREE_PAGES }, + MarkExecutable { start_idx: 0, length: THREE_PAGES }, + ] + ), + ); + } +} |