diff options
Diffstat (limited to 'zjit/src')
73 files changed, 55535 insertions, 0 deletions
diff --git a/zjit/src/asm/arm64/README.md b/zjit/src/asm/arm64/README.md new file mode 100644 index 0000000000..6adfad804d --- /dev/null +++ b/zjit/src/asm/arm64/README.md @@ -0,0 +1,16 @@ +# Arm64 + +This module is responsible for encoding ZJIT operands into an appropriate Arm64 encoding. + +## Architecture + +Every instruction in the Arm64 instruction set is 32 bits wide and is represented in little-endian order. Because they're all going to the same size, we represent each instruction by a struct that implements `From<T> for u32`, which contains the mechanism for encoding each instruction. The encoding for each instruction is shown in the documentation for the struct that ends up being created. + +In general each set of bytes inside of the struct has either a direct value (usually a `u8`/`u16`) or some kind of `enum` that can be converted directly into a `u32`. For more complicated pieces of encoding (e.g., bitmask immediates) a corresponding module under the `arg` namespace is available. + +## Helpful links + +* [Arm A64 Instruction Set Architecture](https://developer.arm.com/documentation/ddi0596/2021-12?lang=en) Official documentation +* [armconverter.com](https://armconverter.com/) A website that encodes Arm assembly syntax +* [hatstone](https://github.com/tenderlove/hatstone) A wrapper around the Capstone disassembler written in Ruby +* [onlinedisassembler.com](https://onlinedisassembler.com/odaweb/) A web-based disassembler diff --git a/zjit/src/asm/arm64/arg/bitmask_imm.rs b/zjit/src/asm/arm64/arg/bitmask_imm.rs new file mode 100644 index 0000000000..70a439afd5 --- /dev/null +++ b/zjit/src/asm/arm64/arg/bitmask_imm.rs @@ -0,0 +1,255 @@ +/// Immediates used by the logical immediate instructions are not actually the +/// immediate value, but instead are encoded into a 13-bit wide mask of 3 +/// elements. This allows many more values to be represented than 13 bits would +/// normally allow, at the expense of not being able to represent every possible +/// value. +/// +/// In order for a number to be encodeable in this form, the binary +/// representation must consist of a single set of contiguous 1s. That pattern +/// must then be replicatable across all of the bits either 1, 2, 4, 8, 16, or +/// 32 times (rotated or not). +/// +/// For example, 1 (0b1), 2 (0b10), 3 (0b11), and 4 (0b100) are all valid. +/// However, 5 (0b101) is invalid, because it contains 2 sets of 1s and cannot +/// be replicated across 64 bits. +/// +/// Some more examples to illustrate the idea of replication: +/// * 0x5555555555555555 is a valid value (0b0101...) because it consists of a +/// single set of 1s which can be replicated across all of the bits 32 times. +/// * 0xf0f0f0f0f0f0f0f0 is a valid value (0b1111000011110000...) because it +/// consists of a single set of 1s which can be replicated across all of the +/// bits 8 times (rotated by 4 bits). +/// * 0x0ff00ff00ff00ff0 is a valid value (0000111111110000...) because it +/// consists of a single set of 1s which can be replicated across all of the +/// bits 4 times (rotated by 12 bits). +/// +/// To encode the values, there are 3 elements: +/// * n = 1 if the pattern is 64-bits wide, 0 otherwise +/// * imms = the size of the pattern, a 0, and then one less than the number of +/// sequential 1s +/// * immr = the number of right rotations to apply to the pattern to get the +/// target value +/// +pub struct BitmaskImmediate { + n: u8, + imms: u8, + immr: u8 +} + +impl TryFrom<u64> for BitmaskImmediate { + type Error = (); + + /// Attempt to convert a u64 into a BitmaskImmediate. + /// + /// The implementation here is largely based on this blog post: + /// <https://dougallj.wordpress.com/2021/10/30/bit-twiddling-optimising-aarch64-logical-immediate-encoding-and-decoding/> + fn try_from(value: u64) -> Result<Self, Self::Error> { + if value == 0 || value == u64::MAX { + return Err(()); + } + + fn rotate_right(value: u64, rotations: u32) -> u64 { + (value >> (rotations & 0x3F)) | + (value << (rotations.wrapping_neg() & 0x3F)) + } + + let rotations = (value & (value + 1)).trailing_zeros(); + let normalized = rotate_right(value, rotations & 0x3F); + + let zeroes = normalized.leading_zeros(); + let ones = (!normalized).trailing_zeros(); + let size = zeroes + ones; + + if rotate_right(value, size & 0x3F) != value { + return Err(()); + } + + Ok(BitmaskImmediate { + n: ((size >> 6) & 1) as u8, + imms: (((size << 1).wrapping_neg() | (ones - 1)) & 0x3F) as u8, + immr: ((rotations.wrapping_neg() & (size - 1)) & 0x3F) as u8 + }) + } +} + +impl BitmaskImmediate { + /// Attempt to make a BitmaskImmediate for a 32 bit register. + /// The result has N==0, which is required for some 32-bit instructions. + /// Note that the exact same BitmaskImmediate produces different values + /// depending on the size of the target register. + pub fn new_32b_reg(value: u32) -> Result<Self, ()> { + // The same bit pattern replicated to u64 + let value = value as u64; + let replicated: u64 = (value << 32) | value; + let converted = Self::try_from(replicated); + if let Ok(ref imm) = converted { + assert_eq!(0, imm.n); + } + + converted + } +} + +impl BitmaskImmediate { + /// Encode a bitmask immediate into a 32-bit value. + pub fn encode(self) -> u32 { + 0 + | ((self.n as u32) << 12) + | ((self.immr as u32) << 6) + | (self.imms as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_failures() { + [5, 9, 10, 11, 13, 17, 18, 19].iter().for_each(|&imm| { + assert!(BitmaskImmediate::try_from(imm).is_err()); + }); + } + + #[test] + fn test_negative() { + let bitmask: BitmaskImmediate = (-9_i64 as u64).try_into().unwrap(); + let encoded: u32 = bitmask.encode(); + assert_eq!(7998, encoded); + } + + #[test] + fn test_size_2_minimum() { + let bitmask = BitmaskImmediate::try_from(0x5555555555555555); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b111100 }))); + } + + #[test] + fn test_size_2_maximum() { + let bitmask = BitmaskImmediate::try_from(0xaaaaaaaaaaaaaaaa); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000001, imms: 0b111100 }))); + } + + #[test] + fn test_size_4_minimum() { + let bitmask = BitmaskImmediate::try_from(0x1111111111111111); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b111000 }))); + } + + #[test] + fn test_size_4_rotated() { + let bitmask = BitmaskImmediate::try_from(0x6666666666666666); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000011, imms: 0b111001 }))); + } + + #[test] + fn test_size_4_maximum() { + let bitmask = BitmaskImmediate::try_from(0xeeeeeeeeeeeeeeee); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000011, imms: 0b111010 }))); + } + + #[test] + fn test_size_8_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0101010101010101); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b110000 }))); + } + + #[test] + fn test_size_8_rotated() { + let bitmask = BitmaskImmediate::try_from(0x1818181818181818); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000101, imms: 0b110001 }))); + } + + #[test] + fn test_size_8_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfefefefefefefefe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000111, imms: 0b110110 }))); + } + + #[test] + fn test_size_16_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0001000100010001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b100000 }))); + } + + #[test] + fn test_size_16_rotated() { + let bitmask = BitmaskImmediate::try_from(0xff8fff8fff8fff8f); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b001001, imms: 0b101100 }))); + } + + #[test] + fn test_size_16_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffefffefffefffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b001111, imms: 0b101110 }))); + } + + #[test] + fn test_size_32_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0000000100000001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b000000, imms: 0b000000 }))); + } + + #[test] + fn test_size_32_rotated() { + let bitmask = BitmaskImmediate::try_from(0x3fffff003fffff00); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b011000, imms: 0b010101 }))); + } + + #[test] + fn test_size_32_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffffffefffffffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 0, immr: 0b011111, imms: 0b011110 }))); + } + + #[test] + fn test_size_64_minimum() { + let bitmask = BitmaskImmediate::try_from(0x0000000000000001); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b000000, imms: 0b000000 }))); + } + + #[test] + fn test_size_64_rotated() { + let bitmask = BitmaskImmediate::try_from(0x0000001fffff0000); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b110000, imms: 0b010100 }))); + } + + #[test] + fn test_size_64_maximum() { + let bitmask = BitmaskImmediate::try_from(0xfffffffffffffffe); + assert!(matches!(bitmask, Ok(BitmaskImmediate { n: 1, immr: 0b111111, imms: 0b111110 }))); + } + + #[test] + fn test_size_64_invalid() { + let bitmask = BitmaskImmediate::try_from(u64::MAX); + assert!(matches!(bitmask, Err(()))); + } + + #[test] + fn test_all_valid_32b_pattern() { + let mut patterns = vec![]; + for pattern_size in [2, 4, 8, 16, 32_u64] { + for ones_count in 1..pattern_size { + for rotation in 0..pattern_size { + let ones = (1_u64 << ones_count) - 1; + let rotated = (ones >> rotation) | + ((ones & ((1 << rotation) - 1)) << (pattern_size - rotation)); + let mut replicated = rotated; + let mut shift = pattern_size; + while shift < 32 { + replicated |= replicated << shift; + shift *= 2; + } + let replicated: u32 = replicated.try_into().unwrap(); + assert!(BitmaskImmediate::new_32b_reg(replicated).is_ok()); + patterns.push(replicated); + } + } + } + patterns.sort(); + patterns.dedup(); + // Up to {size}-1 ones, and a total of {size} possible rotations. + assert_eq!(1*2 + 3*4 + 7*8 + 15*16 + 31*32, patterns.len()); + } +} diff --git a/zjit/src/asm/arm64/arg/condition.rs b/zjit/src/asm/arm64/arg/condition.rs new file mode 100644 index 0000000000..f711b8b0d8 --- /dev/null +++ b/zjit/src/asm/arm64/arg/condition.rs @@ -0,0 +1,52 @@ +/// Various instructions in A64 can have condition codes attached. This enum +/// includes all of the various kinds of conditions along with their respective +/// encodings. +pub struct Condition; + +impl Condition { + pub const EQ: u8 = 0b0000; // equal to + pub const NE: u8 = 0b0001; // not equal to + pub const CS: u8 = 0b0010; // carry set (alias for HS) + pub const CC: u8 = 0b0011; // carry clear (alias for LO) + pub const MI: u8 = 0b0100; // minus, negative + pub const PL: u8 = 0b0101; // positive or zero + pub const VS: u8 = 0b0110; // signed overflow + pub const VC: u8 = 0b0111; // no signed overflow + pub const HI: u8 = 0b1000; // greater than (unsigned) + pub const LS: u8 = 0b1001; // less than or equal to (unsigned) + pub const GE: u8 = 0b1010; // greater than or equal to (signed) + pub const LT: u8 = 0b1011; // less than (signed) + pub const GT: u8 = 0b1100; // greater than (signed) + pub const LE: u8 = 0b1101; // less than or equal to (signed) + pub const AL: u8 = 0b1110; // always + + pub const fn inverse(condition: u8) -> u8 { + match condition { + Condition::EQ => Condition::NE, + Condition::NE => Condition::EQ, + + Condition::CS => Condition::CC, + Condition::CC => Condition::CS, + + Condition::MI => Condition::PL, + Condition::PL => Condition::MI, + + Condition::VS => Condition::VC, + Condition::VC => Condition::VS, + + Condition::HI => Condition::LS, + Condition::LS => Condition::HI, + + Condition::LT => Condition::GE, + Condition::GE => Condition::LT, + + Condition::GT => Condition::LE, + Condition::LE => Condition::GT, + + Condition::AL => Condition::AL, + + _ => panic!("Unknown condition") + + } + } +} diff --git a/zjit/src/asm/arm64/arg/inst_offset.rs b/zjit/src/asm/arm64/arg/inst_offset.rs new file mode 100644 index 0000000000..f4a6bc73a0 --- /dev/null +++ b/zjit/src/asm/arm64/arg/inst_offset.rs @@ -0,0 +1,47 @@ +/// There are a lot of instructions in the AArch64 architectrue that take an +/// offset in terms of number of instructions. Usually they are jump +/// instructions or instructions that load a value relative to the current PC. +/// +/// This struct is used to mark those locations instead of a generic operand in +/// order to give better clarity to the developer when reading the AArch64 +/// backend code. It also helps to clarify that everything is in terms of a +/// number of instructions and not a number of bytes (i.e., the offset is the +/// number of bytes divided by 4). +#[derive(Copy, Clone)] +pub struct InstructionOffset(i32); + +impl InstructionOffset { + /// Create a new instruction offset. + pub fn from_insns(insns: i32) -> Self { + InstructionOffset(insns) + } + + /// Create a new instruction offset from a number of bytes. + pub fn from_bytes(bytes: i32) -> Self { + assert_eq!(bytes % 4, 0, "Byte offset must be a multiple of 4"); + InstructionOffset(bytes / 4) + } +} + +impl From<i32> for InstructionOffset { + /// Convert an i64 into an instruction offset. + fn from(value: i32) -> Self { + InstructionOffset(value) + } +} + +impl From<InstructionOffset> for i32 { + /// Convert an instruction offset into a number of instructions as an i32. + fn from(offset: InstructionOffset) -> Self { + offset.0 + } +} + +impl From<InstructionOffset> for i64 { + /// Convert an instruction offset into a number of instructions as an i64. + /// This is useful for when we're checking how many bits this offset fits + /// into. + fn from(offset: InstructionOffset) -> Self { + offset.0.into() + } +} diff --git a/zjit/src/asm/arm64/arg/mod.rs b/zjit/src/asm/arm64/arg/mod.rs new file mode 100644 index 0000000000..7eb37834f9 --- /dev/null +++ b/zjit/src/asm/arm64/arg/mod.rs @@ -0,0 +1,18 @@ +// This module contains various A64 instruction arguments and the logic +// necessary to encode them. + +mod bitmask_imm; +mod condition; +mod inst_offset; +mod sf; +mod shifted_imm; +mod sys_reg; +mod truncate; + +pub use bitmask_imm::BitmaskImmediate; +pub use condition::Condition; +pub use inst_offset::InstructionOffset; +pub use sf::Sf; +pub use shifted_imm::ShiftedImmediate; +pub use sys_reg::SystemRegister; +pub use truncate::{truncate_imm, truncate_uimm}; diff --git a/zjit/src/asm/arm64/arg/sf.rs b/zjit/src/asm/arm64/arg/sf.rs new file mode 100644 index 0000000000..b6091821e9 --- /dev/null +++ b/zjit/src/asm/arm64/arg/sf.rs @@ -0,0 +1,19 @@ +/// This is commonly the top-most bit in the encoding of the instruction, and +/// represents whether register operands should be treated as 64-bit registers +/// or 32-bit registers. +pub enum Sf { + Sf32 = 0b0, + Sf64 = 0b1 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Sf { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Sf::Sf64, + 32 => Sf::Sf32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} diff --git a/zjit/src/asm/arm64/arg/shifted_imm.rs b/zjit/src/asm/arm64/arg/shifted_imm.rs new file mode 100644 index 0000000000..06daefdef7 --- /dev/null +++ b/zjit/src/asm/arm64/arg/shifted_imm.rs @@ -0,0 +1,80 @@ +/// How much to shift the immediate by. +pub enum Shift { + LSL0 = 0b0, // no shift + LSL12 = 0b1 // logical shift left by 12 bits +} + +/// Some instructions accept a 12-bit immediate that has an optional shift +/// attached to it. This allows encoding larger values than just fit into 12 +/// bits. We attempt to encode those here. If the values are too large we have +/// to bail out. +pub struct ShiftedImmediate { + shift: Shift, + value: u16 +} + +impl TryFrom<u64> for ShiftedImmediate { + type Error = (); + + fn try_from(value: u64) -> Result<Self, Self::Error> { + let current = value; + if current < 2_u64.pow(12) { + return Ok(ShiftedImmediate { shift: Shift::LSL0, value: current as u16 }); + } + + if (current & (2_u64.pow(12) - 1) == 0) && ((current >> 12) < 2_u64.pow(12)) { + return Ok(ShiftedImmediate { shift: Shift::LSL12, value: (current >> 12) as u16 }); + } + + Err(()) + } +} + +impl From<ShiftedImmediate> for u32 { + /// Encode a bitmask immediate into a 32-bit value. + fn from(imm: ShiftedImmediate) -> Self { + 0 + | (((imm.shift as u32) & 1) << 12) + | (imm.value as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_no_shift() { + let expected_value = 256; + let result = ShiftedImmediate::try_from(expected_value); + + match result { + Ok(ShiftedImmediate { shift: Shift::LSL0, value }) => assert_eq!(value as u64, expected_value), + _ => panic!("Unexpected shift value") + } + } + + #[test] + fn test_maximum_no_shift() { + let expected_value = (1 << 12) - 1; + let result = ShiftedImmediate::try_from(expected_value); + + match result { + Ok(ShiftedImmediate { shift: Shift::LSL0, value }) => assert_eq!(value as u64, expected_value), + _ => panic!("Unexpected shift value") + } + } + + #[test] + fn test_with_shift() { + let result = ShiftedImmediate::try_from(256 << 12); + + assert!(matches!(result, Ok(ShiftedImmediate { shift: Shift::LSL12, value: 256 }))); + } + + #[test] + fn test_unencodable() { + let result = ShiftedImmediate::try_from((256 << 12) + 1); + assert!(matches!(result, Err(()))); + } +} diff --git a/zjit/src/asm/arm64/arg/sys_reg.rs b/zjit/src/asm/arm64/arg/sys_reg.rs new file mode 100644 index 0000000000..6229d5c1fd --- /dev/null +++ b/zjit/src/asm/arm64/arg/sys_reg.rs @@ -0,0 +1,6 @@ +/// The encoded representation of an A64 system register. +/// <https://developer.arm.com/documentation/ddi0601/2022-06/AArch64-Registers/> +pub enum SystemRegister { + /// <https://developer.arm.com/documentation/ddi0601/2022-06/AArch64-Registers/NZCV--Condition-Flags?lang=en> + NZCV = 0b1_011_0100_0010_000 +} diff --git a/zjit/src/asm/arm64/arg/truncate.rs b/zjit/src/asm/arm64/arg/truncate.rs new file mode 100644 index 0000000000..85d56ff202 --- /dev/null +++ b/zjit/src/asm/arm64/arg/truncate.rs @@ -0,0 +1,66 @@ +// There are many instances in AArch64 instruction encoding where you represent +// an integer value with a particular bit width that isn't a power of 2. These +// functions represent truncating those integer values down to the appropriate +// number of bits. + +/// Truncate a signed immediate to fit into a compile-time known width. It is +/// assumed before calling this function that the value fits into the correct +/// size. If it doesn't, then this function will panic. +/// +/// When the value is positive, this should effectively be a no-op since we're +/// just dropping leading zeroes. When the value is negative we should only be +/// dropping leading ones. +pub fn truncate_imm<T: Into<i32>, const WIDTH: usize>(imm: T) -> u32 { + let value: i32 = imm.into(); + let masked = (value as u32) & ((1 << WIDTH) - 1); + + // Assert that we didn't drop any bits by truncating. + if value >= 0 { + assert_eq!(value as u32, masked); + } else { + assert_eq!(value as u32, masked | (u32::MAX << WIDTH)); + } + + masked +} + +/// Truncate an unsigned immediate to fit into a compile-time known width. It is +/// assumed before calling this function that the value fits into the correct +/// size. If it doesn't, then this function will panic. +/// +/// This should effectively be a no-op since we're just dropping leading zeroes. +pub fn truncate_uimm<T: Into<u32>, const WIDTH: usize>(uimm: T) -> u32 { + let value: u32 = uimm.into(); + let masked = value & ((1 << WIDTH) - 1); + + // Assert that we didn't drop any bits by truncating. + assert_eq!(value, masked); + + masked +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_truncate_imm_positive() { + let inst = truncate_imm::<i32, 4>(5); + let result: u32 = inst; + assert_eq!(0b0101, result); + } + + #[test] + fn test_truncate_imm_negative() { + let inst = truncate_imm::<i32, 4>(-5); + let result: u32 = inst; + assert_eq!(0b1011, result); + } + + #[test] + fn test_truncate_uimm() { + let inst = truncate_uimm::<u32, 4>(5); + let result: u32 = inst; + assert_eq!(0b0101, result); + } +} diff --git a/zjit/src/asm/arm64/inst/atomic.rs b/zjit/src/asm/arm64/inst/atomic.rs new file mode 100644 index 0000000000..0917a4fd1c --- /dev/null +++ b/zjit/src/asm/arm64/inst/atomic.rs @@ -0,0 +1,86 @@ +/// The size of the register operands to this instruction. +enum Size { + /// Using 32-bit registers. + Size32 = 0b10, + + /// Using 64-bit registers. + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 atomic instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 1 1 1 0 0 0 0 0 0 | +/// | size rs.............. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Atomic { + /// The register holding the value to be loaded. + rt: u8, + + /// The base register. + rn: u8, + + /// The register holding the data value to be operated on. + rs: u8, + + /// The size of the registers used in this instruction. + size: Size +} + +impl Atomic { + /// LDADDAL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDADD--LDADDA--LDADDAL--LDADDL--Atomic-add-on-word-or-doubleword-in-memory-?lang=en> + pub fn ldaddal(rs: u8, rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<Atomic> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Atomic) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | (0b111 << 21) + | ((inst.rs as u32) << 16) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<Atomic> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Atomic) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldaddal() { + let result: u32 = Atomic::ldaddal(20, 21, 22, 64).into(); + assert_eq!(0xf8f402d5, result); + } +} diff --git a/zjit/src/asm/arm64/inst/branch.rs b/zjit/src/asm/arm64/inst/branch.rs new file mode 100644 index 0000000000..2db52e5d31 --- /dev/null +++ b/zjit/src/asm/arm64/inst/branch.rs @@ -0,0 +1,100 @@ +/// Which operation to perform. +enum Op { + /// Perform a BR instruction. + BR = 0b00, + + /// Perform a BLR instruction. + BLR = 0b01, + + /// Perform a RET instruction. + RET = 0b10 +} + +/// The struct that represents an A64 branch instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 1 0 0 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 | +/// | op... rn.............. rm.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Branch { + /// The register holding the address to be branched to. + rn: u8, + + /// The operation to perform. + op: Op +} + +impl Branch { + /// BR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/BR--Branch-to-Register-?lang=en> + pub fn br(rn: u8) -> Self { + Self { rn, op: Op::BR } + } + + /// BLR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/BLR--Branch-with-Link-to-Register-?lang=en> + pub fn blr(rn: u8) -> Self { + Self { rn, op: Op::BLR } + } + + /// RET + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/RET--Return-from-subroutine-?lang=en> + pub fn ret(rn: u8) -> Self { + Self { rn, op: Op::RET } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<Branch> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Branch) -> Self { + 0 + | (0b11 << 30) + | (FAMILY << 26) + | (1 << 25) + | ((inst.op as u32) << 21) + | (0b11111 << 16) + | ((inst.rn as u32) << 5) + } +} + +impl From<Branch> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Branch) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_br() { + let result: u32 = Branch::br(0).into(); + assert_eq!(0xd61f0000, result); + } + + #[test] + fn test_blr() { + let result: u32 = Branch::blr(0).into(); + assert_eq!(0xd63f0000, result); + } + + #[test] + fn test_ret() { + let result: u32 = Branch::ret(30).into(); + assert_eq!(0xd65f03c0, result); + } + + #[test] + fn test_ret_rn() { + let result: u32 = Branch::ret(20).into(); + assert_eq!(0xd65f0280, result); + } +} diff --git a/zjit/src/asm/arm64/inst/branch_cond.rs b/zjit/src/asm/arm64/inst/branch_cond.rs new file mode 100644 index 0000000000..266e9ccb31 --- /dev/null +++ b/zjit/src/asm/arm64/inst/branch_cond.rs @@ -0,0 +1,78 @@ +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The struct that represents an A64 conditional branch instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 1 0 0 0 | +/// | imm19........................................................... cond....... | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct BranchCond { + /// The kind of condition to check before branching. + cond: u8, + + /// The instruction offset from this instruction to branch to. + offset: InstructionOffset +} + +impl BranchCond { + /// B.cond + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/B-cond--Branch-conditionally-> + pub fn bcond(cond: u8, offset: InstructionOffset) -> Self { + Self { cond, offset } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<BranchCond> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: BranchCond) -> Self { + 0 + | (1 << 30) + | (FAMILY << 26) + | (truncate_imm::<_, 19>(inst.offset) << 5) + | (inst.cond as u32) + } +} + +impl From<BranchCond> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: BranchCond) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::super::arg::Condition; + + #[test] + fn test_b_eq() { + let result: u32 = BranchCond::bcond(Condition::EQ, 32.into()).into(); + assert_eq!(0x54000400, result); + } + + #[test] + fn test_b_vs() { + let result: u32 = BranchCond::bcond(Condition::VS, 32.into()).into(); + assert_eq!(0x54000406, result); + } + + #[test] + fn test_b_eq_max() { + let result: u32 = BranchCond::bcond(Condition::EQ, ((1 << 18) - 1).into()).into(); + assert_eq!(0x547fffe0, result); + } + + #[test] + fn test_b_eq_min() { + let result: u32 = BranchCond::bcond(Condition::EQ, (-(1 << 18)).into()).into(); + assert_eq!(0x54800000, result); + } +} diff --git a/zjit/src/asm/arm64/inst/breakpoint.rs b/zjit/src/asm/arm64/inst/breakpoint.rs new file mode 100644 index 0000000000..d66a35c4c6 --- /dev/null +++ b/zjit/src/asm/arm64/inst/breakpoint.rs @@ -0,0 +1,55 @@ +/// The struct that represents an A64 breakpoint instruction that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 0 0 0 1 0 0 0 0 0 | +/// | imm16.................................................. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Breakpoint { + /// The value to be captured by ESR_ELx.ISS + imm16: u16 +} + +impl Breakpoint { + /// BRK + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/BRK--Breakpoint-instruction-> + pub fn brk(imm16: u16) -> Self { + Self { imm16 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en#control> +const FAMILY: u32 = 0b101; + +impl From<Breakpoint> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Breakpoint) -> Self { + let imm16 = inst.imm16 as u32; + + 0 + | (0b11 << 30) + | (FAMILY << 26) + | (1 << 21) + | (imm16 << 5) + } +} + +impl From<Breakpoint> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Breakpoint) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_brk() { + let result: u32 = Breakpoint::brk(7).into(); + assert_eq!(0xd42000e0, result); + } +} diff --git a/zjit/src/asm/arm64/inst/call.rs b/zjit/src/asm/arm64/inst/call.rs new file mode 100644 index 0000000000..fd26d09f8a --- /dev/null +++ b/zjit/src/asm/arm64/inst/call.rs @@ -0,0 +1,104 @@ +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The operation to perform for this instruction. +enum Op { + /// Branch directly, with a hint that this is not a subroutine call or + /// return. + Branch = 0, + + /// Branch directly, with a hint that this is a subroutine call or return. + BranchWithLink = 1 +} + +/// The struct that represents an A64 branch with our without link instruction +/// that can be encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 0 1 | +/// | op imm26.................................................................................... | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Call { + /// The PC-relative offset to jump to in terms of number of instructions. + offset: InstructionOffset, + + /// The operation to perform for this instruction. + op: Op +} + +impl Call { + /// B + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B--Branch-> + pub fn b(offset: InstructionOffset) -> Self { + Self { offset, op: Op::Branch } + } + + /// BL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link-?lang=en> + pub fn bl(offset: InstructionOffset) -> Self { + Self { offset, op: Op::BranchWithLink } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b101; + +impl From<Call> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Call) -> Self { + 0 + | ((inst.op as u32) << 31) + | (FAMILY << 26) + | truncate_imm::<_, 26>(inst.offset) + } +} + +impl From<Call> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Call) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bl() { + let result: u32 = Call::bl(0.into()).into(); + assert_eq!(0x94000000, result); + } + + #[test] + fn test_bl_positive() { + let result: u32 = Call::bl(256.into()).into(); + assert_eq!(0x94000100, result); + } + + #[test] + fn test_bl_negative() { + let result: u32 = Call::bl((-256).into()).into(); + assert_eq!(0x97ffff00, result); + } + + #[test] + fn test_b() { + let result: u32 = Call::b(0.into()).into(); + assert_eq!(0x14000000, result); + } + + #[test] + fn test_b_positive() { + let result: u32 = Call::b(((1 << 25) - 1).into()).into(); + assert_eq!(0x15ffffff, result); + } + + #[test] + fn test_b_negative() { + let result: u32 = Call::b((-(1 << 25)).into()).into(); + assert_eq!(0x16000000, result); + } +} diff --git a/zjit/src/asm/arm64/inst/conditional.rs b/zjit/src/asm/arm64/inst/conditional.rs new file mode 100644 index 0000000000..1e26c7408b --- /dev/null +++ b/zjit/src/asm/arm64/inst/conditional.rs @@ -0,0 +1,73 @@ +use super::super::arg::Sf; + +/// The struct that represents an A64 conditional instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 1 0 1 0 1 0 0 0 0 | +/// | sf rm.............. cond....... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Conditional { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The condition to use for the conditional instruction. + cond: u8, + + /// The number of the second general-purpose source register. + rm: u8, + + /// The size of the registers of this instruction. + sf: Sf +} + +impl Conditional { + /// CSEL + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CSEL--Conditional-Select-?lang=en> + pub fn csel(rd: u8, rn: u8, rm: u8, cond: u8, num_bits: u8) -> Self { + Self { rd, rn, cond, rm, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en#condsel> +const FAMILY: u32 = 0b101; + +impl From<Conditional> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Conditional) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (1 << 28) + | (FAMILY << 25) + | (1 << 23) + | ((inst.rm as u32) << 16) + | ((inst.cond as u32) << 12) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<Conditional> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Conditional) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::super::arg::Condition; + + #[test] + fn test_csel() { + let result: u32 = Conditional::csel(0, 1, 2, Condition::NE, 64).into(); + assert_eq!(0x9a821020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/data_imm.rs b/zjit/src/asm/arm64/inst/data_imm.rs new file mode 100644 index 0000000000..ea71705478 --- /dev/null +++ b/zjit/src/asm/arm64/inst/data_imm.rs @@ -0,0 +1,143 @@ +use super::super::arg::{Sf, ShiftedImmediate}; + +/// The operation being performed by this instruction. +enum Op { + Add = 0b0, + Sub = 0b1 +} + +// Whether or not to update the flags when this instruction is performed. +enum S { + LeaveFlags = 0b0, + UpdateFlags = 0b1 +} + +/// The struct that represents an A64 data processing -- immediate instruction +/// that can be encoded. +/// +/// Add/subtract (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 0 1 0 | +/// | sf op S sh imm12.................................... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct DataImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// How much to shift the immediate by. + imm: ShiftedImmediate, + + /// Whether or not to update the flags when this instruction is performed. + s: S, + + /// The opcode for this instruction. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl DataImm { + /// ADD (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADD--immediate---Add--immediate--?lang=en> + pub fn add(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::LeaveFlags, op: Op::Add, sf: num_bits.into() } + } + + /// ADDS (immediate, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADDS--immediate---Add--immediate---setting-flags-?lang=en> + pub fn adds(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::UpdateFlags, op: Op::Add, sf: num_bits.into() } + } + + /// CMP (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CMP--immediate---Compare--immediate---an-alias-of-SUBS--immediate--?lang=en> + pub fn cmp(rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self::subs(31, rn, imm, num_bits) + } + + /// SUB (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUB--immediate---Subtract--immediate--?lang=en> + pub fn sub(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::LeaveFlags, op: Op::Sub, sf: num_bits.into() } + } + + /// SUBS (immediate, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUBS--immediate---Subtract--immediate---setting-flags-?lang=en> + pub fn subs(rd: u8, rn: u8, imm: ShiftedImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, s: S::UpdateFlags, op: Op::Sub, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<DataImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: DataImm) -> Self { + let imm: u32 = inst.imm.into(); + + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 30) + | ((inst.s as u32) << 29) + | (FAMILY << 25) + | (1 << 24) + | (imm << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<DataImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: DataImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + let inst = DataImm::add(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0x91001c20, result); + } + + #[test] + fn test_adds() { + let inst = DataImm::adds(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb1001c20, result); + } + + #[test] + fn test_cmp() { + let inst = DataImm::cmp(0, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf1001c1f, result); + } + + #[test] + fn test_sub() { + let inst = DataImm::sub(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xd1001c20, result); + } + + #[test] + fn test_subs() { + let inst = DataImm::subs(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf1001c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/data_reg.rs b/zjit/src/asm/arm64/inst/data_reg.rs new file mode 100644 index 0000000000..ed4afa956b --- /dev/null +++ b/zjit/src/asm/arm64/inst/data_reg.rs @@ -0,0 +1,192 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// The operation being performed by this instruction. +enum Op { + Add = 0b0, + Sub = 0b1 +} + +// Whether or not to update the flags when this instruction is performed. +enum S { + LeaveFlags = 0b0, + UpdateFlags = 0b1 +} + +/// The type of shift to perform on the second operand register. +enum Shift { + LSL = 0b00, // logical shift left (unsigned) + LSR = 0b01, // logical shift right (unsigned) + ASR = 0b10 // arithmetic shift right (signed) +} + +/// The struct that represents an A64 data processing -- register instruction +/// that can be encoded. +/// +/// Add/subtract (shifted register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 1 0 | +/// | sf op S shift rm.............. imm6............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct DataReg { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The amount to shift the second operand register by. + imm6: u8, + + /// The register number of the second operand register. + rm: u8, + + /// The type of shift to perform on the second operand register. + shift: Shift, + + /// Whether or not to update the flags when this instruction is performed. + s: S, + + /// The opcode for this instruction. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl DataReg { + /// ADD (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADD--shifted-register---Add--shifted-register--?lang=en> + pub fn add(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::LeaveFlags, + op: Op::Add, + sf: num_bits.into() + } + } + + /// ADDS (shifted register, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ADDS--shifted-register---Add--shifted-register---setting-flags-?lang=en> + pub fn adds(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::UpdateFlags, + op: Op::Add, + sf: num_bits.into() + } + } + + /// CMP (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CMP--shifted-register---Compare--shifted-register---an-alias-of-SUBS--shifted-register--?lang=en> + pub fn cmp(rn: u8, rm: u8, num_bits: u8) -> Self { + Self::subs(31, rn, rm, num_bits) + } + + /// SUB (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUB--shifted-register---Subtract--shifted-register--?lang=en> + pub fn sub(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::LeaveFlags, + op: Op::Sub, + sf: num_bits.into() + } + } + + /// SUBS (shifted register, set flags) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SUBS--shifted-register---Subtract--shifted-register---setting-flags-?lang=en> + pub fn subs(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { + rd, + rn, + imm6: 0, + rm, + shift: Shift::LSL, + s: S::UpdateFlags, + op: Op::Sub, + sf: num_bits.into() + } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en> +const FAMILY: u32 = 0b0101; + +impl From<DataReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: DataReg) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 30) + | ((inst.s as u32) << 29) + | (FAMILY << 25) + | (1 << 24) + | ((inst.shift as u32) << 22) + | ((inst.rm as u32) << 16) + | (truncate_uimm::<_, 6>(inst.imm6) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<DataReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: DataReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + let inst = DataReg::add(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0x8b020020, result); + } + + #[test] + fn test_adds() { + let inst = DataReg::adds(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xab020020, result); + } + + #[test] + fn test_cmp() { + let inst = DataReg::cmp(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xeb01001f, result); + } + + #[test] + fn test_sub() { + let inst = DataReg::sub(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xcb020020, result); + } + + #[test] + fn test_subs() { + let inst = DataReg::subs(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xeb020020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/halfword_imm.rs b/zjit/src/asm/arm64/inst/halfword_imm.rs new file mode 100644 index 0000000000..863ac947dd --- /dev/null +++ b/zjit/src/asm/arm64/inst/halfword_imm.rs @@ -0,0 +1,179 @@ +use super::super::arg::truncate_imm; + +/// Whether this is a load or a store. +enum Op { + Load = 1, + Store = 0 +} + +/// The type of indexing to perform for this instruction. +enum Index { + /// No indexing. + None = 0b00, + + /// Mutate the register after the read. + PostIndex = 0b01, + + /// Mutate the register before the read. + PreIndex = 0b11 +} + +/// The struct that represents an A64 halfword instruction that can be encoded. +/// +/// LDRH/STRH +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 1 1 0 0 1 0 | +/// | op imm12.................................... rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +/// LDRH (pre-index/post-index) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 1 1 0 0 0 0 0 | +/// | op imm9.......................... index rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct HalfwordImm { + /// The number of the 32-bit register to be loaded. + rt: u8, + + /// The number of the 64-bit base register to calculate the memory address. + rn: u8, + + /// The type of indexing to perform for this instruction. + index: Index, + + /// The immediate offset from the base register. + imm: i16, + + /// The operation to perform. + op: Op +} + +impl HalfwordImm { + /// LDRH + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh(rt: u8, rn: u8, imm12: i16) -> Self { + Self { rt, rn, index: Index::None, imm: imm12, op: Op::Load } + } + + /// LDRH (pre-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh_pre(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PreIndex, imm: imm9, op: Op::Load } + } + + /// LDRH (post-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate--> + pub fn ldrh_post(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PostIndex, imm: imm9, op: Op::Load } + } + + /// STRH + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh(rt: u8, rn: u8, imm12: i16) -> Self { + Self { rt, rn, index: Index::None, imm: imm12, op: Op::Store } + } + + /// STRH (pre-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh_pre(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PreIndex, imm: imm9, op: Op::Store } + } + + /// STRH (post-index) + /// <https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/STRH--immediate---Store-Register-Halfword--immediate--> + pub fn strh_post(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, index: Index::PostIndex, imm: imm9, op: Op::Store } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b111100; + +impl From<HalfwordImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: HalfwordImm) -> Self { + let (opc, imm) = match inst.index { + Index::None => { + assert_eq!(inst.imm & 1, 0, "immediate offset must be even"); + let imm12 = truncate_imm::<_, 12>(inst.imm / 2); + (0b100, imm12) + }, + Index::PreIndex | Index::PostIndex => { + let imm9 = truncate_imm::<_, 9>(inst.imm); + (0b000, (imm9 << 2) | (inst.index as u32)) + } + }; + + 0 + | (FAMILY << 25) + | ((opc | (inst.op as u32)) << 22) + | (imm << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<HalfwordImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: HalfwordImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldrh() { + let inst = HalfwordImm::ldrh(0, 1, 8); + let result: u32 = inst.into(); + assert_eq!(0x79401020, result); + } + + #[test] + fn test_ldrh_pre() { + let inst = HalfwordImm::ldrh_pre(0, 1, 16); + let result: u32 = inst.into(); + assert_eq!(0x78410c20, result); + } + + #[test] + fn test_ldrh_post() { + let inst = HalfwordImm::ldrh_post(0, 1, 24); + let result: u32 = inst.into(); + assert_eq!(0x78418420, result); + } + + #[test] + fn test_ldrh_post_negative() { + let inst = HalfwordImm::ldrh_post(0, 1, -24); + let result: u32 = inst.into(); + assert_eq!(0x785e8420, result); + } + + #[test] + fn test_strh() { + let inst = HalfwordImm::strh(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x79000020, result); + } + + #[test] + fn test_strh_pre() { + let inst = HalfwordImm::strh_pre(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78000c20, result); + } + + #[test] + fn test_strh_post() { + let inst = HalfwordImm::strh_post(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78000420, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_literal.rs b/zjit/src/asm/arm64/inst/load_literal.rs new file mode 100644 index 0000000000..37b5f3c7a7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_literal.rs @@ -0,0 +1,91 @@ +#![allow(clippy::identity_op)] + +use super::super::arg::{InstructionOffset, truncate_imm}; + +/// The size of the operands being operated on. +enum Opc { + Size32 = 0b00, + Size64 = 0b01, +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Opc { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Opc::Size64, + 32 => Opc::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load literal instruction that can be encoded. +/// +/// LDR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 0 0 0 | +/// | opc.. imm19........................................................... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadLiteral { + /// The number of the register to load the value into. + rt: u8, + + /// The PC-relative number of instructions to load the value from. + offset: InstructionOffset, + + /// The size of the operands being operated on. + opc: Opc +} + +impl LoadLiteral { + /// LDR (load literal) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--literal---Load-Register--literal--?lang=en> + pub fn ldr_literal(rt: u8, offset: InstructionOffset, num_bits: u8) -> Self { + Self { rt, offset, opc: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadLiteral> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadLiteral) -> Self { + 0 + | ((inst.opc as u32) << 30) + | (1 << 28) + | (FAMILY << 25) + | (truncate_imm::<_, 19>(inst.offset) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadLiteral> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadLiteral) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr_positive() { + let inst = LoadLiteral::ldr_literal(0, 5.into(), 64); + let result: u32 = inst.into(); + assert_eq!(0x580000a0, result); + } + + #[test] + fn test_ldr_negative() { + let inst = LoadLiteral::ldr_literal(0, (-5).into(), 64); + let result: u32 = inst.into(); + assert_eq!(0x58ffff60, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_register.rs b/zjit/src/asm/arm64/inst/load_register.rs new file mode 100644 index 0000000000..80813ffc87 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_register.rs @@ -0,0 +1,108 @@ +/// Whether or not to shift the register. +enum S { + Shift = 1, + NoShift = 0 +} + +/// The option for this instruction. +enum Option { + UXTW = 0b010, + LSL = 0b011, + SXTW = 0b110, + SXTX = 0b111 +} + +/// The size of the operands of this instruction. +enum Size { + Size32 = 0b10, + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load instruction that can be encoded. +/// +/// LDR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 0 1 1 1 0 | +/// | size. rm.............. option.. S rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadRegister { + /// The number of the register to load the value into. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// Whether or not to shift the value of the register. + s: S, + + /// The option associated with this instruction that controls the shift. + option: Option, + + /// The number of the offset register. + rm: u8, + + /// The size of the operands. + size: Size +} + +impl LoadRegister { + /// LDR + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--register---Load-Register--register--?lang=en> + pub fn ldr(rt: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rt, rn, s: S::NoShift, option: Option::LSL, rm, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadRegister> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadRegister) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | (0b11 << 21) + | ((inst.rm as u32) << 16) + | ((inst.option as u32) << 13) + | ((inst.s as u32) << 12) + | (0b10 << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadRegister> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadRegister) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr() { + let inst = LoadRegister::ldr(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8626820, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_store.rs b/zjit/src/asm/arm64/inst/load_store.rs new file mode 100644 index 0000000000..d38e851ed7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_store.rs @@ -0,0 +1,255 @@ +use super::super::arg::truncate_imm; + +/// The size of the operands being operated on. +enum Size { + Size8 = 0b00, + Size16 = 0b01, + Size32 = 0b10, + Size64 = 0b11, +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into an Sf enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The operation to perform for this instruction. +enum Opc { + STR = 0b00, + LDR = 0b01, + LDURSW = 0b10 +} + +/// What kind of indexing to perform for this instruction. +enum Index { + None = 0b00, + PostIndex = 0b01, + PreIndex = 0b11 +} + +/// The struct that represents an A64 load or store instruction that can be +/// encoded. +/// +/// LDR/LDUR/LDURSW/STR/STUR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 1 0 0 0 0 | +/// | size. opc.. imm9.......................... idx.. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadStore { + /// The number of the register to load the value into. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// What kind of indexing to perform for this instruction. + idx: Index, + + /// The optional signed immediate byte offset from the base register. + imm9: i16, + + /// The operation to perform for this instruction. + opc: Opc, + + /// The size of the operands being operated on. + size: Size +} + +impl LoadStore { + /// LDR (immediate, post-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDR--immediate---Load-Register--immediate--> + pub fn ldr_post(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PostIndex, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDR (immediate, pre-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDR--immediate---Load-Register--immediate--> + pub fn ldr_pre(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PreIndex, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDUR (load register, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled--?lang=en> + pub fn ldur(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: num_bits.into() } + } + + /// LDURH Load Register Halfword (unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURH--Load-Register-Halfword--unscaled--?lang=en> + pub fn ldurh(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: Size::Size16 } + } + + /// LDURB (load register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURB--Load-Register-Byte--unscaled--?lang=en> + pub fn ldurb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDR, size: Size::Size8 } + } + + /// LDURSW (load register, unscaled, signed) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDURSW--Load-Register-Signed-Word--unscaled--?lang=en> + pub fn ldursw(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::LDURSW, size: Size::Size32 } + } + + /// STR (immediate, post-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STR--immediate---Store-Register--immediate--> + pub fn str_post(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PostIndex, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STR (immediate, pre-index) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STR--immediate---Store-Register--immediate--> + pub fn str_pre(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::PreIndex, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STUR (store register, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STUR--Store-Register--unscaled--?lang=en> + pub fn stur(rt: u8, rn: u8, imm9: i16, num_bits: u8) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: num_bits.into() } + } + + /// STURH (store register, halfword, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturh(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size16 } + } + + /// STURB (store register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size8 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadStore> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadStore) -> Self { + 0 + | ((inst.size as u32) << 30) + | (0b11 << 28) + | (FAMILY << 25) + | ((inst.opc as u32) << 22) + | (truncate_imm::<_, 9>(inst.imm9) << 12) + | ((inst.idx as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadStore> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadStore) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldr_post() { + let inst = LoadStore::ldr_post(0, 1, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8410420, result); + } + + #[test] + fn test_ldr_pre() { + let inst = LoadStore::ldr_pre(0, 1, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8410c20, result); + } + + #[test] + fn test_ldur() { + let inst = LoadStore::ldur(0, 1, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8400020, result); + } + + #[test] + fn test_ldurb() { + let inst = LoadStore::ldurb(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x38400020, result); + } + + #[test] + fn test_ldurh() { + let inst = LoadStore::ldurh(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0x78400020, result); + } + + #[test] + fn test_ldur_with_imm() { + let inst = LoadStore::ldur(0, 1, 123, 64); + let result: u32 = inst.into(); + assert_eq!(0xf847b020, result); + } + + #[test] + fn test_ldursw() { + let inst = LoadStore::ldursw(0, 1, 0); + let result: u32 = inst.into(); + assert_eq!(0xb8800020, result); + } + + #[test] + fn test_ldursw_with_imm() { + let inst = LoadStore::ldursw(0, 1, 123); + let result: u32 = inst.into(); + assert_eq!(0xb887b020, result); + } + + #[test] + fn test_str_post() { + let inst = LoadStore::str_post(0, 1, -16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81f0420, result); + } + + #[test] + fn test_str_pre() { + let inst = LoadStore::str_pre(0, 1, -16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81f0c20, result); + } + + #[test] + fn test_stur() { + let inst = LoadStore::stur(0, 1, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf8000020, result); + } + + #[test] + fn test_stur_negative_offset() { + let inst = LoadStore::stur(0, 1, -1, 64); + let result: u32 = inst.into(); + assert_eq!(0xf81ff020, result); + } + + #[test] + fn test_stur_positive_offset() { + let inst = LoadStore::stur(0, 1, 255, 64); + let result: u32 = inst.into(); + assert_eq!(0xf80ff020, result); + } +} diff --git a/zjit/src/asm/arm64/inst/load_store_exclusive.rs b/zjit/src/asm/arm64/inst/load_store_exclusive.rs new file mode 100644 index 0000000000..30cb663bdb --- /dev/null +++ b/zjit/src/asm/arm64/inst/load_store_exclusive.rs @@ -0,0 +1,109 @@ +/// The operation being performed for this instruction. +enum Op { + Store = 0, + Load = 1 +} + +/// The size of the registers being operated on. +enum Size { + Size32 = 0b10, + Size64 = 0b11 +} + +/// A convenience function so that we can convert the number of bits of an +/// register operand directly into a Size enum variant. +impl From<u8> for Size { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Size::Size64, + 32 => Size::Size32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 load or store exclusive instruction that +/// can be encoded. +/// +/// LDAXR/STLXR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 0 0 0 0 1 1 1 1 1 1 | +/// | size. op rs.............. rn.............. rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LoadStoreExclusive { + /// The number of the register to be loaded. + rt: u8, + + /// The base register with which to form the address. + rn: u8, + + /// The register to be used for the status result if it applies to this + /// operation. Otherwise it's the zero register. + rs: u8, + + /// The operation being performed for this instruction. + op: Op, + + /// The size of the registers being operated on. + size: Size +} + +impl LoadStoreExclusive { + /// LDAXR + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/LDAXR--Load-Acquire-Exclusive-Register-> + pub fn ldaxr(rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs: 31, op: Op::Load, size: num_bits.into() } + } + + /// STLXR + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/STLXR--Store-Release-Exclusive-Register-> + pub fn stlxr(rs: u8, rt: u8, rn: u8, num_bits: u8) -> Self { + Self { rt, rn, rs, op: Op::Store, size: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<LoadStoreExclusive> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LoadStoreExclusive) -> Self { + 0 + | ((inst.size as u32) << 30) + | (FAMILY << 25) + | ((inst.op as u32) << 22) + | ((inst.rs as u32) << 16) + | (0b111111 << 10) + | ((inst.rn as u32) << 5) + | (inst.rt as u32) + } +} + +impl From<LoadStoreExclusive> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LoadStoreExclusive) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldaxr() { + let inst = LoadStoreExclusive::ldaxr(16, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xc85ffc10, result); + } + + #[test] + fn test_stlxr() { + let inst = LoadStoreExclusive::stlxr(17, 16, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xc811fc10, result); + } +} diff --git a/zjit/src/asm/arm64/inst/logical_imm.rs b/zjit/src/asm/arm64/inst/logical_imm.rs new file mode 100644 index 0000000000..d57ad5f5b7 --- /dev/null +++ b/zjit/src/asm/arm64/inst/logical_imm.rs @@ -0,0 +1,154 @@ +use super::super::arg::{BitmaskImmediate, Sf}; + +// Which operation to perform. +enum Opc { + /// The AND operation. + And = 0b00, + + /// The ORR operation. + Orr = 0b01, + + /// The EOR operation. + Eor = 0b10, + + /// The ANDS operation. + Ands = 0b11 +} + +/// The struct that represents an A64 bitwise immediate instruction that can be +/// encoded. +/// +/// AND/ORR/ANDS (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 0 | +/// | sf opc.. N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LogicalImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The immediate value to test. + imm: BitmaskImmediate, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl LogicalImm { + /// AND (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/AND--immediate---Bitwise-AND--immediate--?lang=en> + pub fn and(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::And, sf: num_bits.into() } + } + + /// ANDS (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ANDS--immediate---Bitwise-AND--immediate---setting-flags-?lang=en> + pub fn ands(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Ands, sf: num_bits.into() } + } + + /// EOR (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/EOR--immediate---Bitwise-Exclusive-OR--immediate--> + pub fn eor(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Eor, sf: num_bits.into() } + } + + /// MOV (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MOV--bitmask-immediate---Move--bitmask-immediate---an-alias-of-ORR--immediate--?lang=en> + pub fn mov(rd: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORR (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORR--immediate---Bitwise-OR--immediate--> + pub fn orr(rd: u8, rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self { rd, rn, imm, opc: Opc::Orr, sf: num_bits.into() } + } + + /// TST (bitmask immediate) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TST--immediate---Test-bits--immediate---an-alias-of-ANDS--immediate--?lang=en> + pub fn tst(rn: u8, imm: BitmaskImmediate, num_bits: u8) -> Self { + Self::ands(31, rn, imm, num_bits) + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#log_imm> +const FAMILY: u32 = 0b1001; + +impl From<LogicalImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LogicalImm) -> Self { + let imm: u32 = inst.imm.encode(); + + 0 + | ((inst.sf as u32) << 31) + | ((inst.opc as u32) << 29) + | (FAMILY << 25) + | (imm << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<LogicalImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LogicalImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_and() { + let inst = LogicalImm::and(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0x92400820, result); + } + + #[test] + fn test_ands() { + let inst = LogicalImm::ands(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf2400820, result); + } + + #[test] + fn test_eor() { + let inst = LogicalImm::eor(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xd2400820, result); + } + + #[test] + fn test_mov() { + let inst = LogicalImm::mov(0, 0x5555555555555555.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb200f3e0, result); + } + + #[test] + fn test_orr() { + let inst = LogicalImm::orr(0, 1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xb2400820, result); + } + + #[test] + fn test_tst() { + let inst = LogicalImm::tst(1, 7.try_into().unwrap(), 64); + let result: u32 = inst.into(); + assert_eq!(0xf240083f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/logical_reg.rs b/zjit/src/asm/arm64/inst/logical_reg.rs new file mode 100644 index 0000000000..18edff606f --- /dev/null +++ b/zjit/src/asm/arm64/inst/logical_reg.rs @@ -0,0 +1,207 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// Whether or not this is a NOT instruction. +enum N { + /// This is not a NOT instruction. + No = 0, + + /// This is a NOT instruction. + Yes = 1 +} + +/// The type of shift to perform on the second operand register. +enum Shift { + LSL = 0b00, // logical shift left (unsigned) + LSR = 0b01, // logical shift right (unsigned) + ASR = 0b10, // arithmetic shift right (signed) + ROR = 0b11 // rotate right (unsigned) +} + +// Which operation to perform. +enum Opc { + /// The AND operation. + And = 0b00, + + /// The ORR operation. + Orr = 0b01, + + /// The EOR operation. + Eor = 0b10, + + /// The ANDS operation. + Ands = 0b11 +} + +/// The struct that represents an A64 logical register instruction that can be +/// encoded. +/// +/// AND/ORR/ANDS (shifted register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 | +/// | sf opc.. shift N rm.............. imm6............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct LogicalReg { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The amount to shift the second operand register. + imm6: u8, + + /// The register number of the second operand register. + rm: u8, + + /// Whether or not this is a NOT instruction. + n: N, + + /// The type of shift to perform on the second operand register. + shift: Shift, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl LogicalReg { + /// AND (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/AND--shifted-register---Bitwise-AND--shifted-register--?lang=en> + pub fn and(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::And, sf: num_bits.into() } + } + + /// ANDS (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/ANDS--shifted-register---Bitwise-AND--shifted-register---setting-flags-?lang=en> + pub fn ands(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Ands, sf: num_bits.into() } + } + + /// EOR (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/EOR--shifted-register---Bitwise-Exclusive-OR--shifted-register--> + pub fn eor(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Eor, sf: num_bits.into() } + } + + /// MOV (register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register--?lang=en> + pub fn mov(rd: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// MVN (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/MVN--Bitwise-NOT--an-alias-of-ORN--shifted-register--?lang=en> + pub fn mvn(rd: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn: 0b11111, imm6: 0, rm, n: N::Yes, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORN (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORN--shifted-register---Bitwise-OR-NOT--shifted-register--> + pub fn orn(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::Yes, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// ORR (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ORR--shifted-register---Bitwise-OR--shifted-register--> + pub fn orr(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Orr, sf: num_bits.into() } + } + + /// TST (shifted register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TST--shifted-register---Test--shifted-register---an-alias-of-ANDS--shifted-register--?lang=en> + pub fn tst(rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd: 31, rn, imm6: 0, rm, n: N::No, shift: Shift::LSL, opc: Opc::Ands, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Register?lang=en> +const FAMILY: u32 = 0b0101; + +impl From<LogicalReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: LogicalReg) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.opc as u32) << 29) + | (FAMILY << 25) + | ((inst.shift as u32) << 22) + | ((inst.n as u32) << 21) + | ((inst.rm as u32) << 16) + | (truncate_uimm::<_, 6>(inst.imm6) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<LogicalReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: LogicalReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_and() { + let inst = LogicalReg::and(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0x8a020020, result); + } + + #[test] + fn test_ands() { + let inst = LogicalReg::ands(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xea020020, result); + } + + #[test] + fn test_eor() { + let inst = LogicalReg::eor(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xca020020, result); + } + + #[test] + fn test_mov() { + let inst = LogicalReg::mov(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa0103e0, result); + } + + #[test] + fn test_mvn() { + let inst = LogicalReg::mvn(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa2103e0, result); + } + + #[test] + fn test_orn() { + let inst = LogicalReg::orn(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa220020, result); + } + + #[test] + fn test_orr() { + let inst = LogicalReg::orr(0, 1, 2, 64); + let result: u32 = inst.into(); + assert_eq!(0xaa020020, result); + } + + #[test] + fn test_tst() { + let inst = LogicalReg::tst(0, 1, 64); + let result: u32 = inst.into(); + assert_eq!(0xea01001f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/madd.rs b/zjit/src/asm/arm64/inst/madd.rs new file mode 100644 index 0000000000..71f2ab230a --- /dev/null +++ b/zjit/src/asm/arm64/inst/madd.rs @@ -0,0 +1,73 @@ +use super::super::arg::Sf; + +/// The struct that represents an A64 multiply-add instruction that can be +/// encoded. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 1 0 1 1 0 0 0 0 | +/// | sf rm.............. ra.............. rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct MAdd { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The number of the third general-purpose source register. + ra: u8, + + /// The number of the second general-purpose source register. + rm: u8, + + /// The size of the registers of this instruction. + sf: Sf +} + +impl MAdd { + /// MUL + /// <https://developer.arm.com/documentation/ddi0602/2023-06/Base-Instructions/MUL--Multiply--an-alias-of-MADD-> + pub fn mul(rd: u8, rn: u8, rm: u8, num_bits: u8) -> Self { + Self { rd, rn, ra: 0b11111, rm, sf: num_bits.into() } + } +} + +impl From<MAdd> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: MAdd) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (0b11011 << 24) + | ((inst.rm as u32) << 16) + | ((inst.ra as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<MAdd> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: MAdd) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mul_32() { + let result: u32 = MAdd::mul(0, 1, 2, 32).into(); + assert_eq!(0x1B027C20, result); + } + + #[test] + fn test_mul_64() { + let result: u32 = MAdd::mul(0, 1, 2, 64).into(); + assert_eq!(0x9B027C20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/mod.rs b/zjit/src/asm/arm64/inst/mod.rs new file mode 100644 index 0000000000..bfffd914ef --- /dev/null +++ b/zjit/src/asm/arm64/inst/mod.rs @@ -0,0 +1,54 @@ +// This module contains various A64 instructions and the logic necessary to +// encode them into u32s. + +mod atomic; +mod branch; +mod branch_cond; +mod breakpoint; +mod call; +mod conditional; +mod data_imm; +mod data_reg; +mod halfword_imm; +mod load_literal; +mod load_register; +mod load_store; +mod load_store_exclusive; +mod logical_imm; +mod logical_reg; +mod madd; +mod smulh; +mod mov; +mod nop; +mod pc_rel; +mod reg_pair; +mod sbfm; +mod shift_imm; +mod sys_reg; +mod test_bit; + +pub use atomic::Atomic; +pub use branch::Branch; +pub use branch_cond::BranchCond; +pub use breakpoint::Breakpoint; +pub use call::Call; +pub use conditional::Conditional; +pub use data_imm::DataImm; +pub use data_reg::DataReg; +pub use halfword_imm::HalfwordImm; +pub use load_literal::LoadLiteral; +pub use load_register::LoadRegister; +pub use load_store::LoadStore; +pub use load_store_exclusive::LoadStoreExclusive; +pub use logical_imm::LogicalImm; +pub use logical_reg::LogicalReg; +pub use madd::MAdd; +pub use smulh::SMulH; +pub use mov::Mov; +pub use nop::Nop; +pub use pc_rel::PCRelative; +pub use reg_pair::RegisterPair; +pub use sbfm::SBFM; +pub use shift_imm::ShiftImm; +pub use sys_reg::SysReg; +pub use test_bit::TestBit; diff --git a/zjit/src/asm/arm64/inst/mov.rs b/zjit/src/asm/arm64/inst/mov.rs new file mode 100644 index 0000000000..e9f9091713 --- /dev/null +++ b/zjit/src/asm/arm64/inst/mov.rs @@ -0,0 +1,192 @@ +use super::super::arg::Sf; + +/// Which operation is being performed. +enum Op { + /// A movn operation which inverts the immediate and zeroes out the other bits. + MOVN = 0b00, + + /// A movz operation which zeroes out the other bits. + MOVZ = 0b10, + + /// A movk operation which keeps the other bits in place. + MOVK = 0b11 +} + +/// How much to shift the immediate by. +enum Hw { + LSL0 = 0b00, + LSL16 = 0b01, + LSL32 = 0b10, + LSL48 = 0b11 +} + +impl From<u8> for Hw { + fn from(shift: u8) -> Self { + match shift { + 0 => Hw::LSL0, + 16 => Hw::LSL16, + 32 => Hw::LSL32, + 48 => Hw::LSL48, + _ => panic!("Invalid value for shift: {shift}"), + } + } +} + +/// The struct that represents a MOVK or MOVZ instruction. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 0 1 | +/// | sf op... hw... imm16.................................................. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Mov { + /// The register number of the destination register. + rd: u8, + + /// The value to move into the register. + imm16: u16, + + /// The shift of the value to move. + hw: Hw, + + /// Which operation is being performed. + op: Op, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl Mov { + /// MOVK + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MOVK--Move-wide-with-keep-?lang=en> + pub fn movk(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVK, sf: num_bits.into() } + } + + /// MOVN + /// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> + pub fn movn(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVN, sf: num_bits.into() } + } + + /// MOVZ + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MOVZ--Move-wide-with-zero-?lang=en> + pub fn movz(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVZ, sf: num_bits.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<Mov> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Mov) -> Self { + 0 + | ((inst.sf as u32) << 31) + | ((inst.op as u32) << 29) + | (FAMILY << 25) + | (0b101 << 23) + | ((inst.hw as u32) << 21) + | ((inst.imm16 as u32) << 5) + | inst.rd as u32 + } +} + +impl From<Mov> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Mov) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_movk_unshifted() { + let inst = Mov::movk(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2800f60, result); + } + + #[test] + fn test_movn_unshifted() { + let inst = Mov::movn(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0x92800f60, result); + } + + #[test] + fn test_movn_shifted_16() { + let inst = Mov::movn(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0x92a00f60, result); + } + + #[test] + fn test_movn_shifted_32() { + let inst = Mov::movn(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0x92c00f60, result); + } + + #[test] + fn test_movn_shifted_48() { + let inst = Mov::movn(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0x92e00f60, result); + } + + #[test] + fn test_movk_shifted_16() { + let inst = Mov::movk(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2a00f60, result); + } + + #[test] + fn test_movk_shifted_32() { + let inst = Mov::movk(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2c00f60, result); + } + + #[test] + fn test_movk_shifted_48() { + let inst = Mov::movk(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0xf2e00f60, result); + } + + #[test] + fn test_movz_unshifted() { + let inst = Mov::movz(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2800f60, result); + } + + #[test] + fn test_movz_shifted_16() { + let inst = Mov::movz(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2a00f60, result); + } + + #[test] + fn test_movz_shifted_32() { + let inst = Mov::movz(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2c00f60, result); + } + + #[test] + fn test_movz_shifted_48() { + let inst = Mov::movz(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0xd2e00f60, result); + } +} diff --git a/zjit/src/asm/arm64/inst/nop.rs b/zjit/src/asm/arm64/inst/nop.rs new file mode 100644 index 0000000000..081d8558f5 --- /dev/null +++ b/zjit/src/asm/arm64/inst/nop.rs @@ -0,0 +1,44 @@ +/// The struct that represents an A64 nop instruction that can be encoded. +/// +/// NOP +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 1 0 0 0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 1 1 1 1 1 | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Nop; + +impl Nop { + /// NOP + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/NOP--No-Operation-> + pub fn nop() -> Self { + Self {} + } +} + +impl From<Nop> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(_inst: Nop) -> Self { + 0b11010101000000110010000000011111 + } +} + +impl From<Nop> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Nop) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_nop() { + let inst = Nop::nop(); + let result: u32 = inst.into(); + assert_eq!(0xd503201f, result); + } +} diff --git a/zjit/src/asm/arm64/inst/pc_rel.rs b/zjit/src/asm/arm64/inst/pc_rel.rs new file mode 100644 index 0000000000..2ea586a778 --- /dev/null +++ b/zjit/src/asm/arm64/inst/pc_rel.rs @@ -0,0 +1,107 @@ +/// Which operation to perform for the PC-relative instruction. +enum Op { + /// Form a PC-relative address. + ADR = 0, + + /// Form a PC-relative address to a 4KB page. + ADRP = 1 +} + +/// The struct that represents an A64 PC-relative address instruction that can +/// be encoded. +/// +/// ADR +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 0 0 | +/// | op immlo immhi........................................................... rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct PCRelative { + /// The number for the general-purpose register to load the address into. + rd: u8, + + /// The number of bytes to add to the PC to form the address. + imm: i32, + + /// Which operation to perform for this instruction. + op: Op +} + +impl PCRelative { + /// ADR + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/ADR--Form-PC-relative-address-> + pub fn adr(rd: u8, imm: i32) -> Self { + Self { rd, imm, op: Op::ADR } + } + + /// ADRP + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page-> + pub fn adrp(rd: u8, imm: i32) -> Self { + Self { rd, imm: imm >> 12, op: Op::ADRP } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en> +const FAMILY: u32 = 0b1000; + +impl From<PCRelative> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: PCRelative) -> Self { + let immlo = (inst.imm & 0b11) as u32; + let mut immhi = ((inst.imm >> 2) & ((1 << 18) - 1)) as u32; + + // Toggle the sign bit if necessary. + if inst.imm < 0 { + immhi |= 1 << 18; + } + + 0 + | ((inst.op as u32) << 31) + | (immlo << 29) + | (FAMILY << 25) + | (immhi << 5) + | inst.rd as u32 + } +} + +impl From<PCRelative> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: PCRelative) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_adr_positive() { + let inst = PCRelative::adr(0, 5); + let result: u32 = inst.into(); + assert_eq!(0x30000020, result); + } + + #[test] + fn test_adr_negative() { + let inst = PCRelative::adr(0, -5); + let result: u32 = inst.into(); + assert_eq!(0x70ffffc0, result); + } + + #[test] + fn test_adrp_positive() { + let inst = PCRelative::adrp(0, 0x4000); + let result: u32 = inst.into(); + assert_eq!(0x90000020, result); + } + + #[test] + fn test_adrp_negative() { + let inst = PCRelative::adrp(0, -0x4000); + let result: u32 = inst.into(); + assert_eq!(0x90ffffe0, result); + } +} diff --git a/zjit/src/asm/arm64/inst/reg_pair.rs b/zjit/src/asm/arm64/inst/reg_pair.rs new file mode 100644 index 0000000000..39a44c2416 --- /dev/null +++ b/zjit/src/asm/arm64/inst/reg_pair.rs @@ -0,0 +1,212 @@ +use super::super::arg::truncate_imm; + +/// The operation to perform for this instruction. +enum Opc { + /// When the registers are 32-bits wide. + Opc32 = 0b00, + + /// When the registers are 64-bits wide. + Opc64 = 0b10 +} + +/// The kind of indexing to perform for this instruction. +enum Index { + StorePostIndex = 0b010, + LoadPostIndex = 0b011, + StoreSignedOffset = 0b100, + LoadSignedOffset = 0b101, + StorePreIndex = 0b110, + LoadPreIndex = 0b111 +} + +/// A convenience function so that we can convert the number of bits of a +/// register operand directly into an Opc variant. +impl From<u8> for Opc { + fn from(num_bits: u8) -> Self { + match num_bits { + 64 => Opc::Opc64, + 32 => Opc::Opc32, + _ => panic!("Invalid number of bits: {num_bits}"), + } + } +} + +/// The struct that represents an A64 register pair instruction that can be +/// encoded. +/// +/// STP/LDP +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 0 1 0 0 | +/// | opc index..... imm7.................... rt2............. rn.............. rt1............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct RegisterPair { + /// The number of the first register to be transferred. + rt1: u8, + + /// The number of the base register. + rn: u8, + + /// The number of the second register to be transferred. + rt2: u8, + + /// The signed immediate byte offset, a multiple of 8. + imm7: i16, + + /// The kind of indexing to use for this instruction. + index: Index, + + /// The operation to be performed (in terms of size). + opc: Opc +} + +impl RegisterPair { + /// Create a register pair instruction with a given indexing mode. + fn new(rt1: u8, rt2: u8, rn: u8, disp: i16, index: Index, num_bits: u8) -> Self { + Self { rt1, rn, rt2, imm7: disp / 8, index, opc: num_bits.into() } + } + + /// LDP (signed offset) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>{, #<imm>}]` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadSignedOffset, num_bits) + } + + /// LDP (pre-index) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>, #<imm>]!` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp_pre(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadPreIndex, num_bits) + } + + /// LDP (post-index) + /// `LDP <Xt1>, <Xt2>, [<Xn|SP>], #<imm>` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-?lang=en> + pub fn ldp_post(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::LoadPostIndex, num_bits) + } + + /// STP (signed offset) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>{, #<imm>}]` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StoreSignedOffset, num_bits) + } + + /// STP (pre-index) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>, #<imm>]!` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp_pre(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StorePreIndex, num_bits) + } + + /// STP (post-index) + /// `STP <Xt1>, <Xt2>, [<Xn|SP>], #<imm>` + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-?lang=en> + pub fn stp_post(rt1: u8, rt2: u8, rn: u8, disp: i16, num_bits: u8) -> Self { + Self::new(rt1, rt2, rn, disp, Index::StorePostIndex, num_bits) + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> +const FAMILY: u32 = 0b0100; + +impl From<RegisterPair> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: RegisterPair) -> Self { + 0 + | ((inst.opc as u32) << 30) + | (1 << 29) + | (FAMILY << 25) + | ((inst.index as u32) << 22) + | (truncate_imm::<_, 7>(inst.imm7) << 15) + | ((inst.rt2 as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rt1 as u32) + } +} + +impl From<RegisterPair> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: RegisterPair) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ldp() { + let inst = RegisterPair::ldp(0, 1, 2, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9400440, result); + } + + #[test] + fn test_ldp_maximum_displacement() { + let inst = RegisterPair::ldp(0, 1, 2, 504, 64); + let result: u32 = inst.into(); + assert_eq!(0xa95f8440, result); + } + + #[test] + fn test_ldp_minimum_displacement() { + let inst = RegisterPair::ldp(0, 1, 2, -512, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9600440, result); + } + + #[test] + fn test_ldp_pre() { + let inst = RegisterPair::ldp_pre(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9d00440, result); + } + + #[test] + fn test_ldp_post() { + let inst = RegisterPair::ldp_post(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa8d00440, result); + } + + #[test] + fn test_stp() { + let inst = RegisterPair::stp(0, 1, 2, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9000440, result); + } + + #[test] + fn test_stp_maximum_displacement() { + let inst = RegisterPair::stp(0, 1, 2, 504, 64); + let result: u32 = inst.into(); + assert_eq!(0xa91f8440, result); + } + + #[test] + fn test_stp_minimum_displacement() { + let inst = RegisterPair::stp(0, 1, 2, -512, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9200440, result); + } + + #[test] + fn test_stp_pre() { + let inst = RegisterPair::stp_pre(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa9900440, result); + } + + #[test] + fn test_stp_post() { + let inst = RegisterPair::stp_post(0, 1, 2, 256, 64); + let result: u32 = inst.into(); + assert_eq!(0xa8900440, result); + } +} diff --git a/zjit/src/asm/arm64/inst/sbfm.rs b/zjit/src/asm/arm64/inst/sbfm.rs new file mode 100644 index 0000000000..12944ba722 --- /dev/null +++ b/zjit/src/asm/arm64/inst/sbfm.rs @@ -0,0 +1,103 @@ +use super::super::arg::{Sf, truncate_uimm}; + +/// The struct that represents an A64 signed bitfield move instruction that can +/// be encoded. +/// +/// SBFM +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 1 0 0 1 1 0 | +/// | sf N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SBFM { + /// The number for the general-purpose register to load the value into. + rd: u8, + + /// The number for the general-purpose register to copy from. + rn: u8, + + /// The leftmost bit number to be moved from the source. + imms: u8, + + // The right rotate amount. + immr: u8, + + /// Whether or not this is a 64-bit operation. + n: bool, + + /// The size of this operation. + sf: Sf +} + +impl SBFM { + /// ASR + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/ASR--immediate---Arithmetic-Shift-Right--immediate---an-alias-of-SBFM-?lang=en> + pub fn asr(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + let (imms, n) = if num_bits == 64 { + (0b111111, true) + } else { + (0b011111, false) + }; + + Self { rd, rn, immr: shift, imms, n, sf: num_bits.into() } + } + + /// SXTW + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/SXTW--Sign-Extend-Word--an-alias-of-SBFM-?lang=en> + pub fn sxtw(rd: u8, rn: u8) -> Self { + Self { rd, rn, immr: 0, imms: 31, n: true, sf: Sf::Sf64 } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#bitfield> +const FAMILY: u32 = 0b1001; + +impl From<SBFM> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SBFM) -> Self { + 0 + | ((inst.sf as u32) << 31) + | (FAMILY << 25) + | (1 << 24) + | ((inst.n as u32) << 22) + | (truncate_uimm::<_, 6>(inst.immr) << 16) + | (truncate_uimm::<_, 6>(inst.imms) << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<SBFM> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SBFM) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_asr_32_bits() { + let inst = SBFM::asr(0, 1, 2, 32); + let result: u32 = inst.into(); + assert_eq!(0x13027c20, result); + } + + #[test] + fn test_asr_64_bits() { + let inst = SBFM::asr(10, 11, 5, 64); + let result: u32 = inst.into(); + assert_eq!(0x9345fd6a, result); + } + + #[test] + fn test_sxtw() { + let inst = SBFM::sxtw(0, 1); + let result: u32 = inst.into(); + assert_eq!(0x93407c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/shift_imm.rs b/zjit/src/asm/arm64/inst/shift_imm.rs new file mode 100644 index 0000000000..9dac9a1408 --- /dev/null +++ b/zjit/src/asm/arm64/inst/shift_imm.rs @@ -0,0 +1,147 @@ +use super::super::arg::Sf; + +/// The operation to perform for this instruction. +enum Opc { + /// Logical left shift + LSL, + + /// Logical shift right + LSR +} + +/// The struct that represents an A64 unsigned bitfield move instruction that +/// can be encoded. +/// +/// LSL (immediate) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 1 0 0 1 1 0 | +/// | sf N immr............... imms............... rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct ShiftImm { + /// The register number of the destination register. + rd: u8, + + /// The register number of the first operand register. + rn: u8, + + /// The immediate value to shift by. + shift: u8, + + /// The opcode for this instruction. + opc: Opc, + + /// Whether or not this instruction is operating on 64-bit operands. + sf: Sf +} + +impl ShiftImm { + /// LSL (immediate) + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LSL--immediate---Logical-Shift-Left--immediate---an-alias-of-UBFM-?lang=en> + pub fn lsl(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + ShiftImm { rd, rn, shift, opc: Opc::LSL, sf: num_bits.into() } + } + + /// LSR (immediate) + /// <https://developer.arm.com/documentation/ddi0602/2021-12/Base-Instructions/LSR--immediate---Logical-Shift-Right--immediate---an-alias-of-UBFM-?lang=en> + pub fn lsr(rd: u8, rn: u8, shift: u8, num_bits: u8) -> Self { + ShiftImm { rd, rn, shift, opc: Opc::LSR, sf: num_bits.into() } + } + + /// Returns a triplet of (n, immr, imms) encoded in u32s for this + /// instruction. This mirrors how they will be encoded in the actual bits. + fn bitmask(&self) -> (u32, u32, u32) { + match self.opc { + // The key insight is a little buried in the docs, but effectively: + // LSL <Wd>, <Wn>, #<shift> == UBFM <Wd>, <Wn>, #(-<shift> MOD 32), #(31-<shift>) + // LSL <Xd>, <Xn>, #<shift> == UBFM <Xd>, <Xn>, #(-<shift> MOD 64), #(63-<shift>) + Opc::LSL => { + let shift = -(self.shift as i16); + + match self.sf { + Sf::Sf32 => ( + 0, + (shift.rem_euclid(32) & 0x3f) as u32, + ((31 - self.shift) & 0x3f) as u32 + ), + Sf::Sf64 => ( + 1, + (shift.rem_euclid(64) & 0x3f) as u32, + ((63 - self.shift) & 0x3f) as u32 + ) + } + }, + // Similar to LSL: + // LSR <Wd>, <Wn>, #<shift> == UBFM <Wd>, <Wn>, #<shift>, #31 + // LSR <Xd>, <Xn>, #<shift> == UBFM <Xd>, <Xn>, #<shift>, #63 + Opc::LSR => { + match self.sf { + Sf::Sf32 => (0, (self.shift & 0x3f) as u32, 31), + Sf::Sf64 => (1, (self.shift & 0x3f) as u32, 63) + } + } + } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Data-Processing----Immediate?lang=en#bitfield> +const FAMILY: u32 = 0b10011; + +impl From<ShiftImm> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: ShiftImm) -> Self { + let (n, immr, imms) = inst.bitmask(); + + 0 + | ((inst.sf as u32) << 31) + | (1 << 30) + | (FAMILY << 24) + | (n << 22) + | (immr << 16) + | (imms << 10) + | ((inst.rn as u32) << 5) + | inst.rd as u32 + } +} + +impl From<ShiftImm> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: ShiftImm) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lsl_32() { + let inst = ShiftImm::lsl(0, 1, 7, 32); + let result: u32 = inst.into(); + assert_eq!(0x53196020, result); + } + + #[test] + fn test_lsl_64() { + let inst = ShiftImm::lsl(0, 1, 7, 64); + let result: u32 = inst.into(); + assert_eq!(0xd379e020, result); + } + + #[test] + fn test_lsr_32() { + let inst = ShiftImm::lsr(0, 1, 7, 32); + let result: u32 = inst.into(); + assert_eq!(0x53077c20, result); + } + + #[test] + fn test_lsr_64() { + let inst = ShiftImm::lsr(0, 1, 7, 64); + let result: u32 = inst.into(); + assert_eq!(0xd347fc20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/smulh.rs b/zjit/src/asm/arm64/inst/smulh.rs new file mode 100644 index 0000000000..f355cb6531 --- /dev/null +++ b/zjit/src/asm/arm64/inst/smulh.rs @@ -0,0 +1,60 @@ +/// The struct that represents an A64 signed multiply high instruction +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 0 0 1 1 0 1 1 0 1 0 0 | +/// | rm.............. ra.............. rn.............. rd.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SMulH { + /// The number of the general-purpose destination register. + rd: u8, + + /// The number of the first general-purpose source register. + rn: u8, + + /// The number of the third general-purpose source register. + ra: u8, + + /// The number of the second general-purpose source register. + rm: u8, +} + +impl SMulH { + /// SMULH + /// <https://developer.arm.com/documentation/ddi0602/2023-06/Base-Instructions/SMULH--Signed-Multiply-High-> + pub fn smulh(rd: u8, rn: u8, rm: u8) -> Self { + Self { rd, rn, ra: 0b11111, rm } + } +} + +impl From<SMulH> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SMulH) -> Self { + 0 + | (0b10011011010 << 21) + | ((inst.rm as u32) << 16) + | ((inst.ra as u32) << 10) + | ((inst.rn as u32) << 5) + | (inst.rd as u32) + } +} + +impl From<SMulH> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SMulH) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_smulh() { + let result: u32 = SMulH::smulh(0, 1, 2).into(); + assert_eq!(0x9b427c20, result); + } +} diff --git a/zjit/src/asm/arm64/inst/sys_reg.rs b/zjit/src/asm/arm64/inst/sys_reg.rs new file mode 100644 index 0000000000..7191dfbfd9 --- /dev/null +++ b/zjit/src/asm/arm64/inst/sys_reg.rs @@ -0,0 +1,86 @@ +use super::super::arg::SystemRegister; + +/// Which operation to perform (loading or storing the system register value). +enum L { + /// Store the value of a general-purpose register in a system register. + MSR = 0, + + /// Store the value of a system register in a general-purpose register. + MRS = 1 +} + +/// The struct that represents an A64 system register instruction that can be +/// encoded. +/// +/// MSR/MRS (register) +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 1 1 0 1 0 1 0 1 0 0 1 | +/// | L o0 op1..... CRn........ CRm........ op2..... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct SysReg { + /// The register to load the system register value into. + rt: u8, + + /// Which system register to load or store. + systemreg: SystemRegister, + + /// Which operation to perform (loading or storing the system register value). + l: L +} + +impl SysReg { + /// MRS (register) + /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MRS--Move-System-Register-?lang=en> + pub fn mrs(rt: u8, systemreg: SystemRegister) -> Self { + SysReg { rt, systemreg, l: L::MRS } + } + + /// MSR (register) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MSR--register---Move-general-purpose-register-to-System-Register-?lang=en> + pub fn msr(systemreg: SystemRegister, rt: u8) -> Self { + SysReg { rt, systemreg, l: L::MSR } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en#systemmove> +const FAMILY: u32 = 0b110101010001; + +impl From<SysReg> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: SysReg) -> Self { + 0 + | (FAMILY << 20) + | ((inst.l as u32) << 21) + | ((inst.systemreg as u32) << 5) + | inst.rt as u32 + } +} + +impl From<SysReg> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: SysReg) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mrs() { + let inst = SysReg::mrs(0, SystemRegister::NZCV); + let result: u32 = inst.into(); + assert_eq!(0xd53b4200, result); + } + + #[test] + fn test_msr() { + let inst = SysReg::msr(SystemRegister::NZCV, 0); + let result: u32 = inst.into(); + assert_eq!(0xd51b4200, result); + } +} diff --git a/zjit/src/asm/arm64/inst/test_bit.rs b/zjit/src/asm/arm64/inst/test_bit.rs new file mode 100644 index 0000000000..45f0c2317e --- /dev/null +++ b/zjit/src/asm/arm64/inst/test_bit.rs @@ -0,0 +1,133 @@ +use super::super::arg::truncate_imm; + +/// The upper bit of the bit number to test. +#[derive(Debug)] +enum B5 { + /// When the bit number is below 32. + B532 = 0, + + /// When the bit number is equal to or above 32. + B564 = 1 +} + +/// A convenience function so that we can convert the bit number directly into a +/// B5 variant. +impl From<u8> for B5 { + fn from(bit_num: u8) -> Self { + match bit_num { + 0..=31 => B5::B532, + 32..=63 => B5::B564, + _ => panic!("Invalid bit number: {bit_num}"), + } + } +} + +/// The operation to perform for this instruction. +enum Op { + /// The test bit zero operation. + TBZ = 0, + + /// The test bit not zero operation. + TBNZ = 1 +} + +/// The struct that represents an A64 test bit instruction that can be encoded. +/// +/// TBNZ/TBZ +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 1 1 0 1 1 | +/// | b5 op b40............. imm14.......................................... rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct TestBit { + /// The number of the register to test. + rt: u8, + + /// The PC-relative offset to the target instruction in term of number of + /// instructions. + imm14: i16, + + /// The lower 5 bits of the bit number to be tested. + b40: u8, + + /// The operation to perform for this instruction. + op: Op, + + /// The upper bit of the bit number to test. + b5: B5 +} + +impl TestBit { + /// TBNZ + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TBNZ--Test-bit-and-Branch-if-Nonzero-?lang=en> + pub fn tbnz(rt: u8, bit_num: u8, offset: i16) -> Self { + Self { rt, imm14: offset, b40: bit_num & 0b11111, op: Op::TBNZ, b5: bit_num.into() } + } + + /// TBZ + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/TBZ--Test-bit-and-Branch-if-Zero-?lang=en> + pub fn tbz(rt: u8, bit_num: u8, offset: i16) -> Self { + Self { rt, imm14: offset, b40: bit_num & 0b11111, op: Op::TBZ, b5: bit_num.into() } + } +} + +/// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en> +const FAMILY: u32 = 0b11011; + +impl From<TestBit> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: TestBit) -> Self { + let b40 = (inst.b40 & 0b11111) as u32; + let imm14 = truncate_imm::<_, 14>(inst.imm14); + + 0 + | ((inst.b5 as u32) << 31) + | (FAMILY << 25) + | ((inst.op as u32) << 24) + | (b40 << 19) + | (imm14 << 5) + | inst.rt as u32 + } +} + +impl From<TestBit> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: TestBit) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tbnz() { + let inst = TestBit::tbnz(0, 0, 0); + let result: u32 = inst.into(); + assert_eq!(0x37000000, result); + } + + #[test] + fn test_tbnz_negative() { + let inst = TestBit::tbnz(0, 0, -1); + let result: u32 = inst.into(); + assert_eq!(0x3707ffe0, result); + } + + #[test] + fn test_tbz() { + let inst = TestBit::tbz(0, 0, 0); + let result: u32 = inst.into(); + assert_eq!(0x36000000, result); + } + + #[test] + fn test_tbz_negative() { + let inst = TestBit::tbz(0, 0, -1); + let result: u32 = inst.into(); + assert_eq!(0x3607ffe0, result); + } +} diff --git a/zjit/src/asm/arm64/mod.rs b/zjit/src/asm/arm64/mod.rs new file mode 100644 index 0000000000..c30520cd69 --- /dev/null +++ b/zjit/src/asm/arm64/mod.rs @@ -0,0 +1,1981 @@ +#![allow(dead_code)] // For instructions and operands we're not currently using. +#![allow(clippy::upper_case_acronyms)] +#![allow(clippy::identity_op)] +#![allow(clippy::self_named_constructors)] +#![allow(clippy::unusual_byte_groupings)] + +use crate::asm::CodeBlock; + +mod arg; +mod inst; +mod opnd; + +use inst::*; + +// We're going to make these public to make using these things easier in the +// backend (so they don't have to have knowledge about the submodule). +pub use arg::*; +pub use opnd::*; + +/// The extend type for register operands in extended register instructions. +/// It's the result size is determined by the destination register and +/// the source size interpreted using the last letter. +#[derive(Clone, Copy)] +pub enum ExtendType { + UXTB = 0b000, // unsigned extend byte + UXTH = 0b001, // unsigned extend halfword + UXTW = 0b010, // unsigned extend word + UXTX = 0b011, // unsigned extend doubleword + SXTB = 0b100, // signed extend byte + SXTH = 0b101, // signed extend halfword + SXTW = 0b110, // signed extend word + SXTX = 0b111, // signed extend doubleword +} + +/// Checks that a signed value fits within the specified number of bits. +pub const fn imm_fits_bits(imm: i64, num_bits: u8) -> bool { + let minimum = if num_bits == 64 { i64::MIN } else { -(2_i64.pow((num_bits as u32) - 1)) }; + let maximum = if num_bits == 64 { i64::MAX } else { 2_i64.pow((num_bits as u32) - 1) - 1 }; + + imm >= minimum && imm <= maximum +} + +/// Checks that an unsigned value fits within the specified number of bits. +pub const fn uimm_fits_bits(uimm: u64, num_bits: u8) -> bool { + let maximum = if num_bits == 64 { u64::MAX } else { 2_u64.pow(num_bits as u32) - 1 }; + + uimm <= maximum +} + +/// ADD - add rn and rm, put the result in rd, don't update flags +pub fn add(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::add(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::add(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::sub(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::add(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to add instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// Encode ADD (extended register) +/// +/// <https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions/ADD--extended-register---Add--extended-register--> +/// +/// 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00 +/// 0 1 0 1 1 0 0 1 │ │ │ │ │ │ │ │ │ │ +/// sf op S └────rm─────┘ └option┘ └─imm3─┘ └────rn─────┘ └────rd─────┘ +fn encode_add_extend(rd: u8, rn: u8, rm: u8, extend_type: ExtendType, shift: u8, num_bits: u8) -> [u8; 4] { + assert!(shift <= 4, "shift must be 0-4"); + + ((Sf::from(num_bits) as u32) << 31 | + 0b0 << 30 | // op = 0 for add + 0b0 << 29 | // S = 0 for non-flag-setting + 0b01011001 << 21 | + (rm as u32) << 16 | + (extend_type as u32) << 13 | + (shift as u32) << 10 | + (rn as u32) << 5 | + rd as u32).to_le_bytes() +} + +/// ADD (extended register) - add rn and rm with UXTX extension (no extension for 64-bit registers) +/// This is equivalent to a regular ADD for 64-bit registers since UXTX with shift 0 means no modification. +/// For reg_no=31, rd and rn mean SP while with rm means the zero register. +pub fn add_extended(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + encode_add_extend(rd.reg_no, rn.reg_no, rm.reg_no, ExtendType::UXTX, 0, rd.num_bits) + }, + _ => panic!("Invalid operand combination to add_extend instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADDS - add rn and rm, put the result in rd, update flags +pub fn adds(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::adds(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::adds(rd.reg_no, rn.reg_no, imm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::subs(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::adds(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to adds instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADR - form a PC-relative address and load it into a register +pub fn adr(cb: &mut CodeBlock, rd: A64Opnd, imm: A64Opnd) { + let bytes: [u8; 4] = match (rd, imm) { + (A64Opnd::Reg(rd), A64Opnd::Imm(imm)) => { + assert!(rd.num_bits == 64, "The destination register must be 64 bits."); + assert!(imm_fits_bits(imm, 21), "The immediate operand must be 21 bits or less."); + + PCRelative::adr(rd.reg_no, imm as i32).into() + }, + _ => panic!("Invalid operand combination to adr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ADRP - form a PC-relative address to a 4KB page and load it into a register. +/// This is effectively the same as ADR except that the immediate must be a +/// multiple of 4KB. +pub fn adrp(cb: &mut CodeBlock, rd: A64Opnd, imm: A64Opnd) { + let bytes: [u8; 4] = match (rd, imm) { + (A64Opnd::Reg(rd), A64Opnd::Imm(imm)) => { + assert!(rd.num_bits == 64, "The destination register must be 64 bits."); + assert!(imm_fits_bits(imm, 32), "The immediate operand must be 32 bits or less."); + + PCRelative::adrp(rd.reg_no, imm as i32).into() + }, + _ => panic!("Invalid operand combination to adr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// AND - and rn and rm, put the result in rd, don't update flags +pub fn and(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::and(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::and(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to and instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ANDS - and rn and rm, put the result in rd, update flags +pub fn ands(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::ands(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::ands(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to ands instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// ASR - arithmetic shift right rn by shift, put the result in rd, don't update +/// flags +pub fn asr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(shift)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + assert!(uimm_fits_bits(shift, 6), "The shift operand must be 6 bits or less."); + + SBFM::asr(rd.reg_no, rn.reg_no, shift.try_into().unwrap(), rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to asr instruction: asr {rd:?}, {rn:?}, {shift:?}"), + }; + + cb.write_bytes(&bytes); +} + +/// Whether or not the offset between two instructions fits into the branch with +/// or without link instruction. If it doesn't, then we have to load the value +/// into a register first. +pub const fn b_offset_fits_bits(offset: i64) -> bool { + imm_fits_bits(offset, 26) +} + +/// B - branch without link (offset is number of instructions to jump) +pub fn b(cb: &mut CodeBlock, offset: InstructionOffset) { + assert!(b_offset_fits_bits(offset.into()), "The immediate operand must be 26 bits or less."); + let bytes: [u8; 4] = Call::b(offset).into(); + + cb.write_bytes(&bytes); +} + +/// Whether or not the offset in number of instructions between two instructions +/// fits into the b.cond instruction. If it doesn't, then we have to load the +/// value into a register first, then use the b.cond instruction to skip past a +/// direct jump. +pub const fn bcond_offset_fits_bits(offset: i64) -> bool { + imm_fits_bits(offset, 19) +} + +/// B.cond - branch to target if condition is true +pub fn bcond(cb: &mut CodeBlock, cond: u8, offset: InstructionOffset) { + _ = Condition; + assert!(bcond_offset_fits_bits(offset.into()), "The offset must be 19 bits or less."); + let bytes: [u8; 4] = BranchCond::bcond(cond, offset).into(); + + cb.write_bytes(&bytes); +} + +/// BL - branch with link (offset is number of instructions to jump) +pub fn bl(cb: &mut CodeBlock, offset: InstructionOffset) { + assert!(b_offset_fits_bits(offset.into()), "The offset must be 26 bits or less."); + let bytes: [u8; 4] = Call::bl(offset).into(); + + cb.write_bytes(&bytes); +} + +/// BLR - branch with link to a register +pub fn blr(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::Reg(rn) => Branch::blr(rn.reg_no).into(), + _ => panic!("Invalid operand to blr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// BR - branch to a register +pub fn br(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::Reg(rn) => Branch::br(rn.reg_no).into(), + _ => panic!("Invalid operand to br instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// BRK - create a breakpoint +pub fn brk(cb: &mut CodeBlock, imm16: A64Opnd) { + let bytes: [u8; 4] = match imm16 { + A64Opnd::None => Breakpoint::brk(0xf000).into(), + A64Opnd::UImm(imm16) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + Breakpoint::brk(imm16 as u16).into() + }, + _ => panic!("Invalid operand combination to brk instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// CMP - compare rn and rm, update flags +pub fn cmp(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rn, rm) { + (A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::cmp(rn.reg_no, rm.reg_no, rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + DataImm::cmp(rn.reg_no, (imm12 as u64).try_into().unwrap(), rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::UImm(imm12)) => { + DataImm::cmp(rn.reg_no, imm12.try_into().unwrap(), rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to cmp instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// CSEL - conditionally select between two registers +pub fn csel(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd, cond: u8) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + Conditional::csel(rd.reg_no, rn.reg_no, rm.reg_no, cond, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to csel instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// EOR - perform a bitwise XOR of rn and rm, put the result in rd, don't update flags +pub fn eor(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::eor(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::eor(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to eor instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDADDAL - atomic add with acquire and release semantics +pub fn ldaddal(cb: &mut CodeBlock, rs: A64Opnd, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rs, rt, rn) { + (A64Opnd::Reg(rs), A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert!( + rs.num_bits == rt.num_bits && rt.num_bits == rn.num_bits, + "All operands must be of the same size." + ); + + Atomic::ldaddal(rs.reg_no, rt.reg_no, rn.reg_no, rs.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldaddal instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDAXR - atomic load with acquire semantics +pub fn ldaxr(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert_eq!(rn.num_bits, 64, "rn must be a 64-bit register."); + + LoadStoreExclusive::ldaxr(rt.reg_no, rn.reg_no, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldaxr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDP (signed offset) - load a pair of registers from memory +pub fn ldp(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDP (pre-index) - load a pair of registers from memory, update the base pointer before loading it +pub fn ldp_pre(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp_pre(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDP (post-index) - load a pair of registers from memory, update the base pointer after loading it +pub fn ldp_post(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::ldp_post(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDR - load a memory address into a register with a register offset +pub fn ldr(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn, rm) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LoadRegister::ldr(rt.reg_no, rn.reg_no, rm.reg_no, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDR - load a PC-relative memory address into a register +pub fn ldr_literal(cb: &mut CodeBlock, rt: A64Opnd, rn: InstructionOffset) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + LoadLiteral::ldr_literal(rt.reg_no, rn, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDRH - load a halfword from memory +pub fn ldrh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 12), "The displacement must be 12 bits or less."); + + HalfwordImm::ldrh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDRH (pre-index) - load a halfword from memory, update the base pointer before loading it +pub fn ldrh_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::ldrh_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LDRH (post-index) - load a halfword from memory, update the base pointer after loading it +pub fn ldrh_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::ldrh_post(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldrh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// Whether or not a memory address displacement fits into the maximum number of +/// bits such that it can be used without loading it into a register first. +pub fn mem_disp_fits_bits(disp: i32) -> bool { + imm_fits_bits(disp.into(), 9) +} + +/// LDR (post-index) - load a register from memory, update the base pointer after loading it +pub fn ldr_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::ldr_post(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDR (pre-index) - load a register from memory, update the base pointer before loading it +pub fn ldr_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::ldr_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to ldr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// LDUR - load a memory address into a register +pub fn ldur(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + + LoadStore::ldur(rt.reg_no, rn.reg_no, 0, rt.num_bits).into() + }, + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldur(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operands for LDUR") + }; + + cb.write_bytes(&bytes); +} + +/// LDURH - load a byte from memory, zero-extend it, and write it to a register +pub fn ldurh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldurh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operands for LDURH") + }; + + cb.write_bytes(&bytes); +} + +/// LDURB - load a byte from memory, zero-extend it, and write it to a register +pub fn ldurb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(rt.num_bits == 8, "Expected registers to have size 8"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldurb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operands for LDURB") + }; + + cb.write_bytes(&bytes); +} + +/// LDURSW - load a 32-bit memory address into a register and sign-extend it +pub fn ldursw(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::ldursw(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to ldursw instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// LSL - logical shift left a register by an immediate +pub fn lsl(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm)) => { + assert!(rd.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(uimm_fits_bits(uimm, 6), "Expected shift to be 6 bits or less"); + + ShiftImm::lsl(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() + }, + _ => panic!("Invalid operands combination {rd:?} {rn:?} {shift:?} to lsl instruction") + }; + + cb.write_bytes(&bytes); +} + +/// LSR - logical shift right a register by an immediate +pub fn lsr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, shift) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm)) => { + assert!(rd.num_bits == rn.num_bits, "Expected registers to be the same size"); + assert!(uimm_fits_bits(uimm, 6), "Expected shift to be 6 bits or less"); + + ShiftImm::lsr(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() + }, + _ => panic!("Invalid operands combination to lsr instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MOV - move a value in a register to another register +pub fn mov(cb: &mut CodeBlock, rd: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rm) { + (A64Opnd::Reg(A64Reg { reg_no: 31, num_bits: 64 }), A64Opnd::Reg(rm)) => { + assert!(rm.num_bits == 64, "Expected rm to be 64 bits"); + + DataImm::add(31, rm.reg_no, 0.try_into().unwrap(), 64).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(A64Reg { reg_no: 31, num_bits: 64 })) => { + assert!(rd.num_bits == 64, "Expected rd to be 64 bits"); + + DataImm::add(rd.reg_no, 31, 0.try_into().unwrap(), 64).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::mov(rd.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::UImm(0)) => { + LogicalReg::mov(rd.reg_no, XZR_REG.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::UImm(imm)) => { + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::mov(rd.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mov instruction: {rd:?}, {rm:?}") + }; + + cb.write_bytes(&bytes); +} + +/// MOVK - move a 16 bit immediate into a register, keep the other bits in place +pub fn movk(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movk(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movk instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MOVN - load a register with the complement of a shifted then zero extended 16-bit immediate +/// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> +pub fn movn(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movn(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movn instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MOVZ - move a 16 bit immediate into a register, zero the other bits +pub fn movz(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movz(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// MRS - move a system register into a general-purpose register +pub fn mrs(cb: &mut CodeBlock, rt: A64Opnd, systemregister: SystemRegister) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + SysReg::mrs(rt.reg_no, systemregister).into() + }, + _ => panic!("Invalid operand combination to mrs instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MSR - move a general-purpose register into a system register +pub fn msr(cb: &mut CodeBlock, systemregister: SystemRegister, rt: A64Opnd) { + let bytes: [u8; 4] = match rt { + A64Opnd::Reg(rt) => { + SysReg::msr(systemregister, rt.reg_no).into() + }, + _ => panic!("Invalid operand combination to msr instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MUL - multiply two registers, put the result in a third register +pub fn mul(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + MAdd::mul(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mul instruction") + }; + + cb.write_bytes(&bytes); +} + +/// SMULH - multiply two 64-bit registers to produce a 128-bit result, put the high 64-bits of the result into rd +pub fn smulh(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + assert!(rd.num_bits == 64, "smulh only applicable to 64-bit registers"); + + SMulH::smulh(rd.reg_no, rn.reg_no, rm.reg_no).into() + }, + _ => panic!("Invalid operand combination to mul instruction") + }; + + cb.write_bytes(&bytes); +} + +/// MVN - move a value in a register to another register, negating it +pub fn mvn(cb: &mut CodeBlock, rd: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::mvn(rd.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to mvn instruction") + }; + + cb.write_bytes(&bytes); +} + +/// NOP - no-operation, used for alignment purposes +pub fn nop(cb: &mut CodeBlock) { + let bytes: [u8; 4] = Nop::nop().into(); + + cb.write_bytes(&bytes); +} + +/// ORN - perform a bitwise OR of rn and NOT rm, put the result in rd, don't update flags +pub fn orn(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, "Expected registers to be the same size"); + + LogicalReg::orn(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to orn instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// ORR - perform a bitwise OR of rn and rm, put the result in rd, don't update flags +pub fn orr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + LogicalReg::orr(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + let bitmask_imm = if rd.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::orr(rd.reg_no, rn.reg_no, bitmask_imm, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to orr instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STLXR - store a value to memory, release exclusive access +pub fn stlxr(cb: &mut CodeBlock, rs: A64Opnd, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rs, rt, rn) { + (A64Opnd::Reg(rs), A64Opnd::Reg(rt), A64Opnd::Reg(rn)) => { + assert_eq!(rs.num_bits, 32, "rs must be a 32-bit register."); + assert_eq!(rn.num_bits, 64, "rn must be a 64-bit register."); + + LoadStoreExclusive::stlxr(rs.reg_no, rt.reg_no, rn.reg_no, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to stlxr instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (signed offset) - store a pair of registers to memory +pub fn stp(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (pre-index) - store a pair of registers to memory, update the base pointer before loading it +pub fn stp_pre(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp_pre(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STP (post-index) - store a pair of registers to memory, update the base pointer after loading it +pub fn stp_post(cb: &mut CodeBlock, rt1: A64Opnd, rt2: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt1, rt2, rn) { + (A64Opnd::Reg(rt1), A64Opnd::Reg(rt2), A64Opnd::Mem(rn)) => { + assert!(rt1.num_bits == rt2.num_bits, "Expected source registers to be the same size"); + assert!(imm_fits_bits(rn.disp.into(), 10), "The displacement must be 10 bits or less."); + assert_ne!(rt1.reg_no, rt2.reg_no, "Behavior is unpredictable with pairs of the same register"); + + RegisterPair::stp_post(rt1.reg_no, rt2.reg_no, rn.base_reg_no, rn.disp as i16, rt1.num_bits).into() + }, + _ => panic!("Invalid operand combination to stp instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STR (post-index) - store a register to memory, update the base pointer after loading it +pub fn str_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::str_post(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to str instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STR (pre-index) - store a register to memory, update the base pointer before loading it +pub fn str_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rt.num_bits == rn.num_bits, "All operands must be of the same size."); + assert!(mem_disp_fits_bits(rn.disp), "The displacement must be 9 bits or less."); + + LoadStore::str_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16, rt.num_bits).into() + }, + _ => panic!("Invalid operand combination to str instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// STRH - store a halfword into memory +pub fn strh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 12), "The displacement must be 12 bits or less."); + + HalfwordImm::strh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STRH (pre-index) - store a halfword into memory, update the base pointer before loading it +pub fn strh_pre(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::strh_pre(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STRH (post-index) - store a halfword into memory, update the base pointer after loading it +pub fn strh_post(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert_eq!(rt.num_bits, 32, "Expected to be loading a halfword"); + assert!(imm_fits_bits(rn.disp.into(), 9), "The displacement must be 9 bits or less."); + + HalfwordImm::strh_post(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to strh instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// STUR - store a value in a register at a memory address +pub fn stur(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 32 || rn.num_bits == 64); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::stur(rt.reg_no, rn.base_reg_no, rn.disp as i16, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to stur instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +/// STURH - store a value in a register at a memory address +pub fn sturh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 16); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + + LoadStore::sturh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturh instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +pub fn sturb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 8); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::sturb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturb instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +/// SUB - subtract rm from rn, put the result in rd, don't update flags +pub fn sub(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::sub(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::sub(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::add(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::sub(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to sub instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// SUBS - subtract rm from rn, put the result in rd, update flags +pub fn subs(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!( + rd.num_bits == rn.num_bits && rn.num_bits == rm.num_bits, + "All operands must be of the same size." + ); + + DataReg::subs(rd.reg_no, rn.reg_no, rm.reg_no, rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::UImm(uimm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + DataImm::subs(rd.reg_no, rn.reg_no, uimm12.try_into().unwrap(), rd.num_bits).into() + }, + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + + if imm12 < 0 { + DataImm::adds(rd.reg_no, rn.reg_no, (-imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } else { + DataImm::subs(rd.reg_no, rn.reg_no, (imm12 as u64).try_into().unwrap(), rd.num_bits).into() + } + }, + _ => panic!("Invalid operand combination to subs instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// SXTW - sign extend a 32-bit register into a 64-bit register +pub fn sxtw(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn)) => { + assert_eq!(rd.num_bits, 64, "rd must be 64-bits wide."); + assert_eq!(rn.num_bits, 32, "rn must be 32-bits wide."); + + SBFM::sxtw(rd.reg_no, rn.reg_no).into() + }, + _ => panic!("Invalid operand combination to sxtw instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// RET - unconditionally return to a location in a register, defaults to X30 +pub fn ret(cb: &mut CodeBlock, rn: A64Opnd) { + let bytes: [u8; 4] = match rn { + A64Opnd::None => Branch::ret(30).into(), + A64Opnd::Reg(reg) => Branch::ret(reg.reg_no).into(), + _ => panic!("Invalid operand to ret instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TBNZ - test bit and branch if not zero +pub fn tbnz(cb: &mut CodeBlock, rt: A64Opnd, bit_num: A64Opnd, offset: A64Opnd) { + let bytes: [u8; 4] = match (rt, bit_num, offset) { + (A64Opnd::Reg(rt), A64Opnd::UImm(bit_num), A64Opnd::Imm(offset)) => { + TestBit::tbnz(rt.reg_no, bit_num.try_into().unwrap(), offset.try_into().unwrap()).into() + }, + _ => panic!("Invalid operand combination to tbnz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TBZ - test bit and branch if zero +pub fn tbz(cb: &mut CodeBlock, rt: A64Opnd, bit_num: A64Opnd, offset: A64Opnd) { + let bytes: [u8; 4] = match (rt, bit_num, offset) { + (A64Opnd::Reg(rt), A64Opnd::UImm(bit_num), A64Opnd::Imm(offset)) => { + TestBit::tbz(rt.reg_no, bit_num.try_into().unwrap(), offset.try_into().unwrap()).into() + }, + _ => panic!("Invalid operand combination to tbz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// TST - test the bits of a register against a mask, then update flags +pub fn tst(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rn, rm) { + (A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rn.num_bits == rm.num_bits, "All operands must be of the same size."); + + LogicalReg::tst(rn.reg_no, rm.reg_no, rn.num_bits).into() + }, + (A64Opnd::Reg(rn), A64Opnd::UImm(imm)) => { + let bitmask_imm = if rn.num_bits == 32 { + BitmaskImmediate::new_32b_reg(imm.try_into().unwrap()) + } else { + imm.try_into() + }.unwrap(); + + LogicalImm::tst(rn.reg_no, bitmask_imm, rn.num_bits).into() + }, + _ => panic!("Invalid operand combination to tst instruction."), + }; + + cb.write_bytes(&bytes); +} + +/// CBZ - branch if a register is zero +pub fn cbz(cb: &mut CodeBlock, rt: A64Opnd, offset: InstructionOffset) { + assert!(imm_fits_bits(offset.into(), 19), "jump offset for cbz must fit in 19 bits"); + let bytes: [u8; 4] = if let A64Opnd::Reg(rt) = rt { + cbz_cbnz(rt.num_bits, false, offset, rt.reg_no) + } else { + panic!("Invalid operand combination to cbz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// CBNZ - branch if a register is non-zero +pub fn cbnz(cb: &mut CodeBlock, rt: A64Opnd, offset: InstructionOffset) { + assert!(imm_fits_bits(offset.into(), 19), "jump offset for cbz must fit in 19 bits"); + let bytes: [u8; 4] = if let A64Opnd::Reg(rt) = rt { + cbz_cbnz(rt.num_bits, true, offset, rt.reg_no) + } else { + panic!("Invalid operand combination to cbnz instruction.") + }; + + cb.write_bytes(&bytes); +} + +/// Encode Compare and Branch on Zero (CBZ) with `op=0` or Compare and Branch on Nonzero (CBNZ) +/// with `op=1`. +/// +/// <https://developer.arm.com/documentation/ddi0602/2024-03/Base-Instructions/CBZ--Compare-and-Branch-on-Zero-> +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | sf 0 1 1 0 1 0 op | +/// | imm19........................................................... Rt.............. | +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +fn cbz_cbnz(num_bits: u8, op: bool, offset: InstructionOffset, rt: u8) -> [u8; 4] { + ((Sf::from(num_bits) as u32) << 31 | + 0b11010 << 25 | + u32::from(op) << 24 | + truncate_imm::<_, 19>(offset) << 5 | + rt as u32).to_le_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + + fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb + } + + #[test] + fn test_imm_fits_bits() { + assert!(imm_fits_bits(i8::MAX.into(), 8)); + assert!(imm_fits_bits(i8::MIN.into(), 8)); + + assert!(imm_fits_bits(i16::MAX.into(), 16)); + assert!(imm_fits_bits(i16::MIN.into(), 16)); + + assert!(imm_fits_bits(i32::MAX.into(), 32)); + assert!(imm_fits_bits(i32::MIN.into(), 32)); + + assert!(imm_fits_bits(i64::MAX, 64)); + assert!(imm_fits_bits(i64::MIN, 64)); + } + + #[test] + fn test_uimm_fits_bits() { + assert!(uimm_fits_bits(u8::MAX.into(), 8)); + assert!(uimm_fits_bits(u16::MAX.into(), 16)); + assert!(uimm_fits_bits(u32::MAX.into(), 32)); + assert!(uimm_fits_bits(u64::MAX, 64)); + } + + #[test] + fn test_add_reg() { + let cb = compile(|cb| add(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028b"); + } + + #[test] + fn test_add_uimm() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_add_imm_positive() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_add_imm_negative() { + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_adds_reg() { + let cb = compile(|cb| adds(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ab"); + } + + #[test] + fn test_adds_uimm() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_adds_imm_positive() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_adds_imm_negative() { + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_adr() { + let cb = compile(|cb| adr(cb, X10, A64Opnd::new_imm(20))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adr x10, #0x14"); + assert_snapshot!(cb.hexdump(), @"aa000010"); + } + + #[test] + fn test_adrp() { + let cb = compile(|cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adrp x10, #0x8000"); + assert_snapshot!(cb.hexdump(), @"4a000090"); + } + + #[test] + fn test_and_register() { + let cb = compile(|cb| and(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028a"); + } + + #[test] + fn test_and_immediate() { + let cb = compile(|cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"20084092"); + } + + #[test] + fn test_and_32b_immediate() { + let cb = compile(|cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and w0, w2, #0xfffff"); + assert_snapshot!(cb.hexdump(), @"404c0012"); + } + + #[test] + fn test_ands_register() { + let cb = compile(|cb| ands(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ea"); + } + + #[test] + fn test_ands_immediate() { + let cb = compile(|cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"200840f2"); + } + + #[test] + fn test_asr() { + let cb = compile(|cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: asr x20, x21, #0xa"); + assert_snapshot!(cb.hexdump(), @"b4fe4a93"); + } + + #[test] + fn test_bcond() { + let offset = InstructionOffset::from_insns(0x100); + let cb = compile(|cb| bcond(cb, Condition::NE, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b.ne #0x400"); + assert_snapshot!(cb.hexdump(), @"01200054"); + } + + #[test] + fn test_b() { + let offset = InstructionOffset::from_insns((1 << 25) - 1); + let cb = compile(|cb| b(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b #0x7fffffc"); + assert_snapshot!(cb.hexdump(), @"ffffff15"); + } + + #[test] + #[should_panic] + fn test_b_too_big() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(1 << 25); + compile(|cb| b(cb, offset)); + } + + #[test] + #[should_panic] + fn test_b_too_small() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(-(1 << 25) - 1); + compile(|cb| b(cb, offset)); + } + + #[test] + fn test_bl() { + let offset = InstructionOffset::from_insns(-(1 << 25)); + let cb = compile(|cb| bl(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: bl #0xfffffffff8000000"); + assert_snapshot!(cb.hexdump(), @"00000096"); + } + + #[test] + #[should_panic] + fn test_bl_too_big() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(1 << 25); + compile(|cb| bl(cb, offset)); + } + + #[test] + #[should_panic] + fn test_bl_too_small() { + // There are 26 bits available + let offset = InstructionOffset::from_insns(-(1 << 25) - 1); + compile(|cb| bl(cb, offset)); + } + + #[test] + fn test_blr() { + let cb = compile(|cb| blr(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: blr x20"); + assert_snapshot!(cb.hexdump(), @"80023fd6"); + } + + #[test] + fn test_br() { + let cb = compile(|cb| br(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: br x20"); + assert_snapshot!(cb.hexdump(), @"80021fd6"); + } + + #[test] + fn test_cbz() { + let offset = InstructionOffset::from_insns(-1); + let cb = compile(|cb| { + cbz(cb, X0, offset); + cbz(cb, W0, offset); + }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: cbz x0, #0xfffffffffffffffc + 0x4: cbz w0, #0 + "); + assert_snapshot!(cb.hexdump(), @"e0ffffb4e0ffff34"); + } + + #[test] + fn test_cbnz() { + let offset = InstructionOffset::from_insns(2); + let cb = compile(|cb| { + cbnz(cb, X20, offset); + cbnz(cb, W20, offset); + }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: cbnz x20, #8 + 0x4: cbnz w20, #0xc + "); + assert_snapshot!(cb.hexdump(), @"540000b554000035"); + } + + #[test] + fn test_brk_none() { + let cb = compile(|cb| brk(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xf000"); + assert_snapshot!(cb.hexdump(), @"00003ed4"); + } + + #[test] + fn test_brk_uimm() { + let cb = compile(|cb| brk(cb, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xe"); + assert_snapshot!(cb.hexdump(), @"c00120d4"); + } + + #[test] + fn test_cmp_register() { + let cb = compile(|cb| cmp(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, x11"); + assert_snapshot!(cb.hexdump(), @"5f010beb"); + } + + #[test] + fn test_cmp_immediate() { + let cb = compile(|cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, #0xe"); + assert_snapshot!(cb.hexdump(), @"5f3900f1"); + } + + #[test] + fn test_csel() { + let cb = compile(|cb| csel(cb, X10, X11, X12, Condition::EQ)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: csel x10, x11, x12, eq"); + assert_snapshot!(cb.hexdump(), @"6a018c9a"); + } + + #[test] + fn test_eor_register() { + let cb = compile(|cb| eor(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010cca"); + } + + #[test] + fn test_eor_immediate() { + let cb = compile(|cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940d2"); + } + + #[test] + fn test_eor_32b_immediate() { + let cb = compile(|cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor w9, w1, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"29040152"); + } + + #[test] + fn test_ldaddal() { + let cb = compile(|cb| ldaddal(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaddal x10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8b01eaf8"); + } + + #[test] + fn test_ldaxr() { + let cb = compile(|cb| ldaxr(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaxr x10, [x11]"); + assert_snapshot!(cb.hexdump(), @"6afd5fc8"); + } + + #[test] + fn test_ldp() { + let cb = compile(|cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d4da9"); + } + + #[test] + fn test_ldp_pre() { + let cb = compile(|cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2dcda9"); + } + + #[test] + fn test_ldp_post() { + let cb = compile(|cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2dcda8"); + } + + #[test] + fn test_ldr() { + let cb = compile(|cb| ldr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, x12]"); + assert_snapshot!(cb.hexdump(), @"6a696cf8"); + } + + #[test] + fn test_ldr_literal() { + let cb = compile(|cb| ldr_literal(cb, X0, 10.into())); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x0, #0x28"); + assert_snapshot!(cb.hexdump(), @"40010058"); + } + + #[test] + fn test_ldr_post() { + let cb = compile(|cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11], #0x10"); + assert_snapshot!(cb.hexdump(), @"6a0541f8"); + } + + #[test] + fn test_ldr_pre() { + let cb = compile(|cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, #0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d41f8"); + } + + #[test] + fn test_ldrh() { + let cb = compile(|cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a194079"); + } + + #[test] + fn test_ldrh_pre() { + let cb = compile(|cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd4078"); + } + + #[test] + fn test_ldrh_post() { + let cb = compile(|cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac54078"); + } + + #[test] + fn test_ldurh_memory() { + let cb = compile(|cb| { + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0)); + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123)); + }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldurh w10, [x1] + 0x4: ldurh w10, [x1, #0x7b] + "); + assert_snapshot!(cb.hexdump(), @"2a0040782ab04778"); + } + + #[test] + fn test_ldur_memory() { + let cb = compile(|cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"20b047f8"); + } + + #[test] + fn test_ldur_register() { + let cb = compile(|cb| ldur(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1]"); + assert_snapshot!(cb.hexdump(), @"200040f8"); + } + + #[test] + fn test_ldursw() { + let cb = compile(|cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldursw x10, [x11, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"6ab187b8"); + } + + #[test] + fn test_lsl() { + let cb = compile(|cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsl x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6ac572d3"); + } + + #[test] + fn test_lsr() { + let cb = compile(|cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsr x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6afd4ed3"); + } + + #[test] + fn test_mov_registers() { + let cb = compile(|cb| mov(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea030baa"); + } + + #[test] + fn test_mov_immediate() { + let cb = compile(|cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, xzr, #0x5555555555555555"); + assert_snapshot!(cb.hexdump(), @"eaf300b2"); + } + + #[test] + fn test_mov_32b_immediate() { + let cb = compile(|cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov w10, #-0x7fffffff"); + assert_snapshot!(cb.hexdump(), @"ea070132"); + } + #[test] + fn test_mov_into_sp() { + let cb = compile(|cb| mov(cb, X31, X0)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov sp, x0"); + assert_snapshot!(cb.hexdump(), @"1f000091"); + } + + #[test] + fn test_mov_from_sp() { + let cb = compile(|cb| mov(cb, X0, X31)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, sp"); + assert_snapshot!(cb.hexdump(), @"e0030091"); + } + + #[test] + fn test_movk() { + let cb = compile(|cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movk x0, #0x7b, lsl #16"); + assert_snapshot!(cb.hexdump(), @"600fa0f2"); + } + + #[test] + fn test_movn() { + let cb = compile(|cb| movn(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #-0x7b0001"); + assert_snapshot!(cb.hexdump(), @"600fa092"); + } + + #[test] + fn test_movz() { + let cb = compile(|cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #0x7b0000"); + assert_snapshot!(cb.hexdump(), @"600fa0d2"); + } + + #[test] + fn test_mrs() { + let cb = compile(|cb| mrs(cb, X10, SystemRegister::NZCV)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mrs x10, nzcv"); + assert_snapshot!(cb.hexdump(), @"0a423bd5"); + } + + #[test] + fn test_msr() { + let cb = compile(|cb| msr(cb, SystemRegister::NZCV, X10)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: msr nzcv, x10"); + assert_snapshot!(cb.hexdump(), @"0a421bd5"); + } + + #[test] + fn test_mul() { + let cb = compile(|cb| mul(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mul x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a7d0c9b"); + } + + #[test] + fn test_mvn() { + let cb = compile(|cb| mvn(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mvn x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea032baa"); + } + + #[test] + fn test_nop() { + let cb = compile(nop); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: nop"); + assert_snapshot!(cb.hexdump(), @"1f2003d5"); + } + + #[test] + fn test_orn() { + let cb = compile(|cb| orn(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orn x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a012caa"); + } + + #[test] + fn test_orr_register() { + let cb = compile(|cb| orr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010caa"); + } + + #[test] + fn test_orr_immediate() { + let cb = compile(|cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940b2"); + } + + #[test] + fn test_orr_32b_immediate() { + let cb = compile(|cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr w10, w11, #1"); + assert_snapshot!(cb.hexdump(), @"6a010032"); + } + + #[test] + fn test_ret_none() { + let cb = compile(|cb| ret(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c0035fd6"); + } + + #[test] + fn test_ret_register() { + let cb = compile(|cb| ret(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret x20"); + assert_snapshot!(cb.hexdump(), @"80025fd6"); + } + + #[test] + fn test_stlxr() { + let cb = compile(|cb| stlxr(cb, W10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stlxr w10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8bfd0ac8"); + } + + #[test] + fn test_stp() { + let cb = compile(|cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d0da9"); + } + + #[test] + fn test_stp_pre() { + let cb = compile(|cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2d8da9"); + } + + #[test] + fn test_stp_post() { + let cb = compile(|cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2d8da8"); + } + + #[test] + fn test_str_post() { + let cb = compile(|cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11], #0xfffffffffffffff0"); + assert_snapshot!(cb.hexdump(), @"6a051ff8"); + } + + #[test] + fn test_str_pre() { + let cb = compile(|cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11, #-0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d1ff8"); + } + + #[test] + fn test_strh() { + let cb = compile(|cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a190079"); + } + + #[test] + fn test_strh_pre() { + let cb = compile(|cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd0078"); + } + + #[test] + fn test_strh_post() { + let cb = compile(|cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac50078"); + } + + #[test] + fn test_stur_64_bits() { + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur x10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108f8"); + } + + #[test] + fn test_stur_32_bits() { + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur w10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108b8"); + } + + #[test] + fn test_sub_reg() { + let cb = compile(|cb| sub(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002cb"); + } + + #[test] + fn test_sub_uimm() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_sub_imm_positive() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); + } + + #[test] + fn test_sub_imm_negative() { + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); + } + + #[test] + fn test_subs_reg() { + let cb = compile(|cb| subs(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002eb"); + } + + #[test] + fn test_subs_imm_positive() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_subs_imm_negative() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); + } + + #[test] + fn test_subs_uimm() { + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); + } + + #[test] + fn test_sxtw() { + let cb = compile(|cb| sxtw(cb, X10, W11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sxtw x10, w11"); + assert_snapshot!(cb.hexdump(), @"6a7d4093"); + } + + #[test] + fn test_tbnz() { + let cb = compile(|cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbnz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005037"); + } + + #[test] + fn test_tbz() { + let cb = compile(|cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005036"); + } + + #[test] + fn test_tst_register() { + let cb = compile(|cb| tst(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); + } + + #[test] + fn test_tst_immediate() { + let cb = compile(|cb| tst(cb, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x1, #7"); + assert_snapshot!(cb.hexdump(), @"3f0840f2"); + } + + #[test] + fn test_tst_32b_immediate() { + let cb = compile(|cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0xffff"); + assert_snapshot!(cb.hexdump(), @"1f3c0072"); + } + + #[test] + fn test_add_extend_various_regs() { + let mut cb = CodeBlock::new_dummy(); + + add_extended(&mut cb, X10, X11, X9); + add_extended(&mut cb, X30, X30, X30); + add_extended(&mut cb, X31, X31, X31); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x10, x11, x9, uxtx + 0x4: add x30, x30, x30, uxtx + 0x8: add sp, sp, xzr + "); + assert_snapshot!(cb.hexdump(), @"6a61298bde633e8bff633f8b"); + } +} diff --git a/zjit/src/asm/arm64/opnd.rs b/zjit/src/asm/arm64/opnd.rs new file mode 100644 index 0000000000..667533ab93 --- /dev/null +++ b/zjit/src/asm/arm64/opnd.rs @@ -0,0 +1,270 @@ +use std::fmt; + +/// This operand represents a register. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct A64Reg +{ + // Size in bits + pub num_bits: u8, + + // Register index number + pub reg_no: u8, +} + +impl A64Reg { + pub fn with_num_bits(&self, num_bits: u8) -> Self { + assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); + Self { num_bits, reg_no: self.reg_no } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct A64Mem +{ + // Size in bits + pub num_bits: u8, + + /// Base register number + pub base_reg_no: u8, + + /// Constant displacement from the base, not scaled + pub disp: i32, +} + +impl A64Mem { + pub fn new(num_bits: u8, reg: A64Opnd, disp: i32) -> Self { + match reg { + A64Opnd::Reg(reg) => { + Self { num_bits, base_reg_no: reg.reg_no, disp } + }, + _ => panic!("Expected register operand") + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum A64Opnd +{ + // Dummy operand + None, + + // Immediate value + Imm(i64), + + // Unsigned immediate + UImm(u64), + + // Register + Reg(A64Reg), + + // Memory + Mem(A64Mem) +} + +impl A64Opnd { + /// Create a new immediate value operand. + pub fn new_imm(value: i64) -> Self { + A64Opnd::Imm(value) + } + + /// Create a new unsigned immediate value operand. + pub fn new_uimm(value: u64) -> Self { + A64Opnd::UImm(value) + } + + /// Creates a new memory operand. + pub fn new_mem(num_bits: u8, reg: A64Opnd, disp: i32) -> Self { + A64Opnd::Mem(A64Mem::new(num_bits, reg, disp)) + } + + /// Convenience function to check if this operand is a register. + pub fn is_reg(&self) -> bool { + matches!(self, A64Opnd::Reg(_)) + } + + /// Unwrap a register from an operand. + pub fn unwrap_reg(&self) -> A64Reg { + match self { + A64Opnd::Reg(reg) => *reg, + _ => panic!("Expected register operand") + } + } +} + +// argument registers +pub const X0_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 0 }; +pub const X1_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 1 }; +pub const X2_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 2 }; +pub const X3_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 3 }; +pub const X4_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 4 }; +pub const X5_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 5 }; + +// caller-save registers +pub const X9_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 9 }; +pub const X10_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 10 }; +pub const X11_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 11 }; +pub const X12_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 12 }; +pub const X13_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 13 }; +pub const X14_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 14 }; +pub const X15_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 15 }; +pub const X16_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 16 }; +pub const X17_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 17 }; + +// callee-save registers +pub const X19_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 19 }; +pub const X20_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 20 }; +pub const X21_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 21 }; +pub const X22_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 22 }; + +// frame pointer (base pointer) +pub const X29_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 29 }; + +// link register +pub const X30_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 30 }; + +// zero register +pub const XZR_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 31 }; + +// 64-bit registers +pub const X0: A64Opnd = A64Opnd::Reg(X0_REG); +pub const X1: A64Opnd = A64Opnd::Reg(X1_REG); +pub const X2: A64Opnd = A64Opnd::Reg(X2_REG); +pub const X3: A64Opnd = A64Opnd::Reg(X3_REG); +pub const X4: A64Opnd = A64Opnd::Reg(X4_REG); +pub const X5: A64Opnd = A64Opnd::Reg(X5_REG); +pub const X6: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 6 }); +pub const X7: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 7 }); +pub const X8: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 8 }); +pub const X9: A64Opnd = A64Opnd::Reg(X9_REG); +pub const X10: A64Opnd = A64Opnd::Reg(X10_REG); +pub const X11: A64Opnd = A64Opnd::Reg(X11_REG); +pub const X12: A64Opnd = A64Opnd::Reg(X12_REG); +pub const X13: A64Opnd = A64Opnd::Reg(X13_REG); +pub const X14: A64Opnd = A64Opnd::Reg(X14_REG); +pub const X15: A64Opnd = A64Opnd::Reg(X15_REG); +pub const X16: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 16 }); +pub const X17: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 17 }); +pub const X18: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 18 }); +pub const X19: A64Opnd = A64Opnd::Reg(X19_REG); +pub const X20: A64Opnd = A64Opnd::Reg(X20_REG); +pub const X21: A64Opnd = A64Opnd::Reg(X21_REG); +pub const X22: A64Opnd = A64Opnd::Reg(X22_REG); +pub const X23: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 23 }); +pub const X24: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 24 }); +pub const X25: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 25 }); +pub const X26: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 26 }); +pub const X27: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 27 }); +pub const X28: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 28 }); +pub const X29: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 29 }); +pub const X30: A64Opnd = A64Opnd::Reg(X30_REG); +pub const X31: A64Opnd = A64Opnd::Reg(XZR_REG); + +// 32-bit registers +pub const W0: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 0 }); +pub const W1: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 1 }); +pub const W2: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 2 }); +pub const W3: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 3 }); +pub const W4: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 4 }); +pub const W5: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 5 }); +pub const W6: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 6 }); +pub const W7: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 7 }); +pub const W8: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 8 }); +pub const W9: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 9 }); +pub const W10: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 10 }); +pub const W11: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 11 }); +pub const W12: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 12 }); +pub const W13: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 13 }); +pub const W14: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 14 }); +pub const W15: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 15 }); +pub const W16: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 16 }); +pub const W17: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 17 }); +pub const W18: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 18 }); +pub const W19: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 19 }); +pub const W20: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 20 }); +pub const W21: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 21 }); +pub const W22: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 22 }); +pub const W23: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 23 }); +pub const W24: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 24 }); +pub const W25: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 25 }); +pub const W26: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 26 }); +pub const W27: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 27 }); +pub const W28: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 28 }); +pub const W29: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 29 }); +pub const W30: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 30 }); +pub const W31: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 31 }); + +// C argument registers +pub const C_ARG_REGS: [A64Opnd; 4] = [X0, X1, X2, X3]; +pub const C_ARG_REGREGS: [A64Reg; 4] = [X0_REG, X1_REG, X2_REG, X3_REG]; + +impl fmt::Display for A64Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match A64Opnd::Reg(*self) { + X0 => write!(f, "x0"), + X1 => write!(f, "x1"), + X2 => write!(f, "x2"), + X3 => write!(f, "x3"), + X4 => write!(f, "x4"), + X5 => write!(f, "x5"), + X6 => write!(f, "x6"), + X7 => write!(f, "x7"), + X8 => write!(f, "x8"), + X9 => write!(f, "x9"), + X10 => write!(f, "x10"), + X11 => write!(f, "x11"), + X12 => write!(f, "x12"), + X13 => write!(f, "x13"), + X14 => write!(f, "x14"), + X15 => write!(f, "x15"), + X16 => write!(f, "x16"), + X17 => write!(f, "x17"), + X18 => write!(f, "x18"), + X19 => write!(f, "x19"), + X20 => write!(f, "x20"), + X21 => write!(f, "x21"), + X22 => write!(f, "x22"), + X23 => write!(f, "x23"), + X24 => write!(f, "x24"), + X25 => write!(f, "x25"), + X26 => write!(f, "x26"), + X27 => write!(f, "x27"), + X28 => write!(f, "x28"), + X29 => write!(f, "x29"), + X30 => write!(f, "x30"), + X31 => write!(f, "x31"), + W0 => write!(f, "w0"), + W1 => write!(f, "w1"), + W2 => write!(f, "w2"), + W3 => write!(f, "w3"), + W4 => write!(f, "w4"), + W5 => write!(f, "w5"), + W6 => write!(f, "w6"), + W7 => write!(f, "w7"), + W8 => write!(f, "w8"), + W9 => write!(f, "w9"), + W10 => write!(f, "w10"), + W11 => write!(f, "w11"), + W12 => write!(f, "w12"), + W13 => write!(f, "w13"), + W14 => write!(f, "w14"), + W15 => write!(f, "w15"), + W16 => write!(f, "w16"), + W17 => write!(f, "w17"), + W18 => write!(f, "w18"), + W19 => write!(f, "w19"), + W20 => write!(f, "w20"), + W21 => write!(f, "w21"), + W22 => write!(f, "w22"), + W23 => write!(f, "w23"), + W24 => write!(f, "w24"), + W25 => write!(f, "w25"), + W26 => write!(f, "w26"), + W27 => write!(f, "w27"), + W28 => write!(f, "w28"), + W29 => write!(f, "w29"), + W30 => write!(f, "w30"), + W31 => write!(f, "w31"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs new file mode 100644 index 0000000000..1e8f3414ec --- /dev/null +++ b/zjit/src/asm/mod.rs @@ -0,0 +1,459 @@ +//! Model for creating generating textual assembler code. + +use std::collections::BTreeMap; +use std::fmt; +use std::ops::Range; +use std::rc::Rc; +use std::cell::RefCell; +use std::mem; +use crate::virtualmem::*; + +// Lots of manual vertical alignment in there that rustfmt doesn't handle well. +#[rustfmt::skip] +#[cfg(target_arch = "x86_64")] +pub mod x86_64; +#[cfg(target_arch = "aarch64")] +pub mod arm64; + +/// Index to a label created by cb.new_label() +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +pub struct Label(pub usize); + +/// The object that knows how to encode the branch instruction. +type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64) -> Result<(), ()>>; + +/// Reference to an ASM label +pub struct LabelRef { + // Position in the code block where the label reference exists + pos: usize, + + // Label which this refers to + label: Label, + + /// The number of bytes that this label reference takes up in the memory. + /// It's necessary to know this ahead of time so that when we come back to + /// patch it it takes the same amount of space. + num_bytes: usize, + + /// The object that knows how to encode the branch instruction. + encode: BranchEncoder, +} + +/// Block of memory into which instructions can be assembled +pub struct CodeBlock { + // Memory for storing the encoded instructions + mem_block: Rc<RefCell<VirtualMem>>, + + // Memory block size + mem_size: usize, + + // Current writing position + write_pos: usize, + + // Table of registered label addresses + label_addrs: Vec<usize>, + + // Table of registered label names + label_names: Vec<String>, + + // References to labels + label_refs: Vec<LabelRef>, + + // A switch for keeping comments. They take up memory. + keep_comments: bool, + + // Comments for assembly instructions, if that feature is enabled + asm_comments: BTreeMap<usize, Vec<String>>, + + // Set if the CodeBlock is unable to output some instructions, + // for example, when there is not enough space or when a jump + // target is too far away. + dropped_bytes: bool, +} + +impl CodeBlock { + /// Make a new CodeBlock + pub fn new(mem_block: Rc<RefCell<VirtualMem>>, keep_comments: bool) -> Self { + let mem_size = mem_block.borrow().virtual_region_size(); + Self { + mem_block, + mem_size, + write_pos: 0, + label_addrs: Vec::new(), + label_names: Vec::new(), + label_refs: Vec::new(), + keep_comments, + asm_comments: BTreeMap::new(), + dropped_bytes: false, + } + } + + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mem_block.borrow().mapped_region_size() + } + + /// Add an assembly comment if the feature is on. + pub fn add_comment(&mut self, comment: &str) { + if !self.keep_comments { + return; + } + + let cur_ptr = self.get_write_ptr().raw_addr(self); + + // If there's no current list of comments for this line number, add one. + let this_line_comments = self.asm_comments.entry(cur_ptr).or_default(); + + // Unless this comment is the same as the last one at this same line, add it. + if this_line_comments.last().map(String::as_str) != Some(comment) { + this_line_comments.push(comment.to_string()); + } + } + + pub fn comments_at(&self, pos: usize) -> Option<&Vec<String>> { + self.asm_comments.get(&pos) + } + + pub fn get_write_pos(&self) -> usize { + self.write_pos + } + + pub fn write_mem(&self, write_ptr: CodePtr, byte: u8) -> Result<(), WriteError> { + self.mem_block.borrow_mut().write_byte(write_ptr, byte) + } + + /// Get a (possibly dangling) direct pointer to the current write position + pub fn get_write_ptr(&self) -> CodePtr { + self.get_ptr(self.write_pos) + } + + /// Set the current write position from a pointer + pub fn set_write_ptr(&mut self, code_ptr: CodePtr) { + let pos = code_ptr.as_offset() - self.mem_block.borrow().start_ptr().as_offset(); + self.write_pos = pos.try_into().unwrap(); + } + + /// Invoke a callback with write_ptr temporarily adjusted to a given address + pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) -> Range<CodePtr> { + // Temporarily update the write_pos. Ignore the dropped_bytes flag at the old address. + let old_write_pos = self.write_pos; + let old_dropped_bytes = self.dropped_bytes; + self.set_write_ptr(code_ptr); + self.dropped_bytes = false; + + // Invoke the callback + callback(self); + + // Build a code range modified by the callback + let ret = code_ptr..self.get_write_ptr(); + + // Restore the original write_pos and dropped_bytes flag. + self.dropped_bytes = old_dropped_bytes; + self.write_pos = old_write_pos; + ret + } + + /// Get a (possibly dangling) direct pointer into the executable memory block + pub fn get_ptr(&self, offset: usize) -> CodePtr { + self.mem_block.borrow().start_ptr().add_bytes(offset) + } + + /// Write a single byte at the current position. + pub fn write_byte(&mut self, byte: u8) { + let write_ptr = self.get_write_ptr(); + // TODO: check has_capacity() + if self.mem_block.borrow_mut().write_byte(write_ptr, byte).is_ok() { + self.write_pos += 1; + } else { + self.dropped_bytes = true; + } + } + + /// Write multiple bytes starting from the current position. + pub fn write_bytes(&mut self, bytes: &[u8]) { + for byte in bytes { + self.write_byte(*byte); + } + } + + /// Write an integer over the given number of bits at the current position. + pub fn write_int(&mut self, val: u64, num_bits: u32) { + assert!(num_bits > 0); + assert!(num_bits % 8 == 0); + + // Switch on the number of bits + match num_bits { + 8 => self.write_byte(val as u8), + 16 => self.write_bytes(&[(val & 0xff) as u8, ((val >> 8) & 0xff) as u8]), + 32 => self.write_bytes(&[ + (val & 0xff) as u8, + ((val >> 8) & 0xff) as u8, + ((val >> 16) & 0xff) as u8, + ((val >> 24) & 0xff) as u8, + ]), + _ => { + let mut cur = val; + + // Write out the bytes + for _byte in 0..(num_bits / 8) { + self.write_byte((cur & 0xff) as u8); + cur >>= 8; + } + } + } + } + + /// Check if bytes have been dropped (unwritten because of insufficient space) + pub fn has_dropped_bytes(&self) -> bool { + self.dropped_bytes + } + + /// Set dropped_bytes to false if the current zjit_alloc_bytes() + code_region_size + /// + page_size is below --zjit-mem-size. + pub fn update_dropped_bytes(&mut self) { + if self.mem_block.borrow().can_allocate() { + self.dropped_bytes = false; + } + } + + /// Allocate a new label with a given name + pub fn new_label(&mut self, name: String) -> Label { + assert!(!name.contains(' '), "use underscores in label names, not spaces"); + + // This label doesn't have an address yet + self.label_addrs.push(0); + self.label_names.push(name); + + Label(self.label_addrs.len() - 1) + } + + /// Write a label at the current address + pub fn write_label(&mut self, label: Label) { + self.label_addrs[label.0] = self.write_pos; + } + + // Add a label reference at the current write position + pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) -> Result<(), ()> + 'static) { + assert!(label.0 < self.label_addrs.len()); + + // Keep track of the reference + self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode: Box::new(encode) }); + + // Move past however many bytes the instruction takes up + if self.write_pos + num_bytes < self.mem_size { + self.write_pos += num_bytes; + } else { + self.dropped_bytes = true; // retry emitting the Insn after next_page + } + } + + // Link internal label references + pub fn link_labels(&mut self) -> Result<(), ()> { + let orig_pos = self.write_pos; + let mut link_result = Ok(()); + + // For each label reference + for label_ref in mem::take(&mut self.label_refs) { + let ref_pos = label_ref.pos; + let label_idx = label_ref.label.0; + assert!(ref_pos < self.mem_size); + + let label_addr = self.label_addrs[label_idx]; + assert!(label_addr < self.mem_size); + + self.write_pos = ref_pos; + let encode_result = (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + link_result = link_result.and(encode_result); + + // Verify number of bytes written when the callback returns Ok + if encode_result.is_ok() { + assert_eq!(self.write_pos, ref_pos + label_ref.num_bytes, "label_ref \ + callback didn't write number of bytes it claimed to write upfront"); + } + } + + self.write_pos = orig_pos; + + // Clear the label positions and references + self.label_addrs.clear(); + self.label_names.clear(); + assert!(self.label_refs.is_empty()); + + link_result + } + + /// Convert a Label to CodePtr + pub fn resolve_label(&self, label: Label) -> CodePtr { + self.get_ptr(self.label_addrs[label.0]) + } + + pub fn clear_labels(&mut self) { + self.label_addrs.clear(); + self.label_names.clear(); + self.label_refs.clear(); + } + + /// Make all the code in the region executable. Call this at the end of a write session. + pub fn mark_all_executable(&mut self) { + self.mem_block.borrow_mut().mark_all_executable(); + } + + /// Call a func with the disasm of generated code for testing + #[allow(unused_variables)] + #[cfg(all(test, feature = "disasm"))] + pub fn disasm(&self) -> String { + let start_addr = self.get_ptr(0).raw_addr(self); + let end_addr = self.get_write_ptr().raw_addr(self); + crate::disasm::disasm_addr_range(self, start_addr, end_addr) + } + + /// Return the hex dump of generated code for testing + #[cfg(test)] + pub fn hexdump(&self) -> String { + format!("{:x}", self) + } +} + +/// Run assert_snapshot! only if cfg!(feature = "disasm"). +/// $actual can be not only `cb.disasm()` but also `disasms!(cb1, cb2, ...)`. +#[cfg(test)] +#[macro_export] +macro_rules! assert_disasm_snapshot { + ($actual: expr, @$($tt: tt)*) => {{ + #[cfg(feature = "disasm")] + assert_snapshot!($actual, @$($tt)*) + }}; +} + +/// Combine multiple cb.disasm() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms { + ($( $cb:expr ),+ $(,)?) => {{ + crate::disasms_with!("", $( $cb ),+) + }}; +} + +/// Basically `disasms!` but allows a non-"" delimiter, such as "\n" +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms_with { + ($join:expr, $( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.disasm() ),+].join($join) + }}; +} + +/// Combine multiple cb.hexdump() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(test)] +#[macro_export] +macro_rules! hexdumps { + ($( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.hexdump() ),+].join("\n") + }}; +} + +/// Produce hex string output from the bytes in a code block +impl fmt::LowerHex for CodeBlock { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + for pos in 0..self.write_pos { + let mem_block = &*self.mem_block.borrow(); + let byte = unsafe { mem_block.start_ptr().raw_ptr(mem_block).add(pos).read() }; + fmtr.write_fmt(format_args!("{byte:02x}"))?; + } + Ok(()) + } +} + +#[cfg(test)] +impl CodeBlock { + /// Stubbed CodeBlock for testing. Can't execute generated code. + pub fn new_dummy() -> Self { + const DEFAULT_MEM_SIZE: usize = 1024 * 1024; + CodeBlock::new_dummy_sized(DEFAULT_MEM_SIZE) + } + + pub fn new_dummy_sized(mem_size: usize) -> Self { + use crate::virtualmem::*; + let virt_mem = VirtualMem::alloc(mem_size, None); + Self::new(Rc::new(RefCell::new(virt_mem)), false) + } +} + +impl crate::virtualmem::CodePtrBase for CodeBlock { + fn base_ptr(&self) -> std::ptr::NonNull<u8> { + self.mem_block.borrow().base_ptr() + } +} + +/// Compute the number of bits needed to encode a signed value +pub fn imm_num_bits(imm: i64) -> u8 +{ + // Compute the smallest size this immediate fits in + if imm >= i8::MIN.into() && imm <= i8::MAX.into() { + return 8; + } + if imm >= i16::MIN.into() && imm <= i16::MAX.into() { + return 16; + } + if imm >= i32::MIN.into() && imm <= i32::MAX.into() { + return 32; + } + + 64 +} + +/// Compute the number of bits needed to encode an unsigned value +pub fn uimm_num_bits(uimm: u64) -> u8 +{ + // Compute the smallest size this immediate fits in + if uimm <= u8::MAX.into() { + return 8; + } + else if uimm <= u16::MAX.into() { + return 16; + } + else if uimm <= u32::MAX.into() { + return 32; + } + + 64 +} + +#[cfg(test)] +mod tests +{ + use super::*; + + #[test] + fn test_imm_num_bits() + { + assert_eq!(imm_num_bits(i8::MIN.into()), 8); + assert_eq!(imm_num_bits(i8::MAX.into()), 8); + + assert_eq!(imm_num_bits(i16::MIN.into()), 16); + assert_eq!(imm_num_bits(i16::MAX.into()), 16); + + assert_eq!(imm_num_bits(i32::MIN.into()), 32); + assert_eq!(imm_num_bits(i32::MAX.into()), 32); + + assert_eq!(imm_num_bits(i64::MIN), 64); + assert_eq!(imm_num_bits(i64::MAX), 64); + } + + #[test] + fn test_uimm_num_bits() { + assert_eq!(uimm_num_bits(u8::MIN.into()), 8); + assert_eq!(uimm_num_bits(u8::MAX.into()), 8); + + assert_eq!(uimm_num_bits(((u8::MAX as u16) + 1).into()), 16); + assert_eq!(uimm_num_bits(u16::MAX.into()), 16); + + assert_eq!(uimm_num_bits(((u16::MAX as u32) + 1).into()), 32); + assert_eq!(uimm_num_bits(u32::MAX.into()), 32); + + assert_eq!(uimm_num_bits((u32::MAX as u64) + 1), 64); + assert_eq!(uimm_num_bits(u64::MAX), 64); + } +} diff --git a/zjit/src/asm/x86_64/mod.rs b/zjit/src/asm/x86_64/mod.rs new file mode 100644 index 0000000000..0eeaae59dd --- /dev/null +++ b/zjit/src/asm/x86_64/mod.rs @@ -0,0 +1,1449 @@ +#![allow(dead_code)] // For instructions we don't currently generate + +use crate::asm::*; + +// Import the assembler tests module +mod tests; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86Imm +{ + // Size in bits + pub num_bits: u8, + + // The value of the immediate + pub value: i64 +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86UImm +{ + // Size in bits + pub num_bits: u8, + + // The value of the immediate + pub value: u64 +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RegType +{ + GP, + //FP, + //XMM, + IP, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct X86Reg +{ + // Size in bits + pub num_bits: u8, + + // Register type + pub reg_type: RegType, + + // Register index number + pub reg_no: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct X86Mem +{ + // Size in bits + pub num_bits: u8, + + /// Base register number + pub base_reg_no: u8, + + /// Index register number + pub idx_reg_no: Option<u8>, + + /// SIB scale exponent value (power of two, two bits) + pub scale_exp: u8, + + /// Constant displacement from the base, not scaled + pub disp: i32, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum X86Opnd +{ + // Dummy operand + None, + + // Immediate value + Imm(X86Imm), + + // Unsigned immediate + UImm(X86UImm), + + // General-purpose register + Reg(X86Reg), + + // Memory location + Mem(X86Mem), + + // IP-relative memory location + IPRel(i32) +} + +impl X86Reg { + pub fn with_num_bits(&self, num_bits: u8) -> Self { + assert!( + num_bits == 8 || + num_bits == 16 || + num_bits == 32 || + num_bits == 64 + ); + Self { + num_bits, + reg_type: self.reg_type, + reg_no: self.reg_no + } + } +} + +impl X86Opnd { + fn rex_needed(&self) -> bool { + match self { + X86Opnd::None => false, + X86Opnd::Imm(_) => false, + X86Opnd::UImm(_) => false, + X86Opnd::Reg(reg) => reg.reg_no > 7 || reg.num_bits == 8 && reg.reg_no >= 4, + X86Opnd::Mem(mem) => mem.base_reg_no > 7 || (mem.idx_reg_no.unwrap_or(0) > 7), + X86Opnd::IPRel(_) => false + } + } + + // Check if an SIB byte is needed to encode this operand + fn sib_needed(&self) -> bool { + match self { + X86Opnd::Mem(mem) => { + mem.idx_reg_no.is_some() || + mem.base_reg_no == RSP_REG_NO || + mem.base_reg_no == R12_REG_NO + }, + _ => false + } + } + + fn disp_size(&self) -> u32 { + match self { + X86Opnd::IPRel(_) => 32, + X86Opnd::Mem(mem) => { + if mem.disp != 0 { + // Compute the required displacement size + let num_bits = imm_num_bits(mem.disp.into()); + if num_bits > 32 { + panic!("displacement does not fit in 32 bits"); + } + + // x86 can only encode 8-bit and 32-bit displacements + if num_bits == 16 { 32 } else { 8 } + } else if mem.base_reg_no == RBP_REG_NO || mem.base_reg_no == R13_REG_NO { + // If EBP or RBP or R13 is used as the base, displacement must be encoded + 8 + } else { + 0 + } + }, + _ => 0 + } + } + + pub fn num_bits(&self) -> u8 { + match self { + X86Opnd::Reg(reg) => reg.num_bits, + X86Opnd::Imm(imm) => imm.num_bits, + X86Opnd::UImm(uimm) => uimm.num_bits, + X86Opnd::Mem(mem) => mem.num_bits, + _ => unreachable!() + } + } + + pub fn is_some(&self) -> bool { + !matches!(self, X86Opnd::None) + } + +} + +// Instruction pointer +pub const RIP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 64, reg_type: RegType::IP, reg_no: 5 }); + +// 64-bit GP registers +const RAX_REG_NO: u8 = 0; +const RSP_REG_NO: u8 = 4; +const RBP_REG_NO: u8 = 5; +const R12_REG_NO: u8 = 12; +const R13_REG_NO: u8 = 13; + +pub const RAX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RAX_REG_NO }; +pub const RCX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 1 }; +pub const RDX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 2 }; +pub const RBX_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 3 }; +pub const RSP_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RSP_REG_NO }; +pub const RBP_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: RBP_REG_NO }; +pub const RSI_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 6 }; +pub const RDI_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 7 }; +pub const R8_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 8 }; +pub const R9_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 9 }; +pub const R10_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 10 }; +pub const R11_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 11 }; +pub const R12_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: R12_REG_NO }; +pub const R13_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: R13_REG_NO }; +pub const R14_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 14 }; +pub const R15_REG: X86Reg = X86Reg { num_bits: 64, reg_type: RegType::GP, reg_no: 15 }; + +pub const RAX: X86Opnd = X86Opnd::Reg(RAX_REG); +pub const RCX: X86Opnd = X86Opnd::Reg(RCX_REG); +pub const RDX: X86Opnd = X86Opnd::Reg(RDX_REG); +pub const RBX: X86Opnd = X86Opnd::Reg(RBX_REG); +pub const RSP: X86Opnd = X86Opnd::Reg(RSP_REG); +pub const RBP: X86Opnd = X86Opnd::Reg(RBP_REG); +pub const RSI: X86Opnd = X86Opnd::Reg(RSI_REG); +pub const RDI: X86Opnd = X86Opnd::Reg(RDI_REG); +pub const R8: X86Opnd = X86Opnd::Reg(R8_REG); +pub const R9: X86Opnd = X86Opnd::Reg(R9_REG); +pub const R10: X86Opnd = X86Opnd::Reg(R10_REG); +pub const R11: X86Opnd = X86Opnd::Reg(R11_REG); +pub const R12: X86Opnd = X86Opnd::Reg(R12_REG); +pub const R13: X86Opnd = X86Opnd::Reg(R13_REG); +pub const R14: X86Opnd = X86Opnd::Reg(R14_REG); +pub const R15: X86Opnd = X86Opnd::Reg(R15_REG); + +// 32-bit GP registers +pub const EAX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 0 }); +pub const ECX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 1 }); +pub const EDX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 2 }); +pub const EBX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 3 }); +pub const ESP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 4 }); +pub const EBP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 5 }); +pub const ESI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 6 }); +pub const EDI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 7 }); +pub const R8D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 8 }); +pub const R9D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 9 }); +pub const R10D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 10 }); +pub const R11D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 11 }); +pub const R12D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 12 }); +pub const R13D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 13 }); +pub const R14D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 14 }); +pub const R15D: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 32, reg_type: RegType::GP, reg_no: 15 }); + +// 16-bit GP registers +pub const AX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 0 }); +pub const CX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 1 }); +pub const DX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 2 }); +pub const BX: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 3 }); +//pub const SP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 4 }); +pub const BP: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 5 }); +pub const SI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 6 }); +pub const DI: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 7 }); +pub const R8W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 8 }); +pub const R9W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 9 }); +pub const R10W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 10 }); +pub const R11W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 11 }); +pub const R12W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 12 }); +pub const R13W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 13 }); +pub const R14W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 14 }); +pub const R15W: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 16, reg_type: RegType::GP, reg_no: 15 }); + +// 8-bit GP registers +pub const AL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 0 }); +pub const CL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 1 }); +pub const DL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 2 }); +pub const BL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 3 }); +pub const SPL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 4 }); +pub const BPL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 5 }); +pub const SIL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 6 }); +pub const DIL: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 7 }); +pub const R8B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 8 }); +pub const R9B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 9 }); +pub const R10B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 10 }); +pub const R11B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 11 }); +pub const R12B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 12 }); +pub const R13B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 13 }); +pub const R14B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 14 }); +pub const R15B: X86Opnd = X86Opnd::Reg(X86Reg { num_bits: 8, reg_type: RegType::GP, reg_no: 15 }); + +//=========================================================================== + +/// Shorthand for memory operand with base register and displacement +pub fn mem_opnd(num_bits: u8, base_reg: X86Opnd, disp: i32) -> X86Opnd +{ + let base_reg = match base_reg { + X86Opnd::Reg(reg) => reg, + _ => unreachable!() + }; + + if base_reg.reg_type == RegType::IP { + X86Opnd::IPRel(disp) + } else { + X86Opnd::Mem( + X86Mem { + num_bits, + base_reg_no: base_reg.reg_no, + idx_reg_no: None, + scale_exp: 0, + disp, + } + ) + } +} + +/// Memory operand with SIB (Scale Index Base) indexing +pub fn mem_opnd_sib(num_bits: u8, base_opnd: X86Opnd, index_opnd: X86Opnd, scale: i32, disp: i32) -> X86Opnd { + if let (X86Opnd::Reg(base_reg), X86Opnd::Reg(index_reg)) = (base_opnd, index_opnd) { + let scale_exp: u8 = match scale { + 8 => 3, + 4 => 2, + 2 => 1, + 1 => 0, + _ => unreachable!() + }; + + X86Opnd::Mem(X86Mem { + num_bits, + base_reg_no: base_reg.reg_no, + idx_reg_no: Some(index_reg.reg_no), + scale_exp, + disp + }) + } else { + unreachable!() + } +} + +pub fn imm_opnd(value: i64) -> X86Opnd +{ + X86Opnd::Imm(X86Imm { num_bits: imm_num_bits(value), value }) +} + +pub fn uimm_opnd(value: u64) -> X86Opnd +{ + X86Opnd::UImm(X86UImm { num_bits: uimm_num_bits(value), value }) +} + +pub fn const_ptr_opnd(ptr: *const u8) -> X86Opnd +{ + uimm_opnd(ptr as u64) +} + +/// Write the REX byte +fn write_rex(cb: &mut CodeBlock, w_flag: bool, reg_no: u8, idx_reg_no: u8, rm_reg_no: u8) { + // 0 1 0 0 w r x b + // w - 64-bit operand size flag + // r - MODRM.reg extension + // x - SIB.index extension + // b - MODRM.rm or SIB.base extension + let w: u8 = if w_flag { 1 } else { 0 }; + let r: u8 = if (reg_no & 8) > 0 { 1 } else { 0 }; + let x: u8 = if (idx_reg_no & 8) > 0 { 1 } else { 0 }; + let b: u8 = if (rm_reg_no & 8) > 0 { 1 } else { 0 }; + + // Encode and write the REX byte + cb.write_byte(0x40 + (w << 3) + (r << 2) + (x << 1) + (b)); +} + +/// Write an opcode byte with an embedded register operand +fn write_opcode(cb: &mut CodeBlock, opcode: u8, reg: X86Reg) { + let op_byte: u8 = opcode | (reg.reg_no & 7); + cb.write_byte(op_byte); +} + +/// Encode an RM instruction +fn write_rm(cb: &mut CodeBlock, sz_pref: bool, rex_w: bool, r_opnd: X86Opnd, rm_opnd: X86Opnd, op_ext: Option<u8>, bytes: &[u8]) { + let op_len = bytes.len(); + assert!(op_len > 0 && op_len <= 3); + assert!(matches!(r_opnd, X86Opnd::Reg(_) | X86Opnd::None), "Can only encode an RM instruction with a register or a none"); + + // Flag to indicate the REX prefix is needed + let need_rex = rex_w || r_opnd.rex_needed() || rm_opnd.rex_needed(); + + // Flag to indicate SIB byte is needed + let need_sib = r_opnd.sib_needed() || rm_opnd.sib_needed(); + + // Add the operand-size prefix, if needed + if sz_pref { + cb.write_byte(0x66); + } + + // Add the REX prefix, if needed + if need_rex { + // 0 1 0 0 w r x b + // w - 64-bit operand size flag + // r - MODRM.reg extension + // x - SIB.index extension + // b - MODRM.rm or SIB.base extension + + let w = if rex_w { 1 } else { 0 }; + let r = match r_opnd { + X86Opnd::None => 0, + X86Opnd::Reg(reg) => if (reg.reg_no & 8) > 0 { 1 } else { 0 }, + _ => unreachable!() + }; + + let x = match (need_sib, rm_opnd) { + (true, X86Opnd::Mem(mem)) => if (mem.idx_reg_no.unwrap_or(0) & 8) > 0 { 1 } else { 0 }, + _ => 0 + }; + + let b = match rm_opnd { + X86Opnd::Reg(reg) => if (reg.reg_no & 8) > 0 { 1 } else { 0 }, + X86Opnd::Mem(mem) => if (mem.base_reg_no & 8) > 0 { 1 } else { 0 }, + _ => 0 + }; + + // Encode and write the REX byte + let rex_byte: u8 = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b); + cb.write_byte(rex_byte); + } + + // Write the opcode bytes to the code block + for byte in bytes { + cb.write_byte(*byte) + } + + // MODRM.mod (2 bits) + // MODRM.reg (3 bits) + // MODRM.rm (3 bits) + + assert!( + !(op_ext.is_some() && r_opnd.is_some()), + "opcode extension and register operand present" + ); + + // Encode the mod field + let rm_mod = match rm_opnd { + X86Opnd::Reg(_) => 3, + X86Opnd::IPRel(_) => 0, + X86Opnd::Mem(_mem) => { + match rm_opnd.disp_size() { + 0 => 0, + 8 => 1, + 32 => 2, + _ => unreachable!() + } + }, + _ => unreachable!() + }; + + // Encode the reg field + let reg: u8; + if let Some(val) = op_ext { + reg = val; + } else { + reg = match r_opnd { + X86Opnd::Reg(reg) => reg.reg_no & 7, + _ => 0 + }; + } + + // Encode the rm field + let rm = match rm_opnd { + X86Opnd::Reg(reg) => reg.reg_no & 7, + X86Opnd::Mem(mem) => if need_sib { 4 } else { mem.base_reg_no & 7 }, + X86Opnd::IPRel(_) => 0b101, + _ => unreachable!() + }; + + // Encode and write the ModR/M byte + let rm_byte: u8 = (rm_mod << 6) + (reg << 3) + (rm); + cb.write_byte(rm_byte); + + // Add the SIB byte, if needed + if need_sib { + // SIB.scale (2 bits) + // SIB.index (3 bits) + // SIB.base (3 bits) + + match rm_opnd { + X86Opnd::Mem(mem) => { + // Encode the scale value + let scale = mem.scale_exp; + + // Encode the index value + let index = mem.idx_reg_no.map(|no| no & 7).unwrap_or(4); + + // Encode the base register + let base = mem.base_reg_no & 7; + + // Encode and write the SIB byte + let sib_byte: u8 = (scale << 6) + (index << 3) + (base); + cb.write_byte(sib_byte); + }, + _ => panic!("Expected mem operand") + } + } + + // Add the displacement + match rm_opnd { + X86Opnd::Mem(mem) => { + let disp_size = rm_opnd.disp_size(); + if disp_size > 0 { + cb.write_int(mem.disp as u64, disp_size); + } + }, + X86Opnd::IPRel(rel) => { + cb.write_int(rel as u64, 32); + }, + _ => () + }; +} + +// Encode a mul-like single-operand RM instruction +fn write_rm_unary(cb: &mut CodeBlock, op_mem_reg_8: u8, op_mem_reg_pref: u8, op_ext: Option<u8>, opnd: X86Opnd) { + assert!(matches!(opnd, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + let opnd_size = opnd.num_bits(); + assert!(opnd_size == 8 || opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd, op_ext, &[op_mem_reg_8]); + } else { + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd, op_ext, &[op_mem_reg_pref]); + } +} + +// Encode an add-like RM instruction with multiple possible encodings +fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_reg_mem8: u8, op_reg_mem_pref: u8, op_mem_imm8: u8, op_mem_imm_sml: u8, op_mem_imm_lrg: u8, op_ext_imm: Option<u8>, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_)), "unexpected opnd0: {opnd0:?}, {opnd1:?}"); + + // Check the size of opnd0 + let opnd_size = opnd0.num_bits(); + assert!(opnd_size == 8 || opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + // Check the size of opnd1 + match opnd1 { + X86Opnd::Reg(reg) => assert_eq!(reg.num_bits, opnd_size), + X86Opnd::Mem(mem) => assert_eq!(mem.num_bits, opnd_size), + X86Opnd::Imm(imm) => assert!(imm.num_bits <= opnd_size), + X86Opnd::UImm(uimm) => assert!(uimm.num_bits <= opnd_size), + _ => () + }; + + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + + match (opnd0, opnd1) { + // R/M + Reg + (X86Opnd::Mem(_), X86Opnd::Reg(_)) | (X86Opnd::Reg(_), X86Opnd::Reg(_)) => { + if opnd_size == 8 { + write_rm(cb, false, false, opnd1, opnd0, None, &[op_mem_reg8]); + } else { + write_rm(cb, sz_pref, rex_w, opnd1, opnd0, None, &[op_mem_reg_pref]); + } + }, + // Reg + R/M/IPRel + (X86Opnd::Reg(_), X86Opnd::Mem(_) | X86Opnd::IPRel(_)) => { + if opnd_size == 8 { + write_rm(cb, false, false, opnd0, opnd1, None, &[op_reg_mem8]); + } else { + write_rm(cb, sz_pref, rex_w, opnd0, opnd1, None, &[op_reg_mem_pref]); + } + }, + // R/M + Imm + (_, X86Opnd::Imm(imm)) => { + if imm.num_bits <= 8 { + // 8-bit immediate + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm8]); + } else { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_sml]); + } + + cb.write_int(imm.value as u64, 8); + } else if imm.num_bits <= 32 { + // 32-bit immediate + + assert!(imm.num_bits <= opnd_size); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_lrg]); + cb.write_int(imm.value as u64, if opnd_size > 32 { 32 } else { opnd_size.into() }); + } else { + panic!("immediate value too large"); + } + }, + // R/M + UImm + (_, X86Opnd::UImm(uimm)) => { + // If the size of left hand operand equals the number of bits + // required to represent the right hand immediate, then we + // don't care about sign extension when calculating the immediate + let num_bits = if opnd0.num_bits() == uimm_num_bits(uimm.value) { + uimm_num_bits(uimm.value) + } else { + imm_num_bits(uimm.value.try_into().unwrap()) + }; + + if num_bits <= 8 { + // 8-bit immediate + + if opnd_size == 8 { + write_rm(cb, false, false, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm8]); + } else { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_sml]); + } + + cb.write_int(uimm.value, 8); + } else if num_bits <= 32 { + // 32-bit immediate + + assert!(num_bits <= opnd_size); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_lrg]); + cb.write_int(uimm.value, if opnd_size > 32 { 32 } else { opnd_size.into() }); + } else { + panic!("immediate value too large (num_bits={}, num={uimm:?})", num_bits); + } + }, + _ => panic!("unknown encoding combo: {opnd0:?} {opnd1:?}") + }; +} + +// LOCK - lock prefix for atomic shared memory operations +pub fn write_lock_prefix(cb: &mut CodeBlock) { + cb.write_byte(0xf0); +} + +/// add - Integer addition +pub fn add(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x00, // opMemReg8 + 0x01, // opMemRegPref + 0x02, // opRegMem8 + 0x03, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x00), // opExtImm + opnd0, + opnd1 + ); +} + +/// and - Bitwise AND +pub fn and(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x20, // opMemReg8 + 0x21, // opMemRegPref + 0x22, // opRegMem8 + 0x23, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x04), // opExtImm + opnd0, + opnd1 + ); +} + +/// call - Call to a pointer with a 32-bit displacement offset +pub fn call_rel32(cb: &mut CodeBlock, rel32: i32) { + // Write the opcode + cb.write_byte(0xe8); + + // Write the relative 32-bit jump offset + cb.write_bytes(&rel32.to_le_bytes()); +} + +/// call - Call a pointer, encode with a 32-bit offset if possible +pub fn call_ptr(cb: &mut CodeBlock, scratch_opnd: X86Opnd, dst_ptr: *const u8) { + if let X86Opnd::Reg(_scratch_reg) = scratch_opnd { + // TODO: implement a counter + + // Pointer to the end of this call instruction + let end_ptr = cb.get_ptr(cb.write_pos + 5); + + // Compute the jump offset + let rel64: i64 = dst_ptr as i64 - end_ptr.raw_ptr(cb) as i64; + + // If the offset fits in 32-bit + if rel64 >= i32::MIN.into() && rel64 <= i32::MAX.into() { + call_rel32(cb, rel64.try_into().unwrap()); + return; + } + + // Move the pointer into the scratch register and call + mov(cb, scratch_opnd, const_ptr_opnd(dst_ptr)); + call(cb, scratch_opnd); + } else { + unreachable!(); + } +} + +/// call - Call to label with 32-bit offset +pub fn call_label(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 5, |cb, src_addr, dst_addr| { + cb.write_byte(0xE8); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// call - Indirect call with an R/M operand +pub fn call(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(2), &[0xff]); +} + +/// Encode a conditional move instruction +fn write_cmov(cb: &mut CodeBlock, opcode1: u8, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(reg) = dst { + match src { + X86Opnd::Reg(_) => (), + X86Opnd::Mem(_) => (), + _ => unreachable!() + }; + + assert!(reg.num_bits >= 16); + let sz_pref = reg.num_bits == 16; + let rex_w = reg.num_bits == 64; + + write_rm(cb, sz_pref, rex_w, dst, src, None, &[0x0f, opcode1]); + } else { + unreachable!() + } +} + +// cmovcc - Conditional move +pub fn cmova(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x47, dst, src); } +pub fn cmovae(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovb(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmovbe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x46, dst, src); } +pub fn cmovc(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmove(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x44, dst, src); } +pub fn cmovg(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4f, dst, src); } +pub fn cmovge(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4d, dst, src); } +pub fn cmovl(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4c, dst, src); } +pub fn cmovle(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4e, dst, src); } +pub fn cmovna(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x46, dst, src); } +pub fn cmovnae(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x42, dst, src); } +pub fn cmovnb(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovnbe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x47, dst, src); } +pub fn cmovnc(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x43, dst, src); } +pub fn cmovne(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x45, dst, src); } +pub fn cmovng(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4e, dst, src); } +pub fn cmovnge(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4c, dst, src); } +pub fn cmovnl(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4d, dst, src); } +pub fn cmovnle(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4f, dst, src); } +pub fn cmovno(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x41, dst, src); } +pub fn cmovnp(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4b, dst, src); } +pub fn cmovns(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x49, dst, src); } +pub fn cmovnz(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x45, dst, src); } +pub fn cmovo(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x40, dst, src); } +pub fn cmovp(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4a, dst, src); } +pub fn cmovpe(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4a, dst, src); } +pub fn cmovpo(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x4b, dst, src); } +pub fn cmovs(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x48, dst, src); } +pub fn cmovz(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { write_cmov(cb, 0x44, dst, src); } + +/// cmp - Compare and set flags +pub fn cmp(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x38, // opMemReg8 + 0x39, // opMemRegPref + 0x3A, // opRegMem8 + 0x3B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x07), // opExtImm + opnd0, + opnd1 + ); +} + +/// cdq - Convert doubleword to quadword +pub fn cdq(cb: &mut CodeBlock) { + cb.write_byte(0x99); +} + +/// cqo - Convert quadword to octaword +pub fn cqo(cb: &mut CodeBlock) { + cb.write_bytes(&[0x48, 0x99]); +} + +/// imul - signed integer multiply +pub fn imul(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(opnd0.num_bits() == 64); + assert!(opnd1.num_bits() == 64); + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + assert!(matches!(opnd1, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + match (opnd0, opnd1) { + (X86Opnd::Reg(_), X86Opnd::Reg(_) | X86Opnd::Mem(_)) => { + //REX.W + 0F AF /rIMUL r64, r/m64 + // Quadword register := Quadword register * r/m64. + write_rm(cb, false, true, opnd0, opnd1, None, &[0x0F, 0xAF]); + } + + _ => unreachable!() + } +} + +/// Interrupt 3 - trap to debugger +pub fn int3(cb: &mut CodeBlock) { + cb.write_byte(0xcc); +} + +// Encode a conditional relative jump to a label +// Note: this always encodes a 32-bit offset +fn write_jcc<const OP: u8>(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 6, |cb, src_addr, dst_addr| { + cb.write_byte(0x0F); + cb.write_byte(OP); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// jcc - relative jumps to a label +pub fn ja_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x87>(cb, label); } +pub fn jae_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jb_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn jbe_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x86>(cb, label); } +pub fn jc_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn je_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x84>(cb, label); } +pub fn jg_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8F>(cb, label); } +pub fn jge_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8D>(cb, label); } +pub fn jl_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8C>(cb, label); } +pub fn jle_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8E>(cb, label); } +pub fn jna_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x86>(cb, label); } +pub fn jnae_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x82>(cb, label); } +pub fn jnb_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jnbe_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x87>(cb, label); } +pub fn jnc_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x83>(cb, label); } +pub fn jne_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x85>(cb, label); } +pub fn jng_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8E>(cb, label); } +pub fn jnge_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x8C>(cb, label); } +pub fn jnl_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8D>(cb, label); } +pub fn jnle_label(cb: &mut CodeBlock, label: Label) { write_jcc::<0x8F>(cb, label); } +pub fn jno_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x81>(cb, label); } +pub fn jnp_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8b>(cb, label); } +pub fn jns_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x89>(cb, label); } +pub fn jnz_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x85>(cb, label); } +pub fn jo_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x80>(cb, label); } +pub fn jp_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8A>(cb, label); } +pub fn jpe_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8A>(cb, label); } +pub fn jpo_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x8B>(cb, label); } +pub fn js_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x88>(cb, label); } +pub fn jz_label (cb: &mut CodeBlock, label: Label) { write_jcc::<0x84>(cb, label); } + +pub fn jmp_label(cb: &mut CodeBlock, label: Label) { + cb.label_ref(label, 5, |cb, src_addr, dst_addr| { + cb.write_byte(0xE9); + cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) + }); +} + +/// Encode a relative jump to a pointer at a 32-bit offset (direct or conditional) +fn write_jcc_ptr(cb: &mut CodeBlock, op0: u8, op1: u8, dst_ptr: CodePtr) { + // Write the opcode + if op0 != 0xFF { + cb.write_byte(op0); + } + + cb.write_byte(op1); + + // Pointer to the end of this jump instruction + let end_ptr = cb.get_ptr(cb.write_pos + 4); + + // Compute the jump offset + let rel64 = dst_ptr.as_offset() - end_ptr.as_offset(); + + if rel64 >= i32::MIN.into() && rel64 <= i32::MAX.into() { + // Write the relative 32-bit jump offset + cb.write_int(rel64 as u64, 32); + } + else { + // Offset doesn't fit in 4 bytes. Report error. + //cb.dropped_bytes = true; + panic!("we should refactor to avoid dropped_bytes"); + } +} + +/// jcc - relative jumps to a pointer (32-bit offset) +pub fn ja_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x87, ptr); } +pub fn jae_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jb_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn jbe_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x86, ptr); } +pub fn jc_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn je_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x84, ptr); } +pub fn jg_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8F, ptr); } +pub fn jge_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8D, ptr); } +pub fn jl_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8C, ptr); } +pub fn jle_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8E, ptr); } +pub fn jna_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x86, ptr); } +pub fn jnae_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x82, ptr); } +pub fn jnb_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jnbe_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x87, ptr); } +pub fn jnc_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x83, ptr); } +pub fn jne_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x85, ptr); } +pub fn jng_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8E, ptr); } +pub fn jnge_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8C, ptr); } +pub fn jnl_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8D, ptr); } +pub fn jnle_ptr(cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8F, ptr); } +pub fn jno_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x81, ptr); } +pub fn jnp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8b, ptr); } +pub fn jns_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x89, ptr); } +pub fn jnz_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x85, ptr); } +pub fn jo_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x80, ptr); } +pub fn jp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8A, ptr); } +pub fn jpe_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8A, ptr); } +pub fn jpo_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x8B, ptr); } +pub fn js_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x88, ptr); } +pub fn jz_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0x0F, 0x84, ptr); } +pub fn jmp_ptr (cb: &mut CodeBlock, ptr: CodePtr) { write_jcc_ptr(cb, 0xFF, 0xE9, ptr); } + +/// jmp - Indirect jump near to an R/M operand. +pub fn jmp_rm(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(4), &[0xff]); +} + +// jmp - Jump with relative 32-bit offset +pub fn jmp32(cb: &mut CodeBlock, offset: i32) { + cb.write_byte(0xE9); + cb.write_int(offset as u64, 32); +} + +/// lea - Load Effective Address +pub fn lea(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(reg) = dst { + assert!(reg.num_bits == 64); + assert!(matches!(src, X86Opnd::Mem(_) | X86Opnd::IPRel(_))); + write_rm(cb, false, true, dst, src, None, &[0x8d]); + } else { + unreachable!(); + } +} + +/// mov - Data move operation +pub fn mov(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + match (dst, src) { + // R + Imm + (X86Opnd::Reg(reg), X86Opnd::Imm(imm)) => { + assert!(imm.num_bits <= reg.num_bits); + + // In case the source immediate could be zero extended to be 64 + // bit, we can use the 32-bit operands version of the instruction. + // For example, we can turn mov(rax, 0x34) into the equivalent + // mov(eax, 0x34). + if (reg.num_bits == 64) && (imm.value > 0) && (imm.num_bits <= 32) { + if dst.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0xB8, reg); + cb.write_int(imm.value as u64, 32); + } else { + if reg.num_bits == 16 { + cb.write_byte(0x66); + } + + if dst.rex_needed() || reg.num_bits == 64 { + write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); + } + + write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); + cb.write_int(imm.value as u64, reg.num_bits.into()); + } + }, + // R + UImm + (X86Opnd::Reg(reg), X86Opnd::UImm(uimm)) => { + assert!(uimm.num_bits <= reg.num_bits); + + // In case the source immediate could be zero extended to be 64 + // bit, we can use the 32-bit operands version of the instruction. + // For example, we can turn mov(rax, 0x34) into the equivalent + // mov(eax, 0x34). + if (reg.num_bits == 64) && (uimm.value <= u32::MAX.into()) { + if dst.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0xB8, reg); + cb.write_int(uimm.value, 32); + } else { + if reg.num_bits == 16 { + cb.write_byte(0x66); + } + + if dst.rex_needed() || reg.num_bits == 64 { + write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); + } + + write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); + cb.write_int(uimm.value, reg.num_bits.into()); + } + }, + // M + Imm + (X86Opnd::Mem(mem), X86Opnd::Imm(imm)) => { + assert!(imm.num_bits <= mem.num_bits); + + if mem.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, dst, None, &[0xc6]); + } else { + write_rm(cb, mem.num_bits == 16, mem.num_bits == 64, X86Opnd::None, dst, Some(0), &[0xc7]); + } + + let output_num_bits:u32 = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; + assert!( + mem.num_bits < 64 || imm_num_bits(imm.value) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); + cb.write_int(imm.value as u64, output_num_bits); + }, + // M + UImm + (X86Opnd::Mem(mem), X86Opnd::UImm(uimm)) => { + assert!(uimm.num_bits <= mem.num_bits); + + if mem.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, dst, None, &[0xc6]); + } + else { + write_rm(cb, mem.num_bits == 16, mem.num_bits == 64, X86Opnd::None, dst, Some(0), &[0xc7]); + } + + let output_num_bits = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; + assert!( + mem.num_bits < 64 || imm_num_bits(uimm.value as i64) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); + cb.write_int(uimm.value, output_num_bits); + }, + // * + Imm/UImm + (_, X86Opnd::Imm(_) | X86Opnd::UImm(_)) => unreachable!(), + // * + * + (_, _) => { + write_rm_multi( + cb, + 0x88, // opMemReg8 + 0x89, // opMemRegPref + 0x8A, // opRegMem8 + 0x8B, // opRegMemPref + 0xC6, // opMemImm8 + 0xFF, // opMemImmSml (not available) + 0xFF, // opMemImmLrg + None, // opExtImm + dst, + src + ); + } + }; +} + +/// A variant of mov used for always writing the value in 64 bits for GC offsets. +pub fn movabs(cb: &mut CodeBlock, dst: X86Opnd, value: u64) { + match dst { + X86Opnd::Reg(reg) => { + assert_eq!(reg.num_bits, 64); + write_rex(cb, true, 0, 0, reg.reg_no); + + write_opcode(cb, 0xb8, reg); + cb.write_int(value, 64); + }, + _ => unreachable!() + } +} + +/// movsx - Move with sign extension (signed integers) +pub fn movsx(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { + if let X86Opnd::Reg(_dst_reg) = dst { + assert!(matches!(src, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + let src_num_bits = src.num_bits(); + let dst_num_bits = dst.num_bits(); + assert!(src_num_bits < dst_num_bits); + + match src_num_bits { + 8 => write_rm(cb, dst_num_bits == 16, dst_num_bits == 64, dst, src, None, &[0x0f, 0xbe]), + 16 => write_rm(cb, dst_num_bits == 16, dst_num_bits == 64, dst, src, None, &[0x0f, 0xbf]), + 32 => write_rm(cb, false, true, dst, src, None, &[0x63]), + _ => unreachable!() + }; + } else { + unreachable!(); + } +} + +/// nop - Noop, one or multiple bytes long +pub fn nop(cb: &mut CodeBlock, length: u32) { + match length { + 0 => {}, + 1 => cb.write_byte(0x90), + 2 => cb.write_bytes(&[0x66, 0x90]), + 3 => cb.write_bytes(&[0x0f, 0x1f, 0x00]), + 4 => cb.write_bytes(&[0x0f, 0x1f, 0x40, 0x00]), + 5 => cb.write_bytes(&[0x0f, 0x1f, 0x44, 0x00, 0x00]), + 6 => cb.write_bytes(&[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00]), + 7 => cb.write_bytes(&[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00]), + 8 => cb.write_bytes(&[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00]), + 9 => cb.write_bytes(&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00]), + _ => { + let mut written: u32 = 0; + while written + 9 <= length { + nop(cb, 9); + written += 9; + } + nop(cb, length - written); + } + }; +} + +/// not - Bitwise NOT +pub fn not(cb: &mut CodeBlock, opnd: X86Opnd) { + write_rm_unary( + cb, + 0xf6, // opMemReg8 + 0xf7, // opMemRegPref + Some(0x02), // opExt + opnd + ); +} + +/// or - Bitwise OR +pub fn or(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x08, // opMemReg8 + 0x09, // opMemRegPref + 0x0A, // opRegMem8 + 0x0B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x01), // opExtImm + opnd0, + opnd1 + ); +} + +/// pop - Pop a register off the stack +pub fn pop(cb: &mut CodeBlock, opnd: X86Opnd) { + match opnd { + X86Opnd::Reg(reg) => { + assert!(reg.num_bits == 64); + + if opnd.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0x58, reg); + }, + X86Opnd::Mem(mem) => { + assert!(mem.num_bits == 64); + + write_rm(cb, false, false, X86Opnd::None, opnd, Some(0), &[0x8f]); + }, + _ => unreachable!() + }; +} + +/// popfq - Pop the flags register (64-bit) +pub fn popfq(cb: &mut CodeBlock) { + // REX.W + 0x9D + cb.write_bytes(&[0x48, 0x9d]); +} + +/// push - Push an operand on the stack +pub fn push(cb: &mut CodeBlock, opnd: X86Opnd) { + match opnd { + X86Opnd::Reg(reg) => { + if opnd.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0x50, reg); + }, + X86Opnd::Mem(_mem) => { + write_rm(cb, false, false, X86Opnd::None, opnd, Some(6), &[0xff]); + }, + _ => unreachable!() + } +} + +/// pushfq - Push the flags register (64-bit) +pub fn pushfq(cb: &mut CodeBlock) { + cb.write_byte(0x9C); +} + +/// ret - Return from call, popping only the return address +pub fn ret(cb: &mut CodeBlock) { + cb.write_byte(0xC3); +} + +// Encode a bitwise shift instruction +fn write_shift(cb: &mut CodeBlock, op_mem_one_pref: u8, op_mem_cl_pref: u8, op_mem_imm_pref: u8, op_ext: u8, opnd0: X86Opnd, opnd1: X86Opnd) { + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + + // Check the size of opnd0 + let opnd_size = opnd0.num_bits(); + assert!(opnd_size == 16 || opnd_size == 32 || opnd_size == 64); + + let sz_pref = opnd_size == 16; + let rex_w = opnd_size == 64; + + match opnd1 { + X86Opnd::UImm(imm) => { + if imm.value == 1 { + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_one_pref]); + } else { + assert!(imm.num_bits <= 8); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_imm_pref]); + cb.write_byte(imm.value as u8); + } + } + + X86Opnd::Reg(reg) => { + // We can only use CL/RCX as the shift amount + assert!(reg.reg_no == RCX_REG.reg_no); + write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, Some(op_ext), &[op_mem_cl_pref]); + } + + _ => { + unreachable!("unsupported operands: {:?}, {:?}", opnd0, opnd1); + } + } +} + +// sal - Shift arithmetic left +pub fn sal(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x04, + opnd0, + opnd1 + ); +} + +/// sar - Shift arithmetic right (signed) +pub fn sar(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x07, + opnd0, + opnd1 + ); +} + +// shl - Shift logical left +pub fn shl(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x04, + opnd0, + opnd1 + ); +} + +/// shr - Shift logical right (unsigned) +pub fn shr(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_shift( + cb, + 0xD1, // opMemOnePref, + 0xD3, // opMemClPref, + 0xC1, // opMemImmPref, + 0x05, + opnd0, + opnd1 + ); +} + +/// sub - Integer subtraction +pub fn sub(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x28, // opMemReg8 + 0x29, // opMemRegPref + 0x2A, // opRegMem8 + 0x2B, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x05), // opExtImm + opnd0, + opnd1 + ); +} + +fn resize_opnd(opnd: X86Opnd, num_bits: u8) -> X86Opnd { + match opnd { + X86Opnd::Reg(reg) => { + let mut cloned = reg; + cloned.num_bits = num_bits; + X86Opnd::Reg(cloned) + }, + X86Opnd::Mem(mem) => { + let mut cloned = mem; + cloned.num_bits = num_bits; + X86Opnd::Mem(cloned) + }, + _ => unreachable!() + } +} + +/// test - Logical Compare +pub fn test(cb: &mut CodeBlock, rm_opnd: X86Opnd, test_opnd: X86Opnd) { + assert!(matches!(rm_opnd, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + let rm_num_bits = rm_opnd.num_bits(); + + match test_opnd { + X86Opnd::UImm(uimm) => { + assert!(uimm.num_bits <= 32); + assert!(uimm.num_bits <= rm_num_bits); + + // Use the smallest operand size possible + assert!(rm_num_bits % 8 == 0); + let rm_resized = resize_opnd(rm_opnd, uimm.num_bits); + + if uimm.num_bits == 8 { + write_rm(cb, false, false, X86Opnd::None, rm_resized, Some(0x00), &[0xf6]); + cb.write_int(uimm.value, uimm.num_bits.into()); + } else { + write_rm(cb, uimm.num_bits == 16, false, X86Opnd::None, rm_resized, Some(0x00), &[0xf7]); + cb.write_int(uimm.value, uimm.num_bits.into()); + } + }, + X86Opnd::Imm(imm) => { + // This mode only applies to 64-bit R/M operands with 32-bit signed immediates + assert!(imm.num_bits <= 32); + assert!(rm_num_bits == 64); + + write_rm(cb, false, true, X86Opnd::None, rm_opnd, Some(0x00), &[0xf7]); + cb.write_int(imm.value as u64, 32); + }, + X86Opnd::Reg(reg) => { + assert!(reg.num_bits == rm_num_bits); + + if rm_num_bits == 8 { + write_rm(cb, false, false, test_opnd, rm_opnd, None, &[0x84]); + } else { + write_rm(cb, rm_num_bits == 16, rm_num_bits == 64, test_opnd, rm_opnd, None, &[0x85]); + } + }, + _ => unreachable!("unexpected operands for test: {rm_opnd:?}, {test_opnd:?}") + }; +} + +/// Undefined opcode +pub fn ud2(cb: &mut CodeBlock) { + cb.write_bytes(&[0x0f, 0x0b]); +} + +/// xchg - Exchange Register/Memory with Register +pub fn xchg(cb: &mut CodeBlock, rm_opnd: X86Opnd, r_opnd: X86Opnd) { + if let (X86Opnd::Reg(rm_reg), X86Opnd::Reg(r_reg)) = (rm_opnd, r_opnd) { + assert!(rm_reg.num_bits == 64); + assert!(r_reg.num_bits == 64); + + // If we're exchanging with RAX + if rm_reg.reg_no == RAX_REG_NO { + // Write the REX byte + write_rex(cb, true, 0, 0, r_reg.reg_no); + + // Write the opcode and register number + cb.write_byte(0x90 + (r_reg.reg_no & 7)); + } else { + write_rm(cb, false, true, r_opnd, rm_opnd, None, &[0x87]); + } + } else { + unreachable!(); + } +} + +/// xor - Exclusive bitwise OR +pub fn xor(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { + write_rm_multi( + cb, + 0x30, // opMemReg8 + 0x31, // opMemRegPref + 0x32, // opRegMem8 + 0x33, // opRegMemPref + 0x80, // opMemImm8 + 0x83, // opMemImmSml + 0x81, // opMemImmLrg + Some(0x06), // opExtImm + opnd0, + opnd1 + ); +} + +impl fmt::Display for X86Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match X86Opnd::Reg(*self) { + RAX => write!(f, "rax"), + RCX => write!(f, "rcx"), + RDX => write!(f, "rdx"), + RBX => write!(f, "rbx"), + RSP => write!(f, "rsp"), + RBP => write!(f, "rbp"), + RSI => write!(f, "rsi"), + RDI => write!(f, "rdi"), + R8 => write!(f, "r8"), + R9 => write!(f, "r9"), + R10 => write!(f, "r10"), + R11 => write!(f, "r11"), + R12 => write!(f, "r12"), + R13 => write!(f, "r13"), + R14 => write!(f, "r14"), + R15 => write!(f, "r15"), + EAX => write!(f, "eax"), + ECX => write!(f, "ecx"), + EDX => write!(f, "edx"), + EBX => write!(f, "ebx"), + ESP => write!(f, "esp"), + EBP => write!(f, "ebp"), + ESI => write!(f, "esi"), + EDI => write!(f, "edi"), + R8D => write!(f, "r8d"), + R9D => write!(f, "r9d"), + R10D => write!(f, "r10d"), + R11D => write!(f, "r11d"), + R12D => write!(f, "r12d"), + R13D => write!(f, "r13d"), + R14D => write!(f, "r14d"), + R15D => write!(f, "r15d"), + AX => write!(f, "ax"), + CX => write!(f, "cx"), + DX => write!(f, "dx"), + BX => write!(f, "bx"), + BP => write!(f, "bp"), + SI => write!(f, "si"), + DI => write!(f, "di"), + R8W => write!(f, "r8w"), + R9W => write!(f, "r9w"), + R10W => write!(f, "r10w"), + R11W => write!(f, "r11w"), + R12W => write!(f, "r12w"), + R13W => write!(f, "r13w"), + R14W => write!(f, "r14w"), + R15W => write!(f, "r15w"), + AL => write!(f, "al"), + CL => write!(f, "cl"), + DL => write!(f, "dl"), + BL => write!(f, "bl"), + SPL => write!(f, "spl"), + BPL => write!(f, "bpl"), + SIL => write!(f, "sil"), + DIL => write!(f, "dil"), + R8B => write!(f, "r8b"), + R9B => write!(f, "r9b"), + R10B => write!(f, "r10b"), + R11B => write!(f, "r11b"), + R12B => write!(f, "r12b"), + R13B => write!(f, "r13b"), + R14B => write!(f, "r14b"), + R15B => write!(f, "r15b"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/x86_64/tests.rs b/zjit/src/asm/x86_64/tests.rs new file mode 100644 index 0000000000..268fe6b1c0 --- /dev/null +++ b/zjit/src/asm/x86_64/tests.rs @@ -0,0 +1,966 @@ +#![cfg(test)] + +use insta::assert_snapshot; + +#[cfg(feature = "disasm")] +use crate::disasms; +use crate::{asm::x86_64::*, hexdumps, assert_disasm_snapshot}; + +/// Check that the bytes for an instruction sequence match a hex string +fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + assert_eq!(format!("{:x}", cb), bytes); +} + +fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb +} + +#[test] +fn test_add() { + let cb01 = compile(|cb| add(cb, CL, imm_opnd(3))); + let cb02 = compile(|cb| add(cb, CL, BL)); + let cb03 = compile(|cb| add(cb, CL, SPL)); + let cb04 = compile(|cb| add(cb, CX, BX)); + let cb05 = compile(|cb| add(cb, RAX, RBX)); + let cb06 = compile(|cb| add(cb, ECX, EDX)); + let cb07 = compile(|cb| add(cb, RDX, R14)); + let cb08 = compile(|cb| add(cb, mem_opnd(64, RAX, 0), RDX)); + let cb09 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 0))); + let cb10 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 8))); + let cb11 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 255))); + let cb12 = compile(|cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); + let cb13 = compile(|cb| add(cb, mem_opnd(32, RAX, 0), EDX)); + let cb14 = compile(|cb| add(cb, RSP, imm_opnd(8))); + let cb15 = compile(|cb| add(cb, ECX, imm_opnd(8))); + let cb16 = compile(|cb| add(cb, ECX, imm_opnd(255))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @r" + 0x0: add cl, 3 + 0x0: add cl, bl + 0x0: add cl, spl + 0x0: add cx, bx + 0x0: add rax, rbx + 0x0: add ecx, edx + 0x0: add rdx, r14 + 0x0: add qword ptr [rax], rdx + 0x0: add rdx, qword ptr [rax] + 0x0: add rdx, qword ptr [rax + 8] + 0x0: add rdx, qword ptr [rax + 0xff] + 0x0: add qword ptr [rax + 0x7f], 0xff + 0x0: add dword ptr [rax], edx + 0x0: add rsp, 8 + 0x0: add ecx, 8 + 0x0: add ecx, 0xff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @r" + 80c103 + 00d9 + 4000e1 + 6601d9 + 4801d8 + 01d1 + 4c01f2 + 480110 + 480310 + 48035008 + 480390ff000000 + 4881407fff000000 + 0110 + 4883c408 + 83c108 + 81c1ff000000 + "); +} + +#[test] +fn test_add_unsigned() { + // ADD r/m8, imm8 + let cb1 = compile(|cb| add(cb, R8B, uimm_opnd(1))); + let cb2 = compile(|cb| add(cb, R8B, imm_opnd(i8::MAX.into()))); + // ADD r/m16, imm16 + let cb3 = compile(|cb| add(cb, R8W, uimm_opnd(1))); + let cb4 = compile(|cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); + // ADD r/m32, imm32 + let cb5 = compile(|cb| add(cb, R8D, uimm_opnd(1))); + let cb6 = compile(|cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); + // ADD r/m64, imm32 + let cb7 = compile(|cb| add(cb, R8, uimm_opnd(1))); + let cb8 = compile(|cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: add r8b, 1 + 0x0: add r8b, 0x7f + 0x0: add r8w, 1 + 0x0: add r8w, 0x7fff + 0x0: add r8d, 1 + 0x0: add r8d, 0x7fffffff + 0x0: add r8, 1 + 0x0: add r8, 0x7fffffff + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 4180c001 + 4180c07f + 664183c001 + 664181c0ff7f + 4183c001 + 4181c0ffffff7f + 4983c001 + 4981c0ffffff7f + "); +} + +#[test] +fn test_and() { + let cb1 = compile(|cb| and(cb, EBP, R12D)); + let cb2 = compile(|cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: and ebp, r12d + 0x0: and qword ptr [rax], 8 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 4421e5 + 48832008 + "); +} + +#[test] +fn test_call_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("fn".to_owned()); + call_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); +} + +#[test] +fn test_call_ptr() { + // calling a lower address + let cb = compile(|cb| { + let ptr = cb.get_write_ptr(); + call_ptr(cb, RAX, ptr.raw_ptr(cb)); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); +} + +#[test] +fn test_call_reg() { + let cb = compile(|cb| call(cb, RAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call rax"); + assert_snapshot!(cb.hexdump(), @"ffd0"); +} + +#[test] +fn test_call_mem() { + let cb = compile(|cb| call(cb, mem_opnd(64, RSP, 8))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call qword ptr [rsp + 8]"); + assert_snapshot!(cb.hexdump(), @"ff542408"); +} + +#[test] +fn test_cmovcc() { + let cb1 = compile(|cb| cmovg(cb, ESI, EDI)); + let cb2 = compile(|cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); + let cb3 = compile(|cb| cmovl(cb, EAX, ECX)); + let cb4 = compile(|cb| cmovl(cb, RBX, RBP)); + let cb5 = compile(|cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: cmovg esi, edi + 0x0: cmovg esi, dword ptr [rbp + 0xc] + 0x0: cmovl eax, ecx + 0x0: cmovl rbx, rbp + 0x0: cmovle esi, dword ptr [rsp + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 0f4ff7 + 0f4f750c + 0f4cc1 + 480f4cdd + 0f4e742404 + "); +} + +#[test] +fn test_cmp() { + let cb1 = compile(|cb| cmp(cb, CL, DL)); + let cb2 = compile(|cb| cmp(cb, ECX, EDI)); + let cb3 = compile(|cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); + let cb4 = compile(|cb| cmp(cb, RAX, imm_opnd(2))); + let cb5 = compile(|cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: cmp cl, dl + 0x0: cmp ecx, edi + 0x0: cmp rdx, qword ptr [r12] + 0x0: cmp rax, 2 + 0x0: cmp ecx, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 38d1 + 39f9 + 493b1424 + 4883f802 + 81f900000080 + "); +} + +#[test] +fn test_cqo() { + let cb = compile(cqo); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cqo"); + assert_snapshot!(cb.hexdump(), @"4899"); +} + +#[test] +fn test_imul() { + let cb1 = compile(|cb| imul(cb, RAX, RBX)); + let cb2 = compile(|cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: imul rax, rbx + 0x0: imul rdx, qword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 480fafc3 + 480faf10 + "); +} + +#[test] +#[should_panic] +fn test_imul_mem_reg() { + // imul doesn't have (Mem, Reg) encoding. Since multiplication is communicative, imul() could + // swap operands. However, x86_scratch_split may need to move the result to the output operand, + // which can be complicated if the assembler may sometimes change the result operand. + // So x86_scratch_split should be responsible for that swap, not the assembler. + compile(|cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); +} + +#[test] +fn test_jge_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + jge_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jge 0"); + assert_snapshot!(cb.hexdump(), @"0f8dfaffffff"); +} + +#[test] +fn test_jmp_label() { + // Forward jump + let cb1 = compile(|cb| { + let label_idx = cb.new_label("next".to_owned()); + jmp_label(cb, label_idx); + cb.write_label(label_idx); + cb.link_labels().unwrap(); + }); + // Backwards jump + let cb2 = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + cb.write_label(label_idx); + jmp_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: jmp 5 + 0x0: jmp 0 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + e900000000 + e9fbffffff + "); +} + +#[test] +fn test_jmp_rm() { + let cb = compile(|cb| jmp_rm(cb, R12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jmp r12"); + assert_snapshot!(cb.hexdump(), @"41ffe4"); +} + +#[test] +fn test_jo_label() { + let cb = compile(|cb| { + let label_idx = cb.new_label("loop".to_owned()); + jo_label(cb, label_idx); + cb.link_labels().unwrap(); + }); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jo 0"); + assert_snapshot!(cb.hexdump(), @"0f80faffffff"); +} + +#[test] +fn test_lea() { + let cb1 = compile(|cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); + let cb2 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); + let cb3 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); + let cb4 = compile(|cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @r" + 0x0: lea rdx, [rcx + 8] + 0x0: lea rax, [rip] + 0x0: lea rax, [rip + 5] + 0x0: lea rdi, [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @r" + 488d5108 + 488d0500000000 + 488d0505000000 + 488d3d05000000 + "); +} + +#[test] +fn test_mov() { + let cb01 = compile(|cb| mov(cb, EAX, imm_opnd(7))); + let cb02 = compile(|cb| mov(cb, EAX, imm_opnd(-3))); + let cb03 = compile(|cb| mov(cb, R15, imm_opnd(3))); + let cb04 = compile(|cb| mov(cb, EAX, EBX)); + let cb05 = compile(|cb| mov(cb, EAX, ECX)); + let cb06 = compile(|cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); + let cb07 = compile(|cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); + // Test `mov rax, 3` => `mov eax, 3` optimization + let cb08 = compile(|cb| mov(cb, R8, imm_opnd(0x34))); + let cb09 = compile(|cb| mov(cb, R8, imm_opnd(0x80000000))); + let cb10 = compile(|cb| mov(cb, R8, imm_opnd(-1))); + let cb11 = compile(|cb| mov(cb, RAX, imm_opnd(0x34))); + let cb12 = compile(|cb| mov(cb, RAX, imm_opnd(-18014398509481982))); + let cb13 = compile(|cb| mov(cb, RAX, imm_opnd(0x80000000))); + let cb14 = compile(|cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword + let cb15 = compile(|cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword + let cb16 = compile(|cb| mov(cb, CL, R9B)); + let cb17 = compile(|cb| mov(cb, RBX, RAX)); + let cb18 = compile(|cb| mov(cb, RDI, RBX)); + let cb19 = compile(|cb| mov(cb, SIL, imm_opnd(11))); + let cb20 = compile(|cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); + let cb21 = compile(|cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); + //let cb = compile(|cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? + let cb22 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); + let cb23 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), uimm_opnd(0x80000001))); + let cb24 = compile(|cb| mov(cb, mem_opnd(32, R8, 20), EBX)); + let cb25 = compile(|cb| mov(cb, mem_opnd(64, R11, 0), R10)); + let cb26 = compile(|cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + + assert_disasm_snapshot!(disasms!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26, + ), @r" + 0x0: mov eax, 7 + 0x0: mov eax, 0xfffffffd + 0x0: mov r15d, 3 + 0x0: mov eax, ebx + 0x0: mov eax, ecx + 0x0: mov edx, dword ptr [rbx + 0x80] + 0x0: mov rax, qword ptr [rsp + 4] + 0x0: mov r8d, 0x34 + 0x0: movabs r8, 0x80000000 + 0x0: movabs r8, 0xffffffffffffffff + 0x0: mov eax, 0x34 + 0x0: movabs rax, 0xffc0000000000002 + 0x0: movabs rax, 0x80000000 + 0x0: movabs rax, 0xffffffffffffffcc + 0x0: movabs rax, 0xffffffffffffffff + 0x0: mov cl, r9b + 0x0: mov rbx, rax + 0x0: mov rdi, rbx + 0x0: mov sil, 0xb + 0x0: mov byte ptr [rsp], 0xfd + 0x0: mov qword ptr [rdi + 8], 1 + 0x0: mov dword ptr [rax + 4], 0x11 + 0x0: mov dword ptr [rax + 4], 0x80000001 + 0x0: mov dword ptr [r8 + 0x14], ebx + 0x0: mov qword ptr [r11], r10 + 0x0: mov qword ptr [rdx - 8], 0xfffffffffffffff4 + "); + + assert_snapshot!(hexdumps!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26 + ), @r" + b807000000 + b8fdffffff + 41bf03000000 + 89d8 + 89c8 + 8b9380000000 + 488b442404 + 41b834000000 + 49b80000008000000000 + 49b8ffffffffffffffff + b834000000 + 48b8020000000000c0ff + 48b80000008000000000 + 48b8ccffffffffffffff + 48b8ffffffffffffffff + 4488c9 + 4889c3 + 4889df + 40b60b + c60424fd + 48c7470801000000 + c7400411000000 + c7400401000080 + 41895814 + 4d8913 + 48c742f8f4ffffff + "); +} + +#[test] +fn test_movabs() { + let cb1 = compile(|cb| movabs(cb, R8, 0x34)); + let cb2 = compile(|cb| movabs(cb, R8, 0x80000000)); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: movabs r8, 0x34 + 0x0: movabs r8, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 49b83400000000000000 + 49b80000008000000000 + "); +} + +#[test] +fn test_mov_unsigned() { + // MOV AL, imm8 + let cb01 = compile(|cb| mov(cb, AL, uimm_opnd(1))); + let cb02 = compile(|cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); + // MOV AX, imm16 + let cb03 = compile(|cb| mov(cb, AX, uimm_opnd(1))); + let cb04 = compile(|cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); + // MOV EAX, imm32 + let cb05 = compile(|cb| mov(cb, EAX, uimm_opnd(1))); + let cb06 = compile(|cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); + let cb07 = compile(|cb| mov(cb, R8, uimm_opnd(0))); + let cb08 = compile(|cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); + // MOV RAX, imm64, will move down into EAX since it fits into 32 bits + let cb09 = compile(|cb| mov(cb, RAX, uimm_opnd(1))); + let cb10 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); + // MOV RAX, imm64, will not move down into EAX since it does not fit into 32 bits + let cb11 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); + let cb12 = compile(|cb| mov(cb, RAX, uimm_opnd(u64::MAX))); + let cb13 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + // MOV r8, imm8 + let cb14 = compile(|cb| mov(cb, R8B, uimm_opnd(1))); + let cb15 = compile(|cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); + // MOV r16, imm16 + let cb16 = compile(|cb| mov(cb, R8W, uimm_opnd(1))); + let cb17 = compile(|cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); + // MOV r32, imm32 + let cb18 = compile(|cb| mov(cb, R8D, uimm_opnd(1))); + let cb19 = compile(|cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); + // MOV r64, imm64, will move down into 32 bit since it fits into 32 bits + let cb20 = compile(|cb| mov(cb, R8, uimm_opnd(1))); + // MOV r64, imm64, will not move down into 32 bit since it does not fit into 32 bits + let cb21 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @r" + 0x0: mov al, 1 + 0x0: mov al, 0xff + 0x0: mov ax, 1 + 0x0: mov ax, 0xffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: mov r8d, 0 + 0x0: mov r8d, 0xffffffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: movabs rax, 0x100000000 + 0x0: movabs rax, 0xffffffffffffffff + 0x0: movabs r8, 0xffffffffffffffff + 0x0: mov r8b, 1 + 0x0: mov r8b, 0xff + 0x0: mov r8w, 1 + 0x0: mov r8w, 0xffff + 0x0: mov r8d, 1 + 0x0: mov r8d, 0xffffffff + 0x0: mov r8d, 1 + 0x0: movabs r8, 0xffffffffffffffff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @r" + b001 + b0ff + 66b80100 + 66b8ffff + b801000000 + b8ffffffff + 41b800000000 + 41b8ffffffff + b801000000 + b8ffffffff + 48b80000000001000000 + 48b8ffffffffffffffff + 49b8ffffffffffffffff + 41b001 + 41b0ff + 6641b80100 + 6641b8ffff + 41b801000000 + 41b8ffffffff + 41b801000000 + 49b8ffffffffffffffff + "); +} + +#[test] +fn test_mov_iprel() { + let cb1 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); + let cb2 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); + let cb3 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); + let cb4 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); + let cb5 = compile(|cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: mov eax, dword ptr [rip] + 0x0: mov eax, dword ptr [rip + 5] + 0x0: mov rax, qword ptr [rip] + 0x0: mov rax, qword ptr [rip + 5] + 0x0: mov rdi, qword ptr [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 8b0500000000 + 8b0505000000 + 488b0500000000 + 488b0505000000 + 488b3d05000000 + "); +} + +#[test] +fn test_movsx() { + let cb1 = compile(|cb| movsx(cb, AX, AL)); + let cb2 = compile(|cb| movsx(cb, EDX, AL)); + let cb3 = compile(|cb| movsx(cb, RAX, BL)); + let cb4 = compile(|cb| movsx(cb, ECX, AX)); + let cb5 = compile(|cb| movsx(cb, R11, CL)); + let cb6 = compile(|cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); + let cb7 = compile(|cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); + let cb8 = compile(|cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: movsx ax, al + 0x0: movsx edx, al + 0x0: movsx rax, bl + 0x0: movsx ecx, ax + 0x0: movsx r11, cl + 0x0: movsxd r10, dword ptr [rsp + 0xc] + 0x0: movsx rax, byte ptr [rsp] + 0x0: movsx rdx, word ptr [r13 + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 660fbec0 + 0fbed0 + 480fbec3 + 0fbfc8 + 4c0fbed9 + 4c6354240c + 480fbe0424 + 490fbf5504 + "); +} + +#[test] +fn test_nop() { + let cb01 = compile(|cb| nop(cb, 1)); + let cb02 = compile(|cb| nop(cb, 2)); + let cb03 = compile(|cb| nop(cb, 3)); + let cb04 = compile(|cb| nop(cb, 4)); + let cb05 = compile(|cb| nop(cb, 5)); + let cb06 = compile(|cb| nop(cb, 6)); + let cb07 = compile(|cb| nop(cb, 7)); + let cb08 = compile(|cb| nop(cb, 8)); + let cb09 = compile(|cb| nop(cb, 9)); + let cb10 = compile(|cb| nop(cb, 10)); + let cb11 = compile(|cb| nop(cb, 11)); + let cb12 = compile(|cb| nop(cb, 12)); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @r" + 0x0: nop + 0x0: nop + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop dword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @r" + 90 + 6690 + 0f1f00 + 0f1f4000 + 0f1f440000 + 660f1f440000 + 0f1f8000000000 + 0f1f840000000000 + 660f1f840000000000 + 660f1f84000000000090 + 660f1f8400000000006690 + 660f1f8400000000000f1f00 + "); +} + +#[test] +fn test_not() { + let cb01 = compile(|cb| not(cb, AX)); + let cb02 = compile(|cb| not(cb, EAX)); + let cb03 = compile(|cb| not(cb, mem_opnd(64, R12, 0))); + let cb04 = compile(|cb| not(cb, mem_opnd(32, RSP, 301))); + let cb05 = compile(|cb| not(cb, mem_opnd(32, RSP, 0))); + let cb06 = compile(|cb| not(cb, mem_opnd(32, RSP, 3))); + let cb07 = compile(|cb| not(cb, mem_opnd(32, RBP, 0))); + let cb08 = compile(|cb| not(cb, mem_opnd(32, RBP, 13))); + let cb09 = compile(|cb| not(cb, RAX)); + let cb10 = compile(|cb| not(cb, R11)); + let cb11 = compile(|cb| not(cb, mem_opnd(32, RAX, 0))); + let cb12 = compile(|cb| not(cb, mem_opnd(32, RSI, 0))); + let cb13 = compile(|cb| not(cb, mem_opnd(32, RDI, 0))); + let cb14 = compile(|cb| not(cb, mem_opnd(32, RDX, 55))); + let cb15 = compile(|cb| not(cb, mem_opnd(32, RDX, 1337))); + let cb16 = compile(|cb| not(cb, mem_opnd(32, RDX, -55))); + let cb17 = compile(|cb| not(cb, mem_opnd(32, RDX, -555))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @r" + 0x0: not ax + 0x0: not eax + 0x0: not qword ptr [r12] + 0x0: not dword ptr [rsp + 0x12d] + 0x0: not dword ptr [rsp] + 0x0: not dword ptr [rsp + 3] + 0x0: not dword ptr [rbp] + 0x0: not dword ptr [rbp + 0xd] + 0x0: not rax + 0x0: not r11 + 0x0: not dword ptr [rax] + 0x0: not dword ptr [rsi] + 0x0: not dword ptr [rdi] + 0x0: not dword ptr [rdx + 0x37] + 0x0: not dword ptr [rdx + 0x539] + 0x0: not dword ptr [rdx - 0x37] + 0x0: not dword ptr [rdx - 0x22b] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @r" + 66f7d0 + f7d0 + 49f71424 + f794242d010000 + f71424 + f7542403 + f75500 + f7550d + 48f7d0 + 49f7d3 + f710 + f716 + f717 + f75237 + f79239050000 + f752c9 + f792d5fdffff + "); +} + +#[test] +fn test_or() { + let cb = compile(|cb| or(cb, EDX, ESI)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or edx, esi"); + assert_snapshot!(cb.hexdump(), @"09f2"); +} + +#[test] +fn test_pop() { + let cb01 = compile(|cb| pop(cb, RAX)); + let cb02 = compile(|cb| pop(cb, RBX)); + let cb03 = compile(|cb| pop(cb, RSP)); + let cb04 = compile(|cb| pop(cb, RBP)); + let cb05 = compile(|cb| pop(cb, R12)); + let cb06 = compile(|cb| pop(cb, mem_opnd(64, RAX, 0))); + let cb07 = compile(|cb| pop(cb, mem_opnd(64, R8, 0))); + let cb08 = compile(|cb| pop(cb, mem_opnd(64, R8, 3))); + let cb09 = compile(|cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb10 = compile(|cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @r" + 0x0: pop rax + 0x0: pop rbx + 0x0: pop rsp + 0x0: pop rbp + 0x0: pop r12 + 0x0: pop qword ptr [rax] + 0x0: pop qword ptr [r8] + 0x0: pop qword ptr [r8 + 3] + 0x0: pop qword ptr [rax + rcx*8 + 3] + 0x0: pop qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @r" + 58 + 5b + 5c + 5d + 415c + 8f00 + 418f00 + 418f4003 + 8f44c803 + 418f44c803 + "); +} + +#[test] +fn test_push() { + let cb1 = compile(|cb| push(cb, RAX)); + let cb2 = compile(|cb| push(cb, RBX)); + let cb3 = compile(|cb| push(cb, R12)); + let cb4 = compile(|cb| push(cb, mem_opnd(64, RAX, 0))); + let cb5 = compile(|cb| push(cb, mem_opnd(64, R8, 0))); + let cb6 = compile(|cb| push(cb, mem_opnd(64, R8, 3))); + let cb7 = compile(|cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb8 = compile(|cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: push rax + 0x0: push rbx + 0x0: push r12 + 0x0: push qword ptr [rax] + 0x0: push qword ptr [r8] + 0x0: push qword ptr [r8 + 3] + 0x0: push qword ptr [rax + rcx*8 + 3] + 0x0: push qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 50 + 53 + 4154 + ff30 + 41ff30 + 41ff7003 + ff74c803 + 41ff74c803 + "); +} + +#[test] +fn test_ret() { + let cb = compile(ret); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c3"); +} + +#[test] +fn test_sal() { + let cb1 = compile(|cb| sal(cb, CX, uimm_opnd(1))); + let cb2 = compile(|cb| sal(cb, ECX, uimm_opnd(1))); + let cb3 = compile(|cb| sal(cb, EBP, uimm_opnd(5))); + let cb4 = compile(|cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); + let cb5 = compile(|cb| sal(cb, RCX, CL)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: shl cx, 1 + 0x0: shl ecx, 1 + 0x0: shl ebp, 5 + 0x0: shl dword ptr [rsp + 0x44], 1 + 0x0: shl rcx, cl + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 66d1e1 + d1e1 + c1e505 + d1642444 + 48d3e1 + "); +} + +#[test] +fn test_sar() { + let cb = compile(|cb| sar(cb, EDX, uimm_opnd(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sar edx, 1"); + assert_snapshot!(cb.hexdump(), @"d1fa"); +} + +#[test] +fn test_shr() { + let cb = compile(|cb| shr(cb, R14, uimm_opnd(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: shr r14, 7"); + assert_snapshot!(cb.hexdump(), @"49c1ee07"); +} + +#[test] +fn test_sub() { + let cb1 = compile(|cb| sub(cb, EAX, imm_opnd(1))); + let cb2 = compile(|cb| sub(cb, RAX, imm_opnd(2))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: sub eax, 1 + 0x0: sub rax, 2 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 83e801 + 4883e802 + "); +} + +#[test] +#[should_panic] +fn test_sub_uimm_too_large() { + // This immediate becomes a different value after + // sign extension, so not safe to encode. + compile(|cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); +} + +#[test] +fn test_test() { + let cb01 = compile(|cb| test(cb, AL, AL)); + let cb02 = compile(|cb| test(cb, AX, AX)); + let cb03 = compile(|cb| test(cb, CL, uimm_opnd(8))); + let cb04 = compile(|cb| test(cb, DL, uimm_opnd(7))); + let cb05 = compile(|cb| test(cb, RCX, uimm_opnd(8))); + let cb06 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); + let cb07 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); + let cb08 = compile(|cb| test(cb, DX, uimm_opnd(0xffff))); + let cb09 = compile(|cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); + let cb10 = compile(|cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); + let cb11 = compile(|cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); + let cb12 = compile(|cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); + let cb13 = compile(|cb| test(cb, mem_opnd(32, RSI, 64), EAX)); + let cb14 = compile(|cb| test(cb, mem_opnd(64, RDI, 42), RAX)); + let cb15 = compile(|cb| test(cb, RAX, RAX)); + let cb16 = compile(|cb| test(cb, RAX, RSI)); + let cb17 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); + let cb18 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); + let cb19 = compile(|cb| test(cb, RCX, imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @r" + 0x0: test al, al + 0x0: test ax, ax + 0x0: test cl, 8 + 0x0: test dl, 7 + 0x0: test cl, 8 + 0x0: test byte ptr [rdx + 8], 8 + 0x0: test byte ptr [rdx + 8], 0xff + 0x0: test dx, 0xffff + 0x0: test word ptr [rdx + 8], 0xffff + 0x0: test byte ptr [rsi], 1 + 0x0: test byte ptr [rsi + 0x10], 1 + 0x0: test byte ptr [rsi - 0x10], 1 + 0x0: test dword ptr [rsi + 0x40], eax + 0x0: test qword ptr [rdi + 0x2a], rax + 0x0: test rax, rax + 0x0: test rax, rsi + 0x0: test qword ptr [rsi + 0x40], -9 + 0x0: test qword ptr [rsi + 0x40], 8 + 0x0: test rcx, 8 + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @r" + 84c0 + 6685c0 + f6c108 + f6c207 + f6c108 + f6420808 + f64208ff + 66f7c2ffff + 66f74208ffff + f60601 + f6461001 + f646f001 + 854640 + 4885472a + 4885c0 + 4885f0 + 48f74640f7ffffff + 48f7464008000000 + 48f7c108000000 + "); +} + +#[test] +fn test_xchg() { + let cb1 = compile(|cb| xchg(cb, RAX, RCX)); + let cb2 = compile(|cb| xchg(cb, RAX, R13)); + let cb3 = compile(|cb| xchg(cb, RCX, RBX)); + let cb4 = compile(|cb| xchg(cb, R9, R15)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @r" + 0x0: xchg rcx, rax + 0x0: xchg r13, rax + 0x0: xchg rcx, rbx + 0x0: xchg r9, r15 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @r" + 4891 + 4995 + 4887d9 + 4d87f9 + "); +} + +#[test] +fn test_xor() { + let cb = compile(|cb| xor(cb, EAX, EAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor eax, eax"); + assert_snapshot!(cb.hexdump(), @"31c0"); +} + +#[test] +#[cfg(feature = "disasm")] +fn basic_capstone_usage() -> std::result::Result<(), capstone::Error> { + // Test drive Capstone with simple input + use capstone::prelude::*; + let cs = Capstone::new() + .x86() + .mode(arch::x86::ArchMode::Mode64) + .syntax(arch::x86::ArchSyntax::Intel) + .build()?; + + let insns = cs.disasm_all(&[0xCC], 0x1000)?; + + match insns.as_ref() { + [insn] => { + assert_eq!(Some("int3"), insn.mnemonic()); + Ok(()) + } + _ => Err(capstone::Error::CustomError( + "expected to disassemble to int3", + )), + } +} diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs new file mode 100644 index 0000000000..d06e84536f --- /dev/null +++ b/zjit/src/backend/arm64/mod.rs @@ -0,0 +1,2901 @@ +use std::mem::take; + +use crate::asm::{CodeBlock, Label}; +use crate::asm::arm64::*; +use crate::codegen::split_patch_point; +use crate::cruby::*; +use crate::backend::lir::*; +use crate::options::asm_dump; +use crate::stats::CompileError; +use crate::virtualmem::CodePtr; +use crate::cast::*; + +// Use the arm64 register type for this platform +pub type Reg = A64Reg; + +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_no } +} + +// Callee-saved registers +pub const CFP: Opnd = Opnd::Reg(X19_REG); +pub const EC: Opnd = Opnd::Reg(X20_REG); +pub const SP: Opnd = Opnd::Reg(X21_REG); + +// C argument registers on this platform +pub const C_ARG_OPNDS: [Opnd; 6] = [ + Opnd::Reg(X0_REG), + Opnd::Reg(X1_REG), + Opnd::Reg(X2_REG), + Opnd::Reg(X3_REG), + Opnd::Reg(X4_REG), + Opnd::Reg(X5_REG) +]; + +// C return value register on this platform +pub const C_RET_REG: Reg = X0_REG; +pub const C_RET_OPND: Opnd = Opnd::Reg(X0_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(XZR_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(X29_REG); + +// These constants define the way we work with Arm64's stack pointer. The stack +// pointer always needs to be aligned to a 16-byte boundary. +pub const C_SP_REG: A64Opnd = X31; +pub const C_SP_STEP: i32 = 16; + +impl CodeBlock { + // The maximum number of bytes that can be generated by emit_jmp_ptr. + pub fn jmp_ptr_bytes(&self) -> usize { + // b instruction's offset is encoded as imm26 times 4. It can jump to + // +/-128MiB, so this can be used when --yjit-exec-mem-size <= 128. + /* + let num_insns = if b_offset_fits_bits(self.virtual_region_size() as i64 / 4) { + 1 // b instruction + } else { + 5 // 4 instructions to load a 64-bit absolute address + br instruction + }; + */ + let num_insns = 5; // TODO: support virtual_region_size() check + num_insns * 4 + } + + // The maximum number of instructions that can be generated by emit_conditional_jump. + fn conditional_jump_insns(&self) -> i32 { + // The worst case is instructions for a jump + bcond. + self.jmp_ptr_bytes() as i32 / 4 + 1 + } +} + +/// Map Opnd to A64Opnd +impl From<Opnd> for A64Opnd { + fn from(opnd: Opnd) -> Self { + match opnd { + Opnd::UImm(value) => A64Opnd::new_uimm(value), + Opnd::Imm(value) => A64Opnd::new_imm(value), + Opnd::Reg(reg) => A64Opnd::Reg(reg), + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), num_bits, disp }) => { + A64Opnd::new_mem(num_bits, A64Opnd::Reg(A64Reg { num_bits, reg_no }), disp) + }, + Opnd::Mem(Mem { base: MemBase::VReg(_), .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::VReg base") + }, + Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::Stack base") + }, + Opnd::VReg { .. } => panic!("attempted to lower an Opnd::VReg"), + Opnd::Value(_) => panic!("attempted to lower an Opnd::Value"), + Opnd::None => panic!( + "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." + ), + } + } +} + +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for A64Opnd { + fn from(opnd: &Opnd) -> Self { + A64Opnd::from(*opnd) + } +} + +/// Call emit_jmp_ptr and immediately invalidate the written range. +/// This is needed when next_page also moves other_cb that is not invalidated +/// by compile_with_regs. Doing it here allows you to avoid invalidating a lot +/// more than necessary when other_cb jumps from a position early in the page. +/// This invalidates a small range of cb twice, but we accept the small cost. +fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { + let start = cb.get_write_ptr(); + emit_jmp_ptr(cb, dst_ptr, true); + let end = cb.get_write_ptr(); + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; +} + +fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { + let src_addr = cb.get_write_ptr().as_offset(); + let dst_addr = dst_ptr.as_offset(); + + // If the offset is short enough, then we'll use the + // branch instruction. Otherwise, we'll move the + // destination into a register and use the branch + // register instruction. + let num_insns = if b_offset_fits_bits((dst_addr - src_addr) / 4) { + b(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + 1 + } else { + let num_insns = emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + num_insns + 1 + }; + + if padding { + // Make sure it's always a consistent number of + // instructions in case it gets patched and has to + // use the other branch. + assert!(num_insns * 4 <= cb.jmp_ptr_bytes()); + for _ in num_insns..(cb.jmp_ptr_bytes() / 4) { + nop(cb); + } + } +} + +/// Emit the required instructions to load the given value into the +/// given register. Our goal here is to use as few instructions as +/// possible to get this value into the register. +fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { + let mut current = value; + + if current <= 0xffff { + // If the value fits into a single movz + // instruction, then we'll use that. + movz(cb, rd, A64Opnd::new_uimm(current), 0); + 1 + } else if u16::try_from(!value).is_ok() { + // For small negative values, use a single movn + movn(cb, rd, A64Opnd::new_uimm(!value), 0); + 1 + } else if BitmaskImmediate::try_from(current).is_ok() { + // Otherwise, if the immediate can be encoded + // with the special bitmask immediate encoding, + // we'll use that. + mov(cb, rd, A64Opnd::new_uimm(current)); + 1 + } else { + // Finally we'll fall back to encoding the value + // using movz for the first 16 bits and movk for + // each subsequent set of 16 bits as long we + // they are necessary. + movz(cb, rd, A64Opnd::new_uimm(current & 0xffff), 0); + let mut num_insns = 1; + + // (We're sure this is necessary since we + // checked if it only fit into movz above). + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 16); + num_insns += 1; + + if current > 0xffff { + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 32); + num_insns += 1; + } + + if current > 0xffff { + current >>= 16; + movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 48); + num_insns += 1; + } + num_insns + } +} + +/// List of registers that can be used for register allocation. +/// This has the same number of registers for x86_64 and arm64. +/// SCRATCH_OPND, SCRATCH1_OPND, and EMIT_OPND are excluded. +pub const ALLOC_REGS: &[Reg] = &[ + X0_REG, + X1_REG, + X2_REG, + X3_REG, + X4_REG, + X5_REG, + X11_REG, + X12_REG, +]; + +/// Special scratch registers for intermediate processing. They should be used only by +/// [`Assembler::arm64_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(X15_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(X17_REG); +const SCRATCH2_OPND: Opnd = Opnd::Reg(X14_REG); + +impl Assembler { + /// Special register for intermediate processing in arm64_emit. It should be used only by arm64_emit. + const EMIT_REG: Reg = X16_REG; + const EMIT_OPND: A64Opnd = A64Opnd::Reg(Self::EMIT_REG); + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } + + /// Get the list of registers from which we will allocate on this platform + pub fn get_alloc_regs() -> Vec<Reg> { + ALLOC_REGS.to_vec() + } + + /// Get a list of all of the caller-saved registers + pub fn get_caller_save_regs() -> Vec<Reg> { + vec![X1_REG, X9_REG, X10_REG, X11_REG, X12_REG, X13_REG, X14_REG, X15_REG] + } + + /// How many bytes a call and a [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + + /// Split platform-specific instructions + /// The transformations done here are meant to make our lives simpler in later + /// stages of the compilation pipeline. + /// Here we may want to make sure that all instructions (except load and store) + /// have no memory operands. + fn arm64_split(mut self) -> Assembler + { + /// When you're storing a register into a memory location or loading a + /// memory location into a register, the displacement from the base + /// register of the memory location must fit into 9 bits. If it doesn't, + /// then we need to load that memory address into a register first. + fn split_memory_address(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(mem) => { + if mem_disp_fits_bits(mem.disp) { + opnd + } else { + let base = asm.lea(Opnd::Mem(Mem { num_bits: 64, ..mem })); + Opnd::mem(mem.num_bits, base, 0) + } + }, + _ => unreachable!("Can only split memory addresses.") + } + } + + /// Any memory operands you're sending into an Op::Load instruction need + /// to be split in case their displacement doesn't fit into 9 bits. + fn split_load_operand(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => { + let split_opnd = split_memory_address(asm, opnd); + let out_opnd = asm.load(split_opnd); + // Many Arm insns support only 32-bit or 64-bit operands. asm.load with fewer + // bits zero-extends the value, so it's safe to recognize it as a 32-bit value. + if out_opnd.rm_num_bits() < 32 { + out_opnd.with_num_bits(32) + } else { + out_opnd + } + }, + _ => asm.load(opnd) + } + } + + /// Operands that take the place of bitmask immediates must follow a + /// certain encoding. In this function we ensure that those operands + /// do follow that encoding, and if they don't then we load them first. + fn split_bitmask_immediate(asm: &mut Assembler, opnd: Opnd, dest_num_bits: u8) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => split_load_operand(asm, opnd), + Opnd::Imm(imm) => { + if imm == 0 { + Opnd::Reg(XZR_REG) + } else if (dest_num_bits == 64 && + BitmaskImmediate::try_from(imm as u64).is_ok()) || + (dest_num_bits == 32 && + u32::try_from(imm).is_ok() && + BitmaskImmediate::new_32b_reg(imm as u32).is_ok()) { + Opnd::UImm(imm as u64) + } else { + asm.load(opnd).with_num_bits(dest_num_bits) + } + }, + Opnd::UImm(uimm) => { + if (dest_num_bits == 64 && BitmaskImmediate::try_from(uimm).is_ok()) || + (dest_num_bits == 32 && + u32::try_from(uimm).is_ok() && + BitmaskImmediate::new_32b_reg(uimm as u32).is_ok()) { + opnd + } else { + asm.load(opnd).with_num_bits(dest_num_bits) + } + }, + Opnd::None | Opnd::Value(_) => unreachable!() + } + } + + /// Operands that take the place of a shifted immediate must fit within + /// a certain size. If they don't then we need to load them first. + fn split_shifted_immediate(asm: &mut Assembler, opnd: Opnd) -> Opnd { + match opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd, + Opnd::Mem(_) => split_load_operand(asm, opnd), + Opnd::Imm(imm) => if ShiftedImmediate::try_from(imm as u64).is_ok() { + opnd + } else { + asm.load(opnd) + } + Opnd::UImm(uimm) => { + if ShiftedImmediate::try_from(uimm).is_ok() { + opnd + } else { + asm.load(opnd) + } + }, + Opnd::None | Opnd::Value(_) => unreachable!() + } + } + + /// Returns the operands that should be used for a boolean logic + /// instruction. + fn split_boolean_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + match (opnd0, opnd1) { + (Opnd::Reg(_), Opnd::Reg(_)) => { + (opnd0, opnd1) + }, + (reg_opnd @ Opnd::Reg(_), other_opnd) | + (other_opnd, reg_opnd @ Opnd::Reg(_)) => { + let opnd1 = split_bitmask_immediate(asm, other_opnd, reg_opnd.rm_num_bits()); + (reg_opnd, opnd1) + }, + _ => { + let opnd0 = split_load_operand(asm, opnd0); + let opnd1 = split_bitmask_immediate(asm, opnd1, opnd0.rm_num_bits()); + (opnd0, opnd1) + } + } + } + + /// Returns the operands that should be used for a csel instruction. + fn split_csel_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + let opnd0 = match opnd0 { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd0, + _ => split_load_operand(asm, opnd0) + }; + + let opnd1 = match opnd1 { + Opnd::Reg(_) | Opnd::VReg { .. } => opnd1, + _ => split_load_operand(asm, opnd1) + }; + + (opnd0, opnd1) + } + + fn split_less_than_32_cmp(asm: &mut Assembler, opnd0: Opnd) -> Opnd { + match opnd0 { + Opnd::Reg(_) | Opnd::VReg { .. } => { + match opnd0.rm_num_bits() { + 8 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xff)), + 16 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xffff)), + 32 | 64 => opnd0, + bits => unreachable!("Invalid number of bits. {}", bits) + } + } + _ => opnd0 + } + } + + let mut asm_local = Assembler::new_with_asm(&self); + let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); + let mut iterator = self.instruction_iterator(); + let asm = &mut asm_local; + + while let Some((index, mut insn)) = iterator.next(asm) { + // Here we're going to map the operands of the instruction to load + // any Opnd::Value operands into registers if they are heap objects + // such that only the Op::Load instruction needs to handle that + // case. If the values aren't heap objects then we'll treat them as + // if they were just unsigned integer. + let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); + let mut opnd_iter = insn.opnd_iter_mut(); + + while let Some(opnd) = opnd_iter.next() { + if let Opnd::Value(value) = opnd { + if value.special_const_p() { + *opnd = Opnd::UImm(value.as_u64()); + } else if !is_load { + *opnd = asm.load(*opnd); + } + }; + } + + // We are replacing instructions here so we know they are already + // being used. It is okay not to use their output here. + #[allow(unused_must_use)] + match &mut insn { + Insn::Add { left, right, out } => { + match (*left, *right) { + // When one operand is a register, legalize the other operand + // into possibly an immdiate and swap the order if necessary. + // Only the rhs of ADD can be an immediate, but addition is commutative. + (reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. }), other_opnd) | + (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. })) => { + *left = reg_opnd; + *right = split_shifted_immediate(asm, other_opnd); + // Now `right` is either a register or an immediate, both can try to + // merge with a subsequent mov. + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, left, out); + asm.push_insn(insn); + } + _ => { + *left = split_load_operand(asm, *left); + *right = split_shifted_immediate(asm, *right); + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, right, out); + asm.push_insn(insn); + } + } + } + Insn::Sub { left, right, out } => { + *left = split_load_operand(asm, *left); + *right = split_shifted_immediate(asm, *right); + // Now `right` is either a register or an immediate, + // both can try to merge with a subsequent mov. + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, left, out); + asm.push_insn(insn); + } + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + let (opnd0, opnd1) = split_boolean_operands(asm, *left, *right); + *left = opnd0; + *right = opnd1; + + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, right, out); + + asm.push_insn(insn); + } + /* + // Lower to Joz and Jonz for generating CBZ/CBNZ for compare-with-0-and-branch. + ref insn @ Insn::Cmp { ref left, right: ref right @ (Opnd::UImm(0) | Opnd::Imm(0)) } | + ref insn @ Insn::Test { ref left, right: ref right @ (Opnd::InsnOut { .. } | Opnd::Reg(_)) } if { + let same_opnd_if_test = if let Insn::Test { .. } = insn { + left == right + } else { + true + }; + + same_opnd_if_test && if let Some( + Insn::Jz(target) | Insn::Je(target) | Insn::Jnz(target) | Insn::Jne(target) + ) = iterator.peek() { + matches!(target, Target::SideExit { .. }) + } else { + false + } + } => { + let reg = split_load_operand(asm, *left); + match iterator.peek() { + Some(Insn::Jz(target) | Insn::Je(target)) => asm.push_insn(Insn::Joz(reg, *target)), + Some(Insn::Jnz(target) | Insn::Jne(target)) => asm.push_insn(Insn::Jonz(reg, *target)), + _ => () + } + + iterator.map_insn_index(asm); + iterator.next_unmapped(); // Pop merged jump instruction + } + */ + Insn::CCall { opnds, .. } => { + assert!(opnds.len() <= C_ARG_OPNDS.len()); + + // Load each operand into the corresponding argument + // register. + // Note: the iteration order is reversed to avoid corrupting x0, + // which is both the return value and first argument register + if !opnds.is_empty() { + let mut args: Vec<(Opnd, Opnd)> = vec![]; + for (idx, opnd) in opnds.iter_mut().enumerate().rev() { + // If the value that we're sending is 0, then we can use + // the zero register, so in this case we'll just send + // a UImm of 0 along as the argument to the move. + let value = match opnd { + Opnd::UImm(0) | Opnd::Imm(0) => Opnd::UImm(0), + Opnd::Mem(_) => split_memory_address(asm, *opnd), + _ => *opnd + }; + args.push((C_ARG_OPNDS[idx], value)); + } + asm.parallel_mov(args); + } + + // Now we push the CCall without any arguments so that it + // just performs the call. + *opnds = vec![]; + asm.push_insn(insn); + }, + Insn::Cmp { left, right } => { + let opnd0 = split_load_operand(asm, *left); + let opnd0 = split_less_than_32_cmp(asm, opnd0); + let split_right = split_shifted_immediate(asm, *right); + let opnd1 = match split_right { + Opnd::VReg { .. } if opnd0.num_bits() != split_right.num_bits() => { + split_right.with_num_bits(opnd0.num_bits().unwrap()) + }, + _ => split_right + }; + + asm.cmp(opnd0, opnd1); + }, + Insn::CRet(opnd) => { + match opnd { + // If the value is already in the return register, then + // we don't need to do anything. + Opnd::Reg(C_RET_REG) => {}, + + // If the value is a memory address, we need to first + // make sure the displacement isn't too large and then + // load it into the return register. + Opnd::Mem(_) => { + let split = split_memory_address(asm, *opnd); + asm.load_into(C_RET_OPND, split); + }, + + // Otherwise we just need to load the value into the + // return register. + _ => { + asm.load_into(C_RET_OPND, *opnd); + } + } + asm.cret(C_RET_OPND); + }, + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } | + Insn::CSelL { truthy, falsy, out } | + Insn::CSelLE { truthy, falsy, out } | + Insn::CSelG { truthy, falsy, out } | + Insn::CSelGE { truthy, falsy, out } => { + let (opnd0, opnd1) = split_csel_operands(asm, *truthy, *falsy); + *truthy = opnd0; + *falsy = opnd1; + // Merge `csel` and `mov` into a single `csel` when possible + match iterator.peek().map(|(_, insn)| insn) { + Some(Insn::Mov { dest: Opnd::Reg(reg), src }) + if matches!(out, Opnd::VReg { .. }) && *out == *src && live_ranges[out.vreg_idx()].end() == index + 1 => { + *out = Opnd::Reg(*reg); + asm.push_insn(insn); + iterator.next(asm); // Pop merged Insn::Mov + } + _ => { + asm.push_insn(insn); + } + } + }, + Insn::JmpOpnd(opnd) => { + if let Opnd::Mem(_) = opnd { + let opnd0 = split_load_operand(asm, *opnd); + asm.jmp_opnd(opnd0); + } else { + asm.jmp_opnd(*opnd); + } + }, + Insn::Load { opnd, .. } | + Insn::LoadInto { opnd, .. } => { + *opnd = match opnd { + Opnd::Mem(_) => split_memory_address(asm, *opnd), + _ => *opnd + }; + asm.push_insn(insn); + }, + Insn::LoadSExt { opnd, out } => { + match opnd { + // We only want to sign extend if the operand is a + // register, instruction output, or memory address that + // is 32 bits. Otherwise we'll just load the value + // directly since there's no need to sign extend. + Opnd::Reg(Reg { num_bits: 32, .. }) | + Opnd::VReg { num_bits: 32, .. } | + Opnd::Mem(Mem { num_bits: 32, .. }) => { + asm.push_insn(insn); + }, + _ => { + asm.push_insn(Insn::Load { opnd: *opnd, out: *out }); + } + }; + }, + Insn::Mov { dest, src } => { + match (&dest, &src) { + // If we're attempting to load into a memory operand, then + // we'll switch over to the store instruction. + (Opnd::Mem(_), _) => { + let opnd0 = split_memory_address(asm, *dest); + let value = match *src { + // If the first operand is zero, then we can just use + // the zero register. + Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG), + // If the first operand is a memory operand, we're going + // to transform this into a store instruction, so we'll + // need to load this anyway. + Opnd::UImm(_) => asm.load(*src), + // The value that is being moved must be either a + // register or an immediate that can be encoded as a + // bitmask immediate. Otherwise, we'll need to split the + // move into multiple instructions. + _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits()) + }; + + asm.store(opnd0, value); + }, + // If we're loading a memory operand into a register, then + // we'll switch over to the load instruction. + (Opnd::Reg(_) | Opnd::VReg { .. }, Opnd::Mem(_)) => { + let value = split_memory_address(asm, *src); + asm.load_into(*dest, value); + }, + // Otherwise we'll use the normal mov instruction. + (Opnd::Reg(_), _) => { + let value = match *src { + // Unlike other instructions, we can avoid splitting this case, using movz. + Opnd::UImm(uimm) if uimm <= 0xffff => *src, + _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits()), + }; + asm.mov(*dest, value); + }, + _ => unreachable!("unexpected combination of operands in Insn::Mov: {dest:?}, {src:?}") + }; + }, + Insn::Not { opnd, .. } => { + // The value that is being negated must be in a register, so + // if we get anything else we need to load it first. + *opnd = match opnd { + Opnd::Mem(_) => split_load_operand(asm, *opnd), + _ => *opnd + }; + asm.push_insn(insn); + }, + Insn::LShift { opnd, .. } | + Insn::RShift { opnd, .. } | + Insn::URShift { opnd, .. } => { + // The operand must be in a register, so + // if we get anything else we need to load it first. + *opnd = split_load_operand(asm, *opnd); + asm.push_insn(insn); + }, + Insn::Mul { left, right, .. } => { + *left = split_load_operand(asm, *left); + *right = split_load_operand(asm, *right); + asm.push_insn(insn); + }, + Insn::Test { left, right } => { + // The value being tested must be in a register, so if it's + // not already one we'll load it first. + let opnd0 = split_load_operand(asm, *left); + + // The second value must be either a register or an + // unsigned immediate that can be encoded as a bitmask + // immediate. If it's not one of those, we'll need to load + // it first. + let opnd1 = split_bitmask_immediate(asm, *right, opnd0.rm_num_bits()); + asm.test(opnd0, opnd1); + }, + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Split instructions using scratch registers. To maximize the use of the register pool for + /// VRegs, most splits should happen in [`Self::arm64_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so this + /// splits them and uses scratch registers for it. + /// Linearizes all blocks into a single giant block. + fn arm64_scratch_split(self) -> Assembler { + /// If opnd is Opnd::Mem with a too large disp, make the disp smaller using lea. + fn split_large_disp(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(Mem { num_bits, disp, .. }) if !mem_disp_fits_bits(disp) => { + asm.lea_into(scratch_opnd, opnd); + Opnd::mem(num_bits, scratch_opnd, 0) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem with MemBase::Stack, lower it to Opnd::Mem with MemBase::Reg, and split a large disp. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + let opnd = split_only_stack_membase(asm, opnd, scratch_opnd, stack_state); + split_large_disp(asm, opnd, scratch_opnd) + } + + /// split_stack_membase but without split_large_disp. This should be used only by lea. + fn split_only_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) = opnd { + let base = Opnd::Mem(stack_state.stack_membase_to_mem(stack_membase)); + let base = split_large_disp(asm, base, scratch_opnd); + asm.load_into(scratch_opnd, base); + Opnd::Mem(Mem { base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), disp: opnd_disp, num_bits: opnd_num_bits }) + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, lower it to scratch_opnd. You should use this when `opnd` is read by the instruction, not written. + fn split_memory_read(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + if let Opnd::Mem(_) = opnd { + let opnd = split_large_disp(asm, opnd, scratch_opnd); + let scratch_opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + asm.load_into(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for (idx, insn) in linearized_insns.iter().enumerate() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } | + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Mul { left, right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + let reg_out = out.clone(); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + }; + + // If the next instruction is JoMul + if idx + 1 < linearized_insns.len() && matches!(linearized_insns[idx + 1], Insn::JoMul(_)) { + // Produce a register that is all zeros or all ones + // Based on the sign bit of the 64-bit mul result + asm.push_insn(Insn::RShift { out: SCRATCH0_OPND, opnd: reg_out, shift: Opnd::UImm(63) }); + } + } + Insn::LShift { opnd, out, .. } | + Insn::RShift { opnd, out, .. } => { + *opnd = split_memory_read(asm, *opnd, SCRATCH0_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + // For compile_exits, support splitting simple return values here + Insn::CRet(opnd) => { + match opnd { + Opnd::Reg(C_RET_REG) => {}, + _ => asm.load_into(C_RET_OPND, *opnd), + } + asm.cret(C_RET_OPND); + } + Insn::Lea { opnd, out } => { + *opnd = split_only_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = split_stack_membase(asm, *out, SCRATCH1_OPND, &stack_state); + + if let Opnd::Mem(_) = out { + // If NATIVE_STACK_PTR is used as a source for Store, it's handled as xzr, storeing zero. + // To save the content of NATIVE_STACK_PTR, we need to load it into another register first. + if *opnd == NATIVE_STACK_PTR { + asm.load_into(SCRATCH0_OPND, NATIVE_STACK_PTR); + *opnd = SCRATCH0_OPND; + } + asm.store(*out, *opnd); + } else { + asm.push_insn(insn); + } + } + &mut Insn::IncrCounter { mem, value } => { + // Convert Opnd::const_ptr into Opnd::Mem. + // It's split here to support IncrCounter in compile_exits. + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.lea_into(SCRATCH0_OPND, Opnd::mem(64, SCRATCH0_OPND, 0)); + + // Create a local loop to atomically increment a counter using SCRATCH1_OPND to check if it succeeded. + // Note that arm64_emit will peek at the next Cmp to set a status into SCRATCH1_OPND on IncrCounter. + let label = asm.new_label("incr_counter_loop"); + asm.write_label(label.clone()); + asm.incr_counter(SCRATCH0_OPND, value); + asm.cmp(SCRATCH1_OPND, 0.into()); + asm.jne(label); + } + Insn::Store { dest, .. } => { + *dest = split_stack_membase(asm, *dest, SCRATCH0_OPND, &stack_state); + asm.push_insn(insn); + } + Insn::Mov { dest, src } => { + *src = split_stack_membase(asm, *src, SCRATCH0_OPND, &stack_state); + *dest = split_large_disp(asm, *dest, SCRATCH1_OPND); + match dest { + Opnd::Reg(_) => asm.load_into(*dest, *src), + Opnd::Mem(_) => asm.store(*dest, *src), + _ => asm.push_insn(insn), + } + } + // Resolve ParallelMov that couldn't be handled without a scratch register. + Insn::ParallelMov { moves } => { + for (dst, src) in Self::resolve_parallel_moves(moves, Some(SCRATCH0_OPND)).unwrap() { + let src = split_stack_membase(asm, src, SCRATCH1_OPND, &stack_state); + let dst = split_large_disp(asm, dst, SCRATCH2_OPND); + match dst { + Opnd::Reg(_) => asm.load_into(dst, src), + Opnd::Mem(_) => asm.store(dst, src), + _ => asm.mov(dst, src), + } + } + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Emit platform-specific machine code + /// Returns a list of GC offsets. Can return failure to signal caller to retry. + fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { + /// Determine how many instructions it will take to represent moving + /// this value into a register. Note that the return value of this + /// function must correspond to how many instructions are used to + /// represent this load in the emit_load_value function. + fn emit_load_size(value: u64) -> u8 { + if BitmaskImmediate::try_from(value).is_ok() { + return 1; + } + + if value < (1 << 16) { + 1 + } else if value < (1 << 32) { + 2 + } else if value < (1 << 48) { + 3 + } else { + 4 + } + } + + /// Emit a conditional jump instruction to a specific target. This is + /// called when lowering any of the conditional jump instructions. + fn emit_conditional_jump<const CONDITION: u8>(asm: &Assembler, cb: &mut CodeBlock, target: Target) { + fn generate_branch<const CONDITION: u8>(cb: &mut CodeBlock, src_addr: i64, dst_addr: i64) { + let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the jump offset fits into the conditional jump as + // an immediate value and it's properly aligned, then we + // can use the b.cond instruction directly. We're safe + // to use as i32 here since we already checked that it + // fits. + let bytes = (dst_addr - src_addr) as i32; + bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); + + // Here we're going to return 1 because we've only + // written out 1 instruction. + 1 + } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond + // If the jump offset fits into the unconditional jump as + // an immediate value, we can use inverse b.cond + b. + // + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instruction used for branching. + bcond(cb, Condition::inverse(CONDITION), 2.into()); + b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond + + // We've only written out 2 instructions. + 2 + } else { + // Otherwise, we need to load the address into a + // register and use the branch register instruction. + let load_insns: i32 = emit_load_size(dst_addr as u64).into(); + + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instructions used for branching. + bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + + // Here we'll return the number of instructions that it + // took to write out the destination address + 1 for the + // b.cond and 1 for the br. + load_insns + 2 + }; + + // We need to make sure we have at least 6 instructions for + // every kind of jump for invalidation purposes, so we're + // going to write out padding nop instructions here. + assert!(num_insns <= cb.conditional_jump_insns()); + (num_insns..cb.conditional_jump_insns()).for_each(|_| nop(cb)); + } + + let label = match target { + Target::CodePtr(dst_ptr) => { + let dst_addr = dst_ptr.as_offset(); + let src_addr = cb.get_write_ptr().as_offset(); + generate_branch::<CONDITION>(cb, src_addr, dst_addr); + return; + }, + Target::Label(l) => l, + Target::Block(ref edge) => asm.block_label(edge.target), + Target::SideExit { .. } => { + unreachable!("Target::SideExit should have been compiled by compile_exits") + }, + }; + // Try to use a single B.cond instruction + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if bcond_offset_fits_bits(offset) { + bcond(cb, CONDITION, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + } + + /// Emit a CBZ or CBNZ which branches when a register is zero or non-zero + fn emit_cmp_zero_jump(cb: &mut CodeBlock, reg: A64Opnd, branch_if_zero: bool, target: Target) { + if let Target::CodePtr(dst_ptr) = target { + let dst_addr = dst_ptr.as_offset(); + let src_addr = cb.get_write_ptr().as_offset(); + + if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the offset fits in one instruction, generate cbz or cbnz + let bytes = (dst_addr - src_addr) as i32; + if branch_if_zero { + cbz(cb, reg, InstructionOffset::from_bytes(bytes)); + } else { + cbnz(cb, reg, InstructionOffset::from_bytes(bytes)); + } + } else { + // Otherwise, we load the address into a register and + // use the branch register instruction. Note that because + // side exits should always be close, this form should be + // rare or impossible to see. + let dst_addr = dst_ptr.raw_addr(cb) as u64; + let load_insns: i32 = emit_load_size(dst_addr).into(); + + // Write out the inverse condition so that if + // it doesn't match it will skip over the + // instructions used for branching. + if branch_if_zero { + cbnz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); + } else { + cbz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); + } + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr); + br(cb, Assembler::EMIT_OPND); + } + } else { + unreachable!("We should only generate Joz/Jonz with side-exit targets"); + } + } + + /// Do the address calculation of `out_reg = base_reg + disp` + fn load_effective_address(cb: &mut CodeBlock, out: A64Opnd, base_reg_no: u8, disp: i32) { + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + assert_ne!(base_reg_no, out_reg_no, "large displacement need a scratch register"); + + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, out, disp as u64); + add_extended(cb, out, base_reg, out); + }; + } + + /// Load a VALUE to a register and remember it for GC marking and reference updating + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest: A64Opnd, value: VALUE) { + // We dont need to check if it's a special const + // here because we only allow these operands to hit + // this point if they're not a special const. + assert!(!value.special_const_p()); + + // This assumes only load instructions can contain + // references to GC'd Value operands. If the value + // being loaded is a heap object, we'll report that + // back out to the gc_offsets list. + ldr_literal(cb, dest, 2.into()); + b(cb, InstructionOffset::from_bytes(4 + (SIZEOF_VALUE as i32))); + cb.write_bytes(&value.as_u64().to_le_bytes()); + + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } + + /// Push a value to the stack by subtracting from the stack pointer then storing, + /// leaving an 8-byte gap for alignment. + fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) { + str_pre(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + } + + /// Pop a value from the stack by loading `[sp]` then adding to the stack pointer. + fn emit_pop(cb: &mut CodeBlock, opnd: A64Opnd) { + ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); + } + + // List of GC offsets + let mut gc_offsets: Vec<CodePtr> = Vec::new(); + + // Buffered list of PosMarker callbacks to fire if codegen is successful + let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // Install a panic hook to dump Assembler with insn_idx on dev builds + let (_hook, mut hook_insn_idx) = AssemblerPanicHook::new(self, 0); + + // For each instruction + // NOTE: At this point, the assembler should have been linearized into a single giant block + // by either resolve_parallel_mov_pass() or arm64_scratch_split(). + let mut insn_idx: usize = 0; + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { + // Update insn_idx that is shown on panic + hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); + + match insn { + Insn::Comment(text) => { + cb.add_comment(text); + }, + Insn::Label(target) => { + cb.write_label(target.unwrap_label()); + }, + // Report back the current position in the generated code + Insn::PosMarker(..) => { + pos_markers.push((insn_idx, cb.get_write_ptr())) + } + Insn::BakeString(text) => { + for byte in text.as_bytes() { + cb.write_byte(*byte); + } + + // Add a null-terminator byte for safety (in case we pass + // this to C code) + cb.write_byte(0); + + // Pad out the string to the next 4-byte boundary so that + // it's easy to jump past. + for _ in 0..(4 - ((text.len() + 1) % 4)) { + cb.write_byte(0); + } + }, + &Insn::FrameSetup { preserved, mut slot_count } => { + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + // Preserve X29 and set up frame record + stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); + mov(cb, X29, C_SP_REG); + + for regs in preserved.chunks(2) { + // For the body, store pairs and move SP + if let [reg0, reg1] = regs { + stp_pre(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, C_SP_REG, -16)); + } else if let [reg] = regs { + // For overhang, store but don't move SP. Combine movement with + // movement for slots below. + stur(cb, reg.into(), A64Opnd::new_mem(64, C_SP_REG, -8)); + slot_count += 1; + } else { + unreachable!("chunks(2)"); + } + } + // Align slot_count + if slot_count % 2 == 1 { + slot_count += 1 + } + if slot_count > 0 { + let slot_offset = (slot_count * SIZEOF_VALUE) as u64; + // Bail when asked to reserve too many slots in one instruction. + ShiftedImmediate::try_from(slot_offset).ok()?; + sub(cb, C_SP_REG, C_SP_REG, A64Opnd::new_uimm(slot_offset)); + } + } + Insn::FrameTeardown { preserved } => { + // Restore preserved registers below frame pointer. + let mut base_offset = 0; + for regs in preserved.chunks(2) { + if let [reg0, reg1] = regs { + base_offset -= 16; + ldp(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, X29, base_offset)); + } else if let [reg] = regs { + ldur(cb, reg.into(), A64Opnd::new_mem(64, X29, base_offset - 8)); + } else { + unreachable!("chunks(2)"); + } + } + + // SP = X29 (frame pointer) + mov(cb, C_SP_REG, X29); + ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); + } + Insn::Add { left, right, out } => { + // Usually, we issue ADDS, so you could branch on overflow, but ADDS with + // out=31 refers to out=XZR, which discards the sum. So, instead of ADDS + // (aliased to CMN in this case) we issue ADD instead which writes the sum + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + add(cb, out, left.into(), right.into()); + } else { + adds(cb, out, left.into(), right.into()); + } + }, + Insn::Sub { left, right, out } => { + // Usually, we issue SUBS, so you could branch on overflow, but SUBS with + // out=31 refers to out=XZR, which discards the result. So, instead of SUBS + // (aliased to CMP in this case) we issue SUB instead which writes the diff + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + sub(cb, out, left.into(), right.into()); + } else { + subs(cb, out, left.into(), right.into()); + } + }, + Insn::Mul { left, right, out } => { + // If the next instruction is JoMul with RShift created by arm64_scratch_split + match (insns.get(insn_idx + 1), insns.get(insn_idx + 2)) { + (Some(Insn::RShift { out: out_sign, opnd: out_opnd, shift: out_shift }), Some(Insn::JoMul(_))) => { + // Compute the high 64 bits + smulh(cb, Self::EMIT_OPND, left.into(), right.into()); + + // Compute the low 64 bits + // This may clobber one of the input registers, + // so we do it after smulh + mul(cb, out.into(), left.into(), right.into()); + + // Insert the shift instruction created by arm64_scratch_split + // to prepare the register that has the sign bit of the high 64 bits after mul. + asr(cb, out_sign.into(), out_opnd.into(), out_shift.into()); + insn_idx += 1; // skip the next Insn::RShift + + // If the high 64-bits are not all zeros or all ones, + // matching the sign bit, then we have an overflow + cmp(cb, Self::EMIT_OPND, out_sign.into()); + // Insn::JoMul will emit_conditional_jump::<{Condition::NE}> + } + _ => { + mul(cb, out.into(), left.into(), right.into()); + } + } + }, + Insn::And { left, right, out } => { + and(cb, out.into(), left.into(), right.into()); + }, + Insn::Or { left, right, out } => { + orr(cb, out.into(), left.into(), right.into()); + }, + Insn::Xor { left, right, out } => { + eor(cb, out.into(), left.into(), right.into()); + }, + Insn::Not { opnd, out } => { + mvn(cb, out.into(), opnd.into()); + }, + Insn::RShift { opnd, shift, out } => { + asr(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::URShift { opnd, shift, out } => { + lsr(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::LShift { opnd, shift, out } => { + lsl(cb, out.into(), opnd.into(), shift.into()); + }, + Insn::Store { dest, src } => { + // Split src into EMIT0_OPND if necessary + let src_reg: A64Reg = match src { + Opnd::Reg(reg) => *reg, + // Use zero register when possible + Opnd::UImm(0) | Opnd::Imm(0) => XZR_REG, + // Immediates + &Opnd::Imm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm as u64); + Self::EMIT_REG + } + &Opnd::UImm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm); + Self::EMIT_REG + } + &Opnd::Value(value) => { + emit_load_gc_value(cb, &mut gc_offsets, Self::EMIT_OPND, value); + Self::EMIT_REG + } + src_mem @ &Opnd::Mem(Mem { num_bits: src_num_bits, base: MemBase::Reg(src_base_reg_no), disp: src_disp }) => { + // For mem-to-mem store, load the source into EMIT0_OPND + let src_mem = if mem_disp_fits_bits(src_disp) { + src_mem.into() + } else { + // Split the load address into EMIT0_OPND first if necessary + load_effective_address(cb, Self::EMIT_OPND, src_base_reg_no, src_disp); + A64Opnd::new_mem(dest.rm_num_bits(), Self::EMIT_OPND, 0) + }; + let dst = A64Opnd::Reg(Self::EMIT_REG.with_num_bits(src_num_bits)); + match src_num_bits { + 64 | 32 => ldur(cb, dst, src_mem), + 16 => ldurh(cb, dst, src_mem), + 8 => ldurb(cb, dst, src_mem), + num_bits => panic!("unexpected num_bits: {num_bits}") + }; + Self::EMIT_REG + } + src @ (Opnd::Mem(_) | Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during arm64_emit: {src:?}") + }; + let src = A64Opnd::Reg(src_reg); + + // This order may be surprising but it is correct. The way + // the Arm64 assembler works, the register that is going to + // be stored is first and the address is second. However in + // our IR we have the address first and the register second. + match dest.rm_num_bits() { + 64 | 32 => stur(cb, src, dest.into()), + 16 => sturh(cb, src, dest.into()), + 8 => sturb(cb, src, dest.into()), + num_bits => panic!("unexpected dest num_bits: {num_bits} (src: {src:?}, dest: {dest:?})"), + } + }, + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + match *opnd { + Opnd::Reg(_) | Opnd::VReg { .. } => { + mov(cb, out.into(), opnd.into()); + }, + Opnd::UImm(uimm) => { + emit_load_value(cb, out.into(), uimm); + }, + Opnd::Imm(imm) => { + emit_load_value(cb, out.into(), imm as u64); + }, + Opnd::Mem(_) => { + match opnd.rm_num_bits() { + 64 | 32 => ldur(cb, out.into(), opnd.into()), + 16 => ldurh(cb, out.into(), opnd.into()), + 8 => ldurb(cb, out.into(), opnd.into()), + num_bits => panic!("unexpected num_bits: {num_bits}"), + }; + }, + Opnd::Value(value) => { + emit_load_gc_value(cb, &mut gc_offsets, out.into(), value); + }, + Opnd::None => { + unreachable!("Attempted to load from None operand"); + } + }; + }, + Insn::LoadSExt { opnd, out } => { + match *opnd { + Opnd::Reg(Reg { num_bits: 32, .. }) | + Opnd::VReg { num_bits: 32, .. } => { + sxtw(cb, out.into(), opnd.into()); + }, + Opnd::Mem(Mem { num_bits: 32, .. }) => { + ldursw(cb, out.into(), opnd.into()); + }, + _ => unreachable!() + }; + }, + Insn::ParallelMov { .. } => unreachable!("{insn:?} should have been lowered at alloc_regs()"), + Insn::Mov { dest, src } => { + // This supports the following two kinds of immediates: + // * The value fits into a single movz instruction + // * It can be encoded with the special bitmask immediate encoding + // arm64_split() should have split other immediates that require multiple instructions. + match src { + Opnd::UImm(uimm) if *uimm <= 0xffff => { + movz(cb, dest.into(), A64Opnd::new_uimm(*uimm), 0); + }, + _ => { + mov(cb, dest.into(), src.into()); + } + } + }, + Insn::Lea { opnd, out } => { + let &Opnd::Mem(Mem { num_bits: _, base: MemBase::Reg(base_reg_no), disp }) = opnd else { + panic!("Unexpected Insn::Lea operand in arm64_emit: {opnd:?}"); + }; + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + + let out = A64Opnd::from(out); + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use a scratch reg for `out += displacement` + let disp_reg = if out_reg_no == base_reg_no { + Self::EMIT_OPND + } else { + out + }; + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, disp_reg, disp as u64); + add_extended(cb, out, base_reg, disp_reg); + } + } + Insn::LeaJumpTarget { out, target, .. } => { + if let Target::Label(label_idx) = target { + // Set output to the raw address of the label + cb.label_ref(*label_idx, 4, |cb, end_addr, dst_addr| { + adr(cb, Self::EMIT_OPND, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + Ok(()) + }); + + mov(cb, out.into(), Self::EMIT_OPND); + } else { + // Set output to the jump target's raw address + let target_code = target.unwrap_code_ptr(); + let target_addr = target_code.raw_addr(cb).as_u64(); + emit_load_value(cb, out.into(), target_addr); + } + }, + Insn::CPush(opnd) => { + emit_push(cb, opnd.into()); + }, + Insn::CPushPair(opnd0, opnd1) => { + // Second operand ends up at the lower stack address + stp_pre(cb, opnd1.into(), opnd0.into(), A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + }, + Insn::CPop { out } => { + emit_pop(cb, out.into()); + }, + Insn::CPopInto(opnd) => { + emit_pop(cb, opnd.into()); + }, + Insn::CPopPairInto(opnd0, opnd1) => { + // First operand is popped from the lower stack address + ldp_post(cb, opnd0.into(), opnd1.into(), A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); + }, + Insn::CCall { fptr, .. } => { + match fptr { + Opnd::UImm(fptr) => { + // The offset to the call target in bytes + let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; + let dst_addr = *fptr as i64; + + // Use BL if the offset is short enough to encode as an immediate. + // Otherwise, use BLR with a register. + if b_offset_fits_bits((dst_addr - src_addr) / 4) { + bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + } else { + emit_load_value(cb, Self::EMIT_OPND, dst_addr as u64); + blr(cb, Self::EMIT_OPND); + } + } + Opnd::Reg(_) => { + blr(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } + }, + Insn::CRet { .. } => { + ret(cb, A64Opnd::None); + }, + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); + }, + Insn::Test { left, right } => { + tst(cb, left.into(), right.into()); + }, + Insn::JmpOpnd(opnd) => { + br(cb, opnd.into()); + }, + Insn::Jmp(target) => { + match *target { + Target::CodePtr(dst_ptr) => { + emit_jmp_ptr(cb, dst_ptr, true); + }, + Target::Label(label_idx) => { + // Reserve space for a single B instruction + cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::Block(ref edge) => { + let label = self.block_label(edge.target); + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::SideExit { .. } => { + unreachable!("Target::SideExit should have been compiled by compile_exits") + }, + }; + }, + Insn::Je(target) | Insn::Jz(target) => { + emit_conditional_jump::<{Condition::EQ}>(self, cb, target.clone()); + }, + Insn::Jne(target) | Insn::Jnz(target) | Insn::JoMul(target) => { + emit_conditional_jump::<{Condition::NE}>(self, cb, target.clone()); + }, + Insn::Jl(target) => { + emit_conditional_jump::<{Condition::LT}>(self, cb, target.clone()); + }, + Insn::Jg(target) => { + emit_conditional_jump::<{Condition::GT}>(self, cb, target.clone()); + }, + Insn::Jge(target) => { + emit_conditional_jump::<{Condition::GE}>(self, cb, target.clone()); + }, + Insn::Jbe(target) => { + emit_conditional_jump::<{Condition::LS}>(self, cb, target.clone()); + }, + Insn::Jb(target) => { + emit_conditional_jump::<{Condition::CC}>(self, cb, target.clone()); + }, + Insn::Jo(target) => { + emit_conditional_jump::<{Condition::VS}>(self, cb, target.clone()); + }, + Insn::Joz(opnd, target) => { + emit_cmp_zero_jump(cb, opnd.into(), true, target.clone()); + }, + Insn::Jonz(opnd, target) => { + emit_cmp_zero_jump(cb, opnd.into(), false, target.clone()); + }, + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in arm64_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + while cb.get_write_pos().saturating_sub(last_patch_pos) < cb.jmp_ptr_bytes() && !cb.has_dropped_bytes() { + nop(cb); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + Insn::IncrCounter { mem, value } => { + // Get the status register allocated by arm64_scratch_split + let Some(Insn::Cmp { + left: status_reg @ Opnd::Reg(_), + right: Opnd::UImm(_) | Opnd::Imm(_), + }) = insns.get(insn_idx + 1) else { + panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", insns.get(insn_idx + 1)); + }; + + // Attempt to increment a counter + ldaxr(cb, Self::EMIT_OPND, mem.into()); + add(cb, Self::EMIT_OPND, Self::EMIT_OPND, value.into()); + + // The status register that gets used to track whether or + // not the store was successful must be 32 bytes. Since we + // store the EMIT registers as their 64-bit versions, we + // need to rewrap it here. + let status = A64Opnd::Reg(status_reg.unwrap_reg().with_num_bits(32)); + stlxr(cb, status, Self::EMIT_OPND, mem.into()); + }, + Insn::Breakpoint => { + brk(cb, A64Opnd::None); + }, + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::EQ); + }, + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::NE); + }, + Insn::CSelL { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LT); + }, + Insn::CSelLE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LE); + }, + Insn::CSelG { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GT); + }, + Insn::CSelGE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); + } + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + }; + + insn_idx += 1; + } + + // Error if we couldn't write out everything + if cb.has_dropped_bytes() { + None + } else { + // No bytes dropped, so the pos markers point to valid code + for (insn_idx, pos) in pos_markers { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { + callback(pos, cb); + } else { + panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); + } + } + + Some(gc_offsets) + } + } + + /// Optimize and compile the stored instructions + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_reg = !self.accept_scratch_reg; + asm_dump!(self, init); + + let asm = self.arm64_split(); + asm_dump!(asm, split); + + let mut asm = asm.alloc_regs(regs)?; + asm_dump!(asm, alloc_regs); + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + asm.compile_exits(); + asm_dump!(asm, compile_exits); + + if use_scratch_reg { + asm = asm.arm64_scratch_split(); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = asm.resolve_parallel_mov_pass(); + asm_dump!(asm, resolve_parallel_mov); + } + + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.arm64_emit(cb); + + if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + + // Invalidate icache for newly written out region so we don't run stale code. + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + + Ok((start_ptr, gc_offsets)) + } else { + cb.clear_labels(); + Err(CompileError::OutOfMemory) + } + } +} + +/// LIR Instructions that are lowered to an instruction that have 2 input registers and an output +/// register can look to merge with a succeeding `Insn::Mov`. +/// For example: +/// +/// Add out, a, b +/// Mov c, out +/// +/// Can become: +/// +/// Add c, a, b +/// +/// If a, b, and c are all registers. +fn merge_three_reg_mov( + live_ranges: &[LiveRange], + iterator: &mut InsnIter, + asm: &mut Assembler, + left: &Opnd, + right: &Opnd, + out: &mut Opnd, +) { + if let (Opnd::Reg(_) | Opnd::VReg{..}, + Opnd::Reg(_) | Opnd::VReg{..}, + Some((mov_idx, Insn::Mov { dest, src }))) + = (left, right, iterator.peek()) { + if out == src && live_ranges[out.vreg_idx()].end() == *mov_idx && matches!(*dest, Opnd::Reg(_) | Opnd::VReg{..}) { + *out = *dest; + iterator.next(asm); // Pop merged Insn::Mov + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "disasm")] + use crate::disasms_with; + use crate::{assert_disasm_snapshot, hexdumps}; + + use super::*; + use insta::assert_snapshot; + use crate::hir; + + static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; + + fn setup_asm() -> (Assembler, CodeBlock) { + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id(); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.parallel_mov(vec![(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)), (C_ARG_OPNDS[1], Opnd::mem(64, SP, -8))]); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.je(label); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @r" + bb0: + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, x19, x21, x20 + v0 = Add x19, 0x40 + Store [x21 + 0x10], v0 + Joz Exit(Interrupt), v0 + ParallelMov x0 <- w0, x1 <- [x21 - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[x20 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown x19, x21, x20 + "); + } + + #[test] + fn test_mul_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.mul(Opnd::Reg(TEMP_REGS[1]), 3.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #3 + 0x4: mul x0, x9, x0 + 0x8: mov x1, x0 + "); + assert_snapshot!(cb.hexdump(), @"600080d2207d009be10300aa"); + } + + #[test] + fn test_conditional_branch_to_label() { + let (mut asm, mut cb) = setup_asm(); + let start = asm.new_label("start"); + let forward = asm.new_label("forward"); + + let value = asm.load(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0)); + asm.write_label(start.clone()); + asm.cmp(value, 0.into()); + asm.jg(forward.clone()); + asm.jl(start.clone()); + asm.write_label(forward); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [sp] + 0x4: cmp x0, #0 + 0x8: b.gt #0x10 + 0xc: b.lt #4 + "); + assert_snapshot!(cb.hexdump(), @"e00340f81f0000f14c000054cbffff54"); + } + + #[test] + fn sp_movements_are_single_instruction() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + let new_sp = asm.add(sp, 0x20.into()); + asm.mov(sp, new_sp); + let new_sp = asm.sub(sp, 0x20.into()); + asm.mov(sp, new_sp); + + asm.compile_with_num_regs(&mut cb, 2); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #0x20 + 0x4: sub sp, sp, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff830091ff8300d1"); + } + + #[test] + fn add_into() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + asm.add_into(sp, 8.into()); + asm.add_into(Opnd::Reg(X20_REG), 0x20.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #8 + 0x4: adds x20, x20, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff230091948200b1"); + } + + #[test] + fn sub_imm_reg() { + let (mut asm, mut cb) = setup_asm(); + + let difference = asm.sub(0x8.into(), Opnd::Reg(X5_REG)); + asm.load_into(Opnd::Reg(X1_REG), difference); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #8 + 0x4: subs x0, x0, x5 + 0x8: mov x1, x0 + "); + assert_snapshot!(cb.hexdump(), @"000180d2000005ebe10300aa"); + } + + #[test] + fn no_dead_mov_from_vreg() { + let (mut asm, mut cb) = setup_asm(); + + let ret_val = asm.load(Opnd::mem(64, C_RET_OPND, 0)); + asm.cret(ret_val); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [x0] + 0x4: ret + "); + assert_snapshot!(cb.hexdump(), @"000040f8c0035fd6"); + } + + #[test] + fn test_emit_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.add(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_regs(&mut cb, vec![X3_REG]).unwrap(); + + // Assert that only 2 instructions were written. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: adds x3, x0, x1 + 0x4: stur x3, [x2] + "); + assert_snapshot!(cb.hexdump(), @"030001ab430000f8"); + } + + #[test] + fn test_emit_bake_string() { + let (mut asm, mut cb) = setup_asm(); + + asm.bake_string("Hello, world!"); + asm.compile_with_num_regs(&mut cb, 0); + + // Testing that we pad the string to the nearest 4-byte boundary to make + // it easier to jump over. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldnp d8, d25, [x10, #-0x140] + 0x4: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x8: .byte 0x6f, 0x72, 0x6c, 0x64 + 0xc: .byte 0x21, 0x00, 0x00, 0x00 + "); + assert_snapshot!(cb.hexdump(), @"48656c6c6f2c20776f726c6421000000"); + } + + #[test] + fn test_emit_frame() { + let (mut asm, mut cb) = setup_asm(); + + asm.frame_setup(&[]); + asm.frame_teardown(&[]); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: mov sp, x29 + 0xc: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"fd7bbfa9fd030091bf030091fd7bc1a8"); + } + + #[test] + fn frame_setup_and_teardown() { + const THREE_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; + // Test 3 preserved regs (odd), odd slot_count + let cb1 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 3 preserved regs (odd), even slot_count + let cb2 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 4; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 4 preserved regs (even), odd slot_count + let cb3 = { + static FOUR_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(FOUR_REGS); + asm.frame_teardown(FOUR_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + assert_disasm_snapshot!(disasms_with!("\n", cb1, cb2, cb3), @r" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x30 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stp x22, x21, [sp, #-0x10]! + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldp x22, x21, [x29, #-0x20] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(hexdumps!(cb1, cb2, cb3), @r" + fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8 + "); + } + + #[test] + fn test_emit_je_fits_into_bcond() { + let (mut asm, mut cb) = setup_asm(); + + let target: CodePtr = cb.get_write_ptr().add_bytes(80); + + asm.je(Target::CodePtr(target)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: b.eq #0x50 + 0x4: nop + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"800200541f2003d51f2003d51f2003d51f2003d51f2003d5"); + } + + #[test] + fn test_emit_je_does_not_fit_into_bcond() { + let (mut asm, mut cb) = setup_asm(); + + let offset = 1 << 21; + let target: CodePtr = cb.get_write_ptr().add_bytes(offset); + + asm.je(Target::CodePtr(target)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: b.ne #8 + 0x4: b #0x200000 + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"41000054ffff07141f2003d51f2003d51f2003d51f2003d5"); + } + + #[test] + fn test_emit_lea() { + let (mut asm, mut cb) = setup_asm(); + + // Test values that exercise various types of immediates. + // - 9 bit displacement for Load/Store + // - 12 bit ADD/SUB shifted immediate + // - 16 bit MOV family shifted immediates + // - bit mask immediates + for displacement in [i32::MAX, 0x10008, 0x1800, 0x208, -0x208, -0x1800, -0x10008, i32::MIN] { + let mem = Opnd::mem(64, NATIVE_STACK_PTR, displacement); + asm.lea_into(Opnd::Reg(X0_REG), mem); + } + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, xzr, #0x7fffffff + 0x4: add x0, sp, x0 + 0x8: mov x0, #8 + 0xc: movk x0, #1, lsl #16 + 0x10: add x0, sp, x0 + 0x14: mov x0, #0x1800 + 0x18: add x0, sp, x0 + 0x1c: add x0, sp, #0x208 + 0x20: sub x0, sp, #0x208 + 0x24: mov x0, #-0x1800 + 0x28: add x0, sp, x0 + 0x2c: mov x0, #0xfff8 + 0x30: movk x0, #0xfffe, lsl #16 + 0x34: movk x0, #0xffff, lsl #32 + 0x38: movk x0, #0xffff, lsl #48 + 0x3c: add x0, sp, x0 + 0x40: orr x0, xzr, #0xffffffff80000000 + 0x44: add x0, sp, x0 + "); + assert_snapshot!(cb.hexdump(), @"e07b40b2e063208b000180d22000a0f2e063208b000083d2e063208be0230891e02308d1e0ff8292e063208b00ff9fd2c0ffbff2e0ffdff2e0fffff2e063208be08361b2e063208b"); + } + + #[test] + fn test_load_larg_disp_mem() { + let (mut asm, mut cb) = setup_asm(); + + let extended_ivars = asm.load(Opnd::mem(64, NATIVE_STACK_PTR, 0)); + let result = asm.load(Opnd::mem(VALUE_BITS, extended_ivars, 1000 * SIZEOF_VALUE_I32)); + asm.store(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0), result); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [sp] + 0x4: mov x16, #0x1f40 + 0x8: add x0, x0, x16, uxtx + 0xc: ldur x0, [x0] + 0x10: stur x0, [sp] + "); + assert_snapshot!(cb.hexdump(), @"e00340f810e883d20060308b000040f8e00300f8"); + } + + #[test] + fn test_store() { + let (mut asm, mut cb) = setup_asm(); + + // Large memory offsets in combinations of destination and source + let large_mem = Opnd::mem(64, NATIVE_STACK_PTR, -0x305); + let small_mem = Opnd::mem(64, C_RET_OPND, 0); + asm.store(small_mem, large_mem); + asm.store(large_mem, small_mem); + asm.store(large_mem, large_mem); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: sub x16, sp, #0x305 + 0x4: ldur x16, [x16] + 0x8: stur x16, [x0] + 0xc: sub x15, sp, #0x305 + 0x10: ldur x16, [x0] + 0x14: stur x16, [x15] + 0x18: sub x15, sp, #0x305 + 0x1c: sub x16, sp, #0x305 + 0x20: ldur x16, [x16] + 0x24: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"f0170cd1100240f8100000f8ef170cd1100040f8f00100f8ef170cd1f0170cd1100240f8f00100f8"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + // Side exit code are compiled without the split pass, so we directly call emit here to + // emulate that scenario. + let gc_offsets = asm.arm64_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldr x16, #8 + 0x4: b #0x10 + 0x8: .byte 0x00, 0x10, 0x00, 0x00 + 0xc: .byte 0x00, 0x00, 0x00, 0x00 + 0x10: stur x16, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000058030000140010000000000000b00200f8"); + } + + #[test] + fn test_store_with_valid_scratch_reg() { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); + let mut cb = CodeBlock::new_dummy(); + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x3902 + 0x4: movk x16, #8, lsl #16 + 0x8: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"502087d21001a0f2f00100f8"); + } + + #[test] + #[should_panic] + fn test_store_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + } + + #[test] + #[should_panic] + fn test_load_into_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.load_into(scratch_reg, 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + } + + #[test] + fn test_emit_lea_label() { + let (mut asm, mut cb) = setup_asm(); + + let label = asm.new_label("label"); + let opnd = asm.lea_jump_target(label.clone()); + + asm.write_label(label); + asm.bake_string("Hello, world!"); + asm.store(Opnd::mem(64, SP, 0), opnd); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: adr x16, #8 + 0x4: mov x0, x16 + 0x8: ldnp d8, d25, [x10, #-0x140] + 0xc: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x10: .byte 0x6f, 0x72, 0x6c, 0x64 + 0x14: .byte 0x21, 0x00, 0x00, 0x00 + 0x18: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000010e00310aa48656c6c6f2c20776f726c6421000000a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_fits_into_load() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 0)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that two instructions were written: LDUR and STUR. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [x21] + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a00240f8a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_fits_into_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 1 << 10)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that three instructions were written: ADD, LDUR, and STUR. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x21, #0x400 + 0x4: ldur x0, [x0] + 0x8: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a0021091000040f8a00200f8"); + } + + #[test] + fn test_emit_load_mem_disp_does_not_fit_into_add() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::mem(64, SP, 1 << 12 | 1)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that three instructions were written: MOVZ, ADD, LDUR, and STUR. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #0x1001 + 0x4: add x0, x21, x0, uxtx + 0x8: ldur x0, [x0] + 0xc: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"200082d2a062208b000040f8a00200f8"); + } + + #[test] + fn test_emit_load_value_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::Value(Qnil)); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that only two instructions were written since the value is an + // immediate. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #4 + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"800080d2a00200f8"); + } + + #[test] + fn test_emit_load_value_non_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.load(Opnd::Value(VALUE(0xCAFECAFECAFE0000))); + asm.store(Opnd::mem(64, SP, 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that five instructions were written since the value is not an + // immediate and needs to be loaded into a register. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldr x0, #8 + 0x4: b #0x10 + 0x8: eon x0, x0, x30, ror #0 + 0xc: eon x30, x23, x30, ror #50 + 0x10: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"40000058030000140000fecafecafecaa00200f8"); + } + + #[test] + fn test_emit_test_32b_reg_not_bitmask_imm() { + let (mut asm, mut cb) = setup_asm(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); + asm.test(w0, Opnd::UImm(u32::MAX.into())); + // All ones is not encodable with a bitmask immediate, + // so this needs one register + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, xzr, #0xffffffff + 0x4: tst w0, w0 + "); + assert_snapshot!(cb.hexdump(), @"e07f40b21f00006a"); + } + + #[test] + fn test_emit_test_32b_reg_bitmask_imm() { + let (mut asm, mut cb) = setup_asm(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); + asm.test(w0, Opnd::UImm(0x80000001)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"1f040172"); + } + + #[test] + fn test_emit_or() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.or(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001aa400000f8"); + } + + #[test] + fn test_emit_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.lshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: lsl x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00e87bd3400000f8"); + } + + #[test] + fn test_emit_rshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.rshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: asr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc4593400000f8"); + } + + #[test] + fn test_emit_urshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.urshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: lsr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc45d3400000f8"); + } + + #[test] + fn test_emit_test() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); + } + + #[test] + fn test_emit_test_with_encodable_unsigned_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::UImm(7)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); + } + + #[test] + fn test_emit_test_with_unencodable_unsigned_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::UImm(5)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a load and a test instruction were written. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); + } + + #[test] + fn test_emit_test_with_encodable_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(7)); + asm.compile_with_num_regs(&mut cb, 0); + + // Assert that only one instruction was written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); + } + + #[test] + fn test_emit_test_with_unencodable_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(5)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a load and a test instruction were written. + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); + } + + #[test] + fn test_emit_test_with_negative_signed_immediate() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(X0_REG), Opnd::Imm(-7)); + asm.compile_with_num_regs(&mut cb, 1); + + // Assert that a test instruction is written. + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #-7"); + assert_snapshot!(cb.hexdump(), @"1ff47df2"); + } + + #[test] + fn test_32_bit_register_with_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); + asm.cmp(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur w0, [x0, #6] + 0x4: mov x1, #0x1001 + 0x8: cmp w0, w1 + "); + assert_snapshot!(cb.hexdump(), @"006040b8210082d21f00016b"); + } + + #[test] + fn test_16_bit_register_store_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(16, Opnd::Reg(X0_REG), 0); + asm.store(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x1001 + 0x4: sturh w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"300082d210000078"); + } + + #[test] + fn test_32_bit_register_store_some_number() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); + asm.store(shape_opnd, Opnd::UImm(4097)); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x1001 + 0x4: stur w16, [x0, #6] + "); + assert_snapshot!(cb.hexdump(), @"300082d2106000b8"); + } + + #[test] + fn test_emit_xor() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = asm.xor(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); + asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); + + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: eor x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001ca400000f8"); + } + + #[test] + #[cfg(feature = "disasm")] + fn test_simple_disasm() -> std::result::Result<(), capstone::Error> { + // Test drive Capstone with simple input + use capstone::prelude::*; + + let cs = Capstone::new() + .arm64() + .mode(arch::arm64::ArchMode::Arm) + .build()?; + + let insns = cs.disasm_all(&[0x60, 0x0f, 0x80, 0xF2], 0x1000)?; + + match insns.as_ref() { + [insn] => { + assert_eq!(Some("movk"), insn.mnemonic()); + Ok(()) + } + _ => Err(capstone::Error::CustomError( + "expected to disassemble to movk", + )), + } + } + + #[test] + fn test_replace_mov_with_ldur() { + let (mut asm, mut cb) = setup_asm(); + + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::mem(64, CFP, 8)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x1, [x19, #8]"); + assert_snapshot!(cb.hexdump(), @"618240f8"); + } + + #[test] + fn test_not_split_mov() { + let (mut asm, mut cb) = setup_asm(); + + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0xffff)); + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0x10000)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x1, #0xffff + 0x4: orr x1, xzr, #0x10000 + "); + assert_snapshot!(cb.hexdump(), @"e1ff9fd2e10370b2"); + } + + #[test] + fn test_merge_csel_mov() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.csel_l(Qtrue.into(), Qfalse.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x14 + 0x4: mov x1, #0 + 0x8: csel x1, x0, x1, lt + "); + assert_snapshot!(cb.hexdump(), @"800280d2010080d201b0819a"); + } + + #[test] + fn test_exceeding_label_branch_generate_bounds() { + // The immediate in a conditional branch is a 19 bit unsigned integer + // which has a max value of 2^18 - 1. + const IMMEDIATE_MAX_VALUE: usize = 2usize.pow(18) - 1; + + // `IMMEDIATE_MAX_VALUE` number of dummy instructions will be generated + // plus a compare, a jump instruction, and a label. + // Adding page_size to avoid OOM on the last page. + let page_size = unsafe { rb_jit_get_page_size() } as usize; + let memory_required = (IMMEDIATE_MAX_VALUE + 8) * 4 + page_size; + + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id(); + let mut cb = CodeBlock::new_dummy_sized(memory_required); + + let far_label = asm.new_label("far"); + + asm.cmp(Opnd::Reg(X0_REG), Opnd::UImm(1)); + asm.je(far_label.clone()); + + (0..IMMEDIATE_MAX_VALUE).for_each(|_| { + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::Reg(TEMP_REGS[2])); + }); + + asm.write_label(far_label.clone()); + assert_eq!(Err(CompileError::LabelLinkingFailure), asm.compile(&mut cb)); + } + + #[test] + fn test_add_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.add(Opnd::Reg(TEMP_REGS[1]), 1.into()); + let out = asm.add(out, 1_usize.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adds x0, x9, #1 + 0x4: adds x1, x0, #1 + "); + assert_snapshot!(cb.hexdump(), @"200500b1010400b1"); + } + + #[test] + fn test_store_spilled_byte() { + let (mut asm, mut cb) = setup_asm(); + + asm.store(Opnd::mem(8, C_RET_OPND, 0), Opnd::mem(8, C_RET_OPND, 8)); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldurb w16, [x0, #8] + 0x4: sturb w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"1080403810000038"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_no_cycle() { + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov x0, x0 (optimized away) + C_ARG_OPNDS[1], // mov x1, x1 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0 + 0x4: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_single_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[2], // mov x2, x2 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x15 + 0xc: mov x16, #0 + 0x10: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_two_cycles() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle, and x2 and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[3], // mov x2, rcx + C_ARG_OPNDS[2], // mov rcx, x2 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x2 + 0x4: mov x2, x3 + 0x8: mov x3, x15 + 0xc: mov x15, x0 + 0x10: mov x0, x1 + 0x14: mov x1, x15 + 0x18: mov x16, #0 + 0x1c: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0302aae20303aae3030faaef0300aae00301aae1030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, x0 + 0x14: stp x2, x1, [sp, #-0x10]! + 0x18: stp x4, x3, [sp, #-0x10]! + 0x1c: mov x16, #0 + 0x20: blr x16 + 0x24: ldp x4, x3, [sp], #0x10 + 0x28: ldp x2, x1, [sp], #0x10 + 0x2c: adds x4, x4, x1 + 0x30: adds x2, x2, x3 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2e40300aae207bfa9e40fbfa9100080d200023fd6e40fc1a8e207c1a8840001ab420003ab"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, #5 + 0x14: mov x5, x0 + 0x18: stp x2, x1, [sp, #-0x10]! + 0x1c: stp x4, x3, [sp, #-0x10]! + 0x20: str x5, [sp, #-0x10]! + 0x24: mov x16, #0 + 0x28: blr x16 + 0x2c: ldr x5, [sp], #0x10 + 0x30: ldp x4, x3, [sp], #0x10 + 0x34: ldp x2, x1, [sp], #0x10 + 0x38: adds x5, x5, x1 + 0x3c: adds x0, x2, x3 + 0x40: adds x2, x2, x4 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2a40080d2e50300aae207bfa9e40fbfa9e50f1ff8100080d200023fd6e50741f8e40fc1a8e207c1a8a50001ab400003ab420004ab"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_large_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0, x1, and x2 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[2], // mov x1, x2 + C_ARG_OPNDS[0], // mov x2, x0 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x2 + 0xc: mov x2, x15 + 0x10: mov x16, #0 + 0x14: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae10302aae2030faa100080d200023fd6"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: stp x1, x0, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e103bfa9"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: ldp x0, x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e007c1a8"); + } + + #[test] + fn test_split_spilled_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd_vreg = asm.load(1.into()); + let out_vreg = asm.lshift(opnd_vreg, Opnd::UImm(1)); + asm.mov(C_RET_OPND, out_vreg); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #1 + 0x4: stur x16, [x29, #-8] + 0x8: ldur x15, [x29, #-8] + 0xc: lsl x15, x15, #1 + 0x10: stur x15, [x29, #-8] + 0x14: ldur x0, [x29, #-8] + "); + assert_snapshot!(cb.hexdump(), @"300080d2b0831ff8af835ff8eff97fd3af831ff8a0835ff8"); + } + + #[test] + fn test_split_load16_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(16, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldurh w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"0000089100004078"); + } + + #[test] + fn test_split_load32_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(32, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldur w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040b8"); + } + + #[test] + fn test_split_load64_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(64, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldur x0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040f8"); + } +} diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs new file mode 100644 index 0000000000..f2f7bc6165 --- /dev/null +++ b/zjit/src/backend/lir.rs @@ -0,0 +1,3126 @@ +use std::collections::HashMap; +use std::fmt; +use std::mem::take; +use std::panic; +use std::rc::Rc; +use std::sync::{Arc, Mutex}; +use crate::codegen::local_size_and_idx_to_ep_offset; +use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::hir::{Invariant, SideExitReason}; +use crate::hir; +use crate::options::{TraceExits, debug, get_option}; +use crate::cruby::VALUE; +use crate::payload::IseqVersionRef; +use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError}; +use crate::virtualmem::CodePtr; +use crate::asm::{CodeBlock, Label}; +use crate::state::rb_zjit_record_exit_stack; + +/// LIR Block ID. Unique ID for each block, and also defined in LIR so +/// we can differentiate it from HIR block ids. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "l{}", self.0) + } +} + +/// Dummy HIR block ID used when creating test or invalid LIR blocks +const DUMMY_HIR_BLOCK_ID: usize = usize::MAX; +/// Dummy RPO index used when creating test or invalid LIR blocks +const DUMMY_RPO_INDEX: usize = usize::MAX; + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<Opnd>, +} + +#[derive(Clone, Debug)] +pub struct BasicBlock { + // Unique id for this block + pub id: BlockId, + + // HIR block this LIR block was lowered from. Not injective: multiple LIR blocks may share + // the same hir_block_id because we split HIR blocks into multiple LIR blocks during lowering. + pub hir_block_id: hir::BlockId, + + pub is_entry: bool, + + // Instructions in this basic block + pub insns: Vec<Insn>, + + // Input parameters for this block + pub parameters: Vec<Opnd>, + + // RPO position of the source HIR block + pub rpo_index: usize, +} + +pub struct EdgePair(Option<BranchEdge>, Option<BranchEdge>); + +impl BasicBlock { + fn new(id: BlockId, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> Self { + Self { + id, + hir_block_id, + is_entry, + insns: vec![], + parameters: vec![], + rpo_index, + } + } + + pub fn add_parameter(&mut self, param: Opnd) { + self.parameters.push(param); + } + + pub fn push_insn(&mut self, insn: Insn) { + self.insns.push(insn); + } + + pub fn edges(&self) -> EdgePair { + assert!(self.insns.last().unwrap().is_terminator()); + let extract_edge = |insn: &Insn| -> Option<BranchEdge> { + if let Some(Target::Block(edge)) = insn.target() { + Some(edge.clone()) + } else { + None + } + }; + + match self.insns.as_slice() { + [] => panic!("empty block"), + [.., second_last, last] => { + EdgePair(extract_edge(second_last), extract_edge(last)) + }, + [.., last] => { + EdgePair(extract_edge(last), None) + } + } + } + + /// Sort key for scheduling blocks in code layout order + pub fn sort_key(&self) -> (usize, usize) { + (self.rpo_index, self.id.0) + } +} + +pub use crate::backend::current::{ + mem_base_reg, + Reg, + EC, CFP, SP, + NATIVE_STACK_PTR, NATIVE_BASE_PTR, + C_ARG_OPNDS, C_RET_REG, C_RET_OPND, +}; + +pub static JIT_PRESERVED_REGS: &[Opnd] = &[CFP, SP, EC]; + +// Memory operand base +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum MemBase +{ + /// Register: Every Opnd::Mem should have MemBase::Reg as of emit. + Reg(u8), + /// Virtual register: Lowered to MemBase::Reg or MemBase::Stack in alloc_regs. + VReg(usize), + /// Stack slot: Lowered to MemBase::Reg in scratch_split. + Stack { stack_idx: usize, num_bits: u8 }, +} + +// Memory location +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Mem +{ + // Base register number or instruction index + pub base: MemBase, + + // Offset relative to the base pointer + pub disp: i32, + + // Size in bits + pub num_bits: u8, +} + +impl fmt::Display for Mem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.num_bits != 64 { + write!(f, "Mem{}", self.num_bits)?; + } + write!(f, "[")?; + match self.base { + MemBase::Reg(reg_no) => write!(f, "{}", mem_base_reg(reg_no))?, + MemBase::VReg(idx) => write!(f, "v{idx}")?, + MemBase::Stack { stack_idx, num_bits } if num_bits == 64 => write!(f, "Stack[{stack_idx}]")?, + MemBase::Stack { stack_idx, num_bits } => write!(f, "Stack{num_bits}[{stack_idx}]")?, + } + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(f, " {sign} ")?; + if self.disp.abs() >= 10 { + write!(f, "0x")?; + } + write!(f, "{:x}", self.disp.abs())?; + } + write!(f, "]") + } +} + +impl fmt::Debug for Mem { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Mem{}[{:?}", self.num_bits, self.base)?; + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(fmt, " {sign} {}", self.disp.abs())?; + } + + write!(fmt, "]") + } +} + +/// Operand to an IR instruction +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum Opnd +{ + None, // For insns with no output + + // Immediate Ruby value, may be GC'd, movable + Value(VALUE), + + /// Virtual register. Lowered to Reg or Mem in Assembler::alloc_regs(). + VReg{ idx: usize, num_bits: u8 }, + + // Low-level operands, for lowering + Imm(i64), // Raw signed immediate + UImm(u64), // Raw unsigned immediate + Mem(Mem), // Memory location + Reg(Reg), // Machine register +} + +impl fmt::Display for Opnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Opnd::*; + match self { + None => write!(f, "None"), + Value(VALUE(value)) if *value < 10 => write!(f, "Value({value:x})"), + Value(VALUE(value)) => write!(f, "Value(0x{value:x})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(f, "v{idx}"), + VReg { idx, num_bits } => write!(f, "VReg{num_bits}(v{idx})"), + Imm(value) if value.abs() < 10 => write!(f, "Imm({value:x})"), + Imm(value) => write!(f, "Imm(0x{value:x})"), + UImm(value) if *value < 10 => write!(f, "{value:x}"), + UImm(value) => write!(f, "0x{value:x}"), + Mem(mem) => write!(f, "{mem}"), + Reg(reg) => write!(f, "{reg}"), + } + } +} + +impl fmt::Debug for Opnd { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + use Opnd::*; + match self { + Self::None => write!(fmt, "None"), + Value(val) => write!(fmt, "Value({val:?})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(fmt, "VReg({idx})"), + VReg { idx, num_bits } => write!(fmt, "VReg{num_bits}({idx})"), + Imm(signed) => write!(fmt, "{signed:x}_i64"), + UImm(unsigned) => write!(fmt, "{unsigned:x}_u64"), + // Say Mem and Reg only once + Mem(mem) => write!(fmt, "{mem:?}"), + Reg(reg) => write!(fmt, "{reg:?}"), + } + } +} + +impl Opnd +{ + /// Convenience constructor for memory operands + pub fn mem(num_bits: u8, base: Opnd, disp: i32) -> Self { + match base { + Opnd::Reg(base_reg) => { + assert!(base_reg.num_bits == 64); + Opnd::Mem(Mem { + base: MemBase::Reg(base_reg.reg_no), + disp, + num_bits, + }) + }, + + Opnd::VReg{idx, num_bits: out_num_bits } => { + assert!(num_bits <= out_num_bits); + Opnd::Mem(Mem { + base: MemBase::VReg(idx), + disp, + num_bits, + }) + }, + + _ => unreachable!("memory operand with non-register base: {base:?}") + } + } + + /// Constructor for constant pointer operand + pub fn const_ptr<T>(ptr: *const T) -> Self { + Opnd::UImm(ptr as u64) + } + + /// Unwrap a register operand + pub fn unwrap_reg(&self) -> Reg { + match self { + Opnd::Reg(reg) => *reg, + _ => unreachable!("trying to unwrap {:?} into reg", self) + } + } + + /// Unwrap the index of a VReg + pub fn vreg_idx(&self) -> usize { + match self { + Opnd::VReg { idx, .. } => *idx, + _ => unreachable!("trying to unwrap {self:?} into VReg"), + } + } + + /// Get the size in bits for this operand if there is one. + pub fn num_bits(&self) -> Option<u8> { + match *self { + Opnd::Reg(Reg { num_bits, .. }) => Some(num_bits), + Opnd::Mem(Mem { num_bits, .. }) => Some(num_bits), + Opnd::VReg { num_bits, .. } => Some(num_bits), + _ => None + } + } + + /// Return Opnd with a given num_bits if self has num_bits. Panic otherwise. + #[track_caller] + pub fn with_num_bits(&self, num_bits: u8) -> Opnd { + assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); + match *self { + Opnd::Reg(reg) => Opnd::Reg(reg.with_num_bits(num_bits)), + Opnd::Mem(Mem { base, disp, .. }) => Opnd::Mem(Mem { base, disp, num_bits }), + Opnd::VReg { idx, .. } => Opnd::VReg { idx, num_bits }, + _ => unreachable!("with_num_bits should not be used for: {self:?}"), + } + } + + /// Get the size in bits for register/memory operands. + pub fn rm_num_bits(&self) -> u8 { + self.num_bits().unwrap() + } + + /// Maps the indices from a previous list of instructions to a new list of + /// instructions. + pub fn map_index(self, indices: &[usize]) -> Opnd { + match self { + Opnd::VReg { idx, num_bits } => { + Opnd::VReg { idx: indices[idx], num_bits } + } + Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { + Opnd::Mem(Mem { base: MemBase::VReg(indices[idx]), disp, num_bits }) + }, + _ => self + } + } + + /// When there aren't any operands to check against, this is the number of + /// bits that should be used for any given output variable. + const DEFAULT_NUM_BITS: u8 = 64; + + /// Determine the size in bits from the iterator of operands. If any of them + /// are different sizes this will panic. + pub fn match_num_bits_iter<'a>(opnds: impl Iterator<Item = &'a Opnd>) -> u8 { + let mut value: Option<u8> = None; + + for opnd in opnds { + if let Some(num_bits) = opnd.num_bits() { + match value { + None => { + value = Some(num_bits); + }, + Some(value) => { + assert_eq!(value, num_bits, "operands of incompatible sizes"); + } + }; + } + } + + value.unwrap_or(Self::DEFAULT_NUM_BITS) + } + + /// Determine the size in bits of the slice of the given operands. If any of + /// them are different sizes this will panic. + pub fn match_num_bits(opnds: &[Opnd]) -> u8 { + Self::match_num_bits_iter(opnds.iter()) + } +} + +impl From<usize> for Opnd { + fn from(value: usize) -> Self { + Opnd::UImm(value.try_into().unwrap()) + } +} + +impl From<u64> for Opnd { + fn from(value: u64) -> Self { + Opnd::UImm(value) + } +} + +impl From<i64> for Opnd { + fn from(value: i64) -> Self { + Opnd::Imm(value) + } +} + +impl From<i32> for Opnd { + fn from(value: i32) -> Self { + Opnd::Imm(value.into()) + } +} + +impl From<u32> for Opnd { + fn from(value: u32) -> Self { + Opnd::UImm(value as u64) + } +} + +impl From<VALUE> for Opnd { + fn from(value: VALUE) -> Self { + Opnd::Value(value) + } +} + +/// Context for a side exit. If `SideExit` matches, it reuses the same code. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExit { + pub pc: Opnd, + pub stack: Vec<Opnd>, + pub locals: Vec<Opnd>, +} + +/// Branch target (something that we can jump to) +/// for branch instructions +#[derive(Clone)] +pub enum Target +{ + /// Pointer to a piece of ZJIT-generated code + CodePtr(CodePtr), + /// A label within the generated code + Label(Label), + /// An LIR branch edge + Block(BranchEdge), + /// Side exit to the interpreter + SideExit { + /// Context used for compiling the side exit + exit: SideExit, + /// We use this to increment exit counters + reason: SideExitReason, + }, +} + +impl fmt::Debug for Target { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Target::CodePtr(ptr) => write!(f, "CodePtr({:?})", ptr), + Target::Label(label) => write!(f, "Label({:?})", label), + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, "Block({:?})", edge.target) + } else { + write!(f, "Block({:?}(", edge.target)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}", arg)?; + } + write!(f, "))") + } + } + Target::SideExit { exit, reason } => { + write!(f, "SideExit {{ exit: {:?}, reason: {:?} }}", exit, reason) + } + } + } +} + +impl Target +{ + pub fn unwrap_label(&self) -> Label { + match self { + Target::Label(label) => *label, + _ => unreachable!("trying to unwrap {:?} into label", self) + } + } + + pub fn unwrap_code_ptr(&self) -> CodePtr { + match self { + Target::CodePtr(ptr) => *ptr, + _ => unreachable!("trying to unwrap {:?} into code ptr", self) + } + } +} + +impl From<CodePtr> for Target { + fn from(code_ptr: CodePtr) -> Self { + Target::CodePtr(code_ptr) + } +} + +type PosMarkerFn = Rc<dyn Fn(CodePtr, &CodeBlock)>; + +/// ZJIT Low-level IR instruction +#[derive(Clone)] +pub enum Insn { + /// Add two operands together, and return the result as a new operand. + Add { left: Opnd, right: Opnd, out: Opnd }, + + /// This is the same as the OP_ADD instruction, except that it performs the + /// binary AND operation. + And { left: Opnd, right: Opnd, out: Opnd }, + + /// Bake a string directly into the instruction stream. + BakeString(String), + + // Trigger a debugger breakpoint + #[allow(dead_code)] + Breakpoint, + + /// Add a comment into the IR at the point that this instruction is added. + /// It won't have any impact on that actual compiled code. + Comment(String), + + /// Compare two operands + Cmp { left: Opnd, right: Opnd }, + + /// Pop a register from the C stack + CPop { out: Opnd }, + + /// Pop a register from the C stack and store it into another register + CPopInto(Opnd), + + /// Pop a pair of registers from the C stack and store it into a pair of registers. + /// The registers are popped from left to right. + CPopPairInto(Opnd, Opnd), + + /// Push a register onto the C stack + CPush(Opnd), + + /// Push a pair of registers onto the C stack. + /// The registers are pushed from left to right. + CPushPair(Opnd, Opnd), + + // C function call with N arguments (variadic) + CCall { + opnds: Vec<Opnd>, + /// The function pointer to be called. This should be Opnd::const_ptr + /// (Opnd::UImm) in most cases. gen_entry_trampoline() uses Opnd::Reg. + fptr: Opnd, + /// Optional PosMarker to remember the start address of the C call. + /// It's embedded here to insert the PosMarker after push instructions + /// that are split from this CCall on alloc_regs(). + start_marker: Option<PosMarkerFn>, + /// Optional PosMarker to remember the end address of the C call. + /// It's embedded here to insert the PosMarker before pop instructions + /// that are split from this CCall on alloc_regs(). + end_marker: Option<PosMarkerFn>, + out: Opnd, + }, + + // C function return + CRet(Opnd), + + /// Conditionally select if equal + CSelE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater + CSelG { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater or equal + CSelGE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if less + CSelL { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if less or equal + CSelLE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if not equal + CSelNE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if not zero + CSelNZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if zero + CSelZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Set up the frame stack as necessary per the architecture. + FrameSetup { preserved: &'static [Opnd], slot_count: usize }, + + /// Tear down the frame stack as necessary per the architecture. + FrameTeardown { preserved: &'static [Opnd], }, + + // Atomically increment a counter + // Input: memory operand, increment value + // Produces no output + IncrCounter { mem: Opnd, value: Opnd }, + + /// Jump if below or equal (unsigned) + Jbe(Target), + + /// Jump if below (unsigned) + Jb(Target), + + /// Jump if equal + Je(Target), + + /// Jump if lower + Jl(Target), + + /// Jump if greater + Jg(Target), + + /// Jump if greater or equal + Jge(Target), + + // Unconditional jump to a branch target + Jmp(Target), + + // Unconditional jump which takes a reg/mem address operand + JmpOpnd(Opnd), + + /// Jump if not equal + Jne(Target), + + /// Jump if not zero + Jnz(Target), + + /// Jump if overflow + Jo(Target), + + /// Jump if overflow in multiplication + JoMul(Target), + + /// Jump if zero + Jz(Target), + + /// Jump if operand is zero (only used during lowering at the moment) + Joz(Opnd, Target), + + /// Jump if operand is non-zero (only used during lowering at the moment) + Jonz(Opnd, Target), + + // Add a label into the IR at the point that this instruction is added. + Label(Target), + + /// Get the code address of a jump target + LeaJumpTarget { target: Target, out: Opnd }, + + // Load effective address + Lea { opnd: Opnd, out: Opnd }, + + /// Take a specific register. Signal the register allocator to not use it. + LiveReg { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register. + Load { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a specified register. + LoadInto { dest: Opnd, opnd: Opnd }, + + // A low-level instruction that loads a value into a register and + // sign-extends it to a 64-bit value. + LoadSExt { opnd: Opnd, out: Opnd }, + + /// Shift a value left by a certain amount. + LShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + /// A set of parallel moves into registers or memory. + /// The backend breaks cycles if there are any cycles between moves. + ParallelMov { moves: Vec<(Opnd, Opnd)> }, + + // A low-level mov instruction. It accepts two operands. + Mov { dest: Opnd, src: Opnd }, + + // Perform the NOT operation on an individual operand, and return the result + // as a new operand. This operand can then be used as the operand on another + // instruction. + Not { opnd: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary OR operation. + Or { left: Opnd, right: Opnd, out: Opnd }, + + /// Patch point that will be rewritten to a jump to a side exit on invalidation. + PatchPoint { target: Target, invariant: Invariant, version: IseqVersionRef }, + + /// Make sure the last PatchPoint has enough space to insert a jump. + /// We insert this instruction at the end of each block so that the jump + /// will not overwrite the next block or a side exit. + PadPatchPoint, + + // Mark a position in the generated code + PosMarker(PosMarkerFn), + + /// Shift a value right by a certain amount (signed). + RShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // Low-level instruction to store a value to memory. + Store { dest: Opnd, src: Opnd }, + + // This is the same as the add instruction, except for subtraction. + Sub { left: Opnd, right: Opnd, out: Opnd }, + + // Integer multiplication + Mul { left: Opnd, right: Opnd, out: Opnd }, + + // Bitwise AND test instruction + Test { left: Opnd, right: Opnd }, + + /// Shift a value right by a certain amount (unsigned). + URShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary XOR operation. + Xor { left: Opnd, right: Opnd, out: Opnd } +} + +impl Insn { + /// Create an iterator that will yield a non-mutable reference to each + /// operand in turn for this instruction. + pub(super) fn opnd_iter(&self) -> InsnOpndIterator<'_> { + InsnOpndIterator::new(self) + } + + /// Create an iterator that will yield a mutable reference to each operand + /// in turn for this instruction. + pub(super) fn opnd_iter_mut(&mut self) -> InsnOpndMutIterator<'_> { + InsnOpndMutIterator::new(self) + } + + /// Get a mutable reference to a Target if it exists. + pub(super) fn target_mut(&mut self) -> Option<&mut Target> { + match self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Joz(_, target) | + Insn::Jonz(_, target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + Some(target) + } + _ => None, + } + } + + /// Returns a string that describes which operation this instruction is + /// performing. This is used for debugging. + fn op(&self) -> &'static str { + match self { + Insn::Add { .. } => "Add", + Insn::And { .. } => "And", + Insn::BakeString(_) => "BakeString", + Insn::Breakpoint => "Breakpoint", + Insn::Comment(_) => "Comment", + Insn::Cmp { .. } => "Cmp", + Insn::CPop { .. } => "CPop", + Insn::CPopInto(_) => "CPopInto", + Insn::CPopPairInto(_, _) => "CPopPairInto", + Insn::CPush(_) => "CPush", + Insn::CPushPair(_, _) => "CPushPair", + Insn::CCall { .. } => "CCall", + Insn::CRet(_) => "CRet", + Insn::CSelE { .. } => "CSelE", + Insn::CSelG { .. } => "CSelG", + Insn::CSelGE { .. } => "CSelGE", + Insn::CSelL { .. } => "CSelL", + Insn::CSelLE { .. } => "CSelLE", + Insn::CSelNE { .. } => "CSelNE", + Insn::CSelNZ { .. } => "CSelNZ", + Insn::CSelZ { .. } => "CSelZ", + Insn::FrameSetup { .. } => "FrameSetup", + Insn::FrameTeardown { .. } => "FrameTeardown", + Insn::IncrCounter { .. } => "IncrCounter", + Insn::Jbe(_) => "Jbe", + Insn::Jb(_) => "Jb", + Insn::Je(_) => "Je", + Insn::Jl(_) => "Jl", + Insn::Jg(_) => "Jg", + Insn::Jge(_) => "Jge", + Insn::Jmp(_) => "Jmp", + Insn::JmpOpnd(_) => "JmpOpnd", + Insn::Jne(_) => "Jne", + Insn::Jnz(_) => "Jnz", + Insn::Jo(_) => "Jo", + Insn::JoMul(_) => "JoMul", + Insn::Jz(_) => "Jz", + Insn::Joz(..) => "Joz", + Insn::Jonz(..) => "Jonz", + Insn::Label(_) => "Label", + Insn::LeaJumpTarget { .. } => "LeaJumpTarget", + Insn::Lea { .. } => "Lea", + Insn::LiveReg { .. } => "LiveReg", + Insn::Load { .. } => "Load", + Insn::LoadInto { .. } => "LoadInto", + Insn::LoadSExt { .. } => "LoadSExt", + Insn::LShift { .. } => "LShift", + Insn::ParallelMov { .. } => "ParallelMov", + Insn::Mov { .. } => "Mov", + Insn::Not { .. } => "Not", + Insn::Or { .. } => "Or", + Insn::PatchPoint { .. } => "PatchPoint", + Insn::PadPatchPoint => "PadPatchPoint", + Insn::PosMarker(_) => "PosMarker", + Insn::RShift { .. } => "RShift", + Insn::Store { .. } => "Store", + Insn::Sub { .. } => "Sub", + Insn::Mul { .. } => "Mul", + Insn::Test { .. } => "Test", + Insn::URShift { .. } => "URShift", + Insn::Xor { .. } => "Xor" + } + } + + /// Return a non-mutable reference to the out operand for this instruction + /// if it has one. + pub fn out_opnd(&self) -> Option<&Opnd> { + match self { + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaJumpTarget { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::Mul { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Return a mutable reference to the out operand for this instruction if it + /// has one. + pub fn out_opnd_mut(&mut self) -> Option<&mut Opnd> { + match self { + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaJumpTarget { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::Mul { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Returns the target for this instruction if there is one. + pub fn target(&self) -> Option<&Target> { + match self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Joz(_, target) | + Insn::Jonz(_, target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => Some(target), + _ => None + } + } + + /// Returns the text associated with this instruction if there is some. + pub fn text(&self) -> Option<&String> { + match self { + Insn::BakeString(text) | + Insn::Comment(text) => Some(text), + _ => None + } + } + + /// Returns true if this instruction is a terminator (ends a basic block). + pub fn is_terminator(&self) -> bool { + match self { + Insn::Jbe(_) | + Insn::Jb(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jg(_) | + Insn::Jge(_) | + Insn::Jmp(_) | + Insn::JmpOpnd(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::JoMul(_) | + Insn::Jz(_) | + Insn::Joz(..) | + Insn::Jonz(..) | + Insn::CRet(_) => true, + _ => false + } + } +} + +/// An iterator that will yield a non-mutable reference to each operand in turn +/// for the given instruction. +pub(super) struct InsnOpndIterator<'a> { + insn: &'a Insn, + idx: usize, +} + +impl<'a> InsnOpndIterator<'a> { + fn new(insn: &'a Insn) -> Self { + Self { insn, idx: 0 } + } +} + +impl<'a> Iterator for InsnOpndIterator<'a> { + type Item = &'a Opnd; + + fn next(&mut self) -> Option<Self::Item> { + match self.insn { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + if self.idx < edge.args.len() { + let opnd = &edge.args[self.idx]; + self.idx += 1; + return Some(opnd); + } + None + } + _ => None + } + } + + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + if self.idx == 0 { + self.idx += 1; + return Some(opnd); + } + + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + let arg_idx = self.idx - 1; + if arg_idx < edge.args.len() { + let opnd = &edge.args[arg_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + _ => None + } + } + + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::PadPatchPoint | + Insn::PosMarker(_) => None, + + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { + match self.idx { + 0 => { + self.idx += 1; + Some(opnd) + }, + _ => None + } + }, + Insn::Add { left: opnd0, right: opnd1, .. } | + Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | + Insn::Cmp { left: opnd0, right: opnd1 } | + Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | + Insn::LoadInto { dest: opnd0, opnd: opnd1 } | + Insn::LShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Mov { dest: opnd0, src: opnd1 } | + Insn::Or { left: opnd0, right: opnd1, .. } | + Insn::RShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Store { dest: opnd0, src: opnd1 } | + Insn::Sub { left: opnd0, right: opnd1, .. } | + Insn::Mul { left: opnd0, right: opnd1, .. } | + Insn::Test { left: opnd0, right: opnd1 } | + Insn::URShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Xor { left: opnd0, right: opnd1, .. } => { + match self.idx { + 0 => { + self.idx += 1; + Some(opnd0) + } + 1 => { + self.idx += 1; + Some(opnd1) + } + _ => None + } + }, + Insn::CCall { opnds, .. } => { + if self.idx < opnds.len() { + let opnd = &opnds[self.idx]; + self.idx += 1; + Some(opnd) + } else { + None + } + }, + Insn::ParallelMov { moves } => { + if self.idx < moves.len() * 2 { + let move_idx = self.idx / 2; + let opnd = if self.idx % 2 == 0 { + &moves[move_idx].0 + } else { + &moves[move_idx].1 + }; + self.idx += 1; + Some(opnd) + } else { + None + } + }, + Insn::FrameSetup { preserved, .. } | + Insn::FrameTeardown { preserved } => { + if self.idx < preserved.len() { + let opnd = &preserved[self.idx]; + self.idx += 1; + Some(opnd) + } else { + None + } + } + } + } +} + +/// An iterator that will yield each operand in turn for the given instruction. +pub(super) struct InsnOpndMutIterator<'a> { + insn: &'a mut Insn, + idx: usize, +} + +impl<'a> InsnOpndMutIterator<'a> { + fn new(insn: &'a mut Insn) -> Self { + Self { insn, idx: 0 } + } + + pub(super) fn next(&mut self) -> Option<&mut Opnd> { + match self.insn { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + if self.idx < edge.args.len() { + let opnd = &mut edge.args[self.idx]; + self.idx += 1; + return Some(opnd); + } + None + } + _ => None + } + } + + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + if self.idx == 0 { + self.idx += 1; + return Some(opnd); + } + + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + let arg_idx = self.idx - 1; + if arg_idx < edge.args.len() { + let opnd = &mut edge.args[arg_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + _ => None + } + } + + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::FrameSetup { .. } | + Insn::FrameTeardown { .. } | + Insn::PadPatchPoint | + Insn::PosMarker(_) => None, + + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { + match self.idx { + 0 => { + self.idx += 1; + Some(opnd) + }, + _ => None + } + }, + Insn::Add { left: opnd0, right: opnd1, .. } | + Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | + Insn::Cmp { left: opnd0, right: opnd1 } | + Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | + Insn::LoadInto { dest: opnd0, opnd: opnd1 } | + Insn::LShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Mov { dest: opnd0, src: opnd1 } | + Insn::Or { left: opnd0, right: opnd1, .. } | + Insn::RShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Store { dest: opnd0, src: opnd1 } | + Insn::Sub { left: opnd0, right: opnd1, .. } | + Insn::Mul { left: opnd0, right: opnd1, .. } | + Insn::Test { left: opnd0, right: opnd1 } | + Insn::URShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Xor { left: opnd0, right: opnd1, .. } => { + match self.idx { + 0 => { + self.idx += 1; + Some(opnd0) + } + 1 => { + self.idx += 1; + Some(opnd1) + } + _ => None + } + }, + Insn::CCall { opnds, .. } => { + if self.idx < opnds.len() { + let opnd = &mut opnds[self.idx]; + self.idx += 1; + Some(opnd) + } else { + None + } + }, + Insn::ParallelMov { moves } => { + if self.idx < moves.len() * 2 { + let move_idx = self.idx / 2; + let opnd = if self.idx % 2 == 0 { + &mut moves[move_idx].0 + } else { + &mut moves[move_idx].1 + }; + self.idx += 1; + Some(opnd) + } else { + None + } + }, + } + } +} + +impl fmt::Debug for Insn { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "{}(", self.op())?; + + // Print list of operands + let mut opnd_iter = self.opnd_iter(); + if let Insn::FrameSetup { slot_count, .. } = self { + write!(fmt, "{slot_count}")?; + } + if let Some(first_opnd) = opnd_iter.next() { + write!(fmt, "{first_opnd:?}")?; + } + for opnd in opnd_iter { + write!(fmt, ", {opnd:?}")?; + } + write!(fmt, ")")?; + + // Print text, target, and pos if they are present + if let Some(text) = self.text() { + write!(fmt, " {text:?}")? + } + if let Some(target) = self.target() { + write!(fmt, " target={target:?}")?; + } + + write!(fmt, " -> {:?}", self.out_opnd().unwrap_or(&Opnd::None)) + } +} + +/// Live range of a VReg +/// TODO: Consider supporting lifetime holes +#[derive(Clone, Debug, PartialEq)] +pub struct LiveRange { + /// Index of the first instruction that used the VReg (inclusive) + pub start: Option<usize>, + /// Index of the last instruction that used the VReg (inclusive) + pub end: Option<usize>, +} + +impl LiveRange { + /// Shorthand for self.start.unwrap() + pub fn start(&self) -> usize { + self.start.unwrap() + } + + /// Shorthand for self.end.unwrap() + pub fn end(&self) -> usize { + self.end.unwrap() + } +} + +/// StackState manages which stack slots are used by which VReg +pub struct StackState { + /// The maximum number of spilled VRegs at a time + stack_size: usize, + /// Map from index at the C stack for spilled VRegs to Some(vreg_idx) if allocated + stack_slots: Vec<Option<usize>>, + /// Copy of Assembler::stack_base_idx. Used for calculating stack slot offsets. + stack_base_idx: usize, +} + +impl StackState { + /// Initialize a stack allocator + pub(super) fn new(stack_base_idx: usize) -> Self { + StackState { + stack_size: 0, + stack_slots: vec![], + stack_base_idx, + } + } + + /// Allocate a stack slot for a given vreg_idx + fn alloc_stack(&mut self, vreg_idx: usize) -> Opnd { + for stack_idx in 0..self.stack_size { + if self.stack_slots[stack_idx].is_none() { + self.stack_slots[stack_idx] = Some(vreg_idx); + return Opnd::mem(64, NATIVE_BASE_PTR, self.stack_idx_to_disp(stack_idx)); + } + } + // Every stack slot is in use. Allocate a new stack slot. + self.stack_size += 1; + self.stack_slots.push(Some(vreg_idx)); + Opnd::mem(64, NATIVE_BASE_PTR, self.stack_idx_to_disp(self.stack_slots.len() - 1)) + } + + /// Deallocate a stack slot for a given disp + fn dealloc_stack(&mut self, disp: i32) { + let stack_idx = self.disp_to_stack_idx(disp); + if self.stack_slots[stack_idx].is_some() { + self.stack_slots[stack_idx] = None; + } + } + + /// Convert the `disp` of a stack slot operand to the stack index + fn disp_to_stack_idx(&self, disp: i32) -> usize { + (-disp / SIZEOF_VALUE_I32) as usize - self.stack_base_idx - 1 + } + + /// Convert a stack index to the `disp` of the stack slot + fn stack_idx_to_disp(&self, stack_idx: usize) -> i32 { + (self.stack_base_idx + stack_idx + 1) as i32 * -SIZEOF_VALUE_I32 + } + + /// Convert Mem to MemBase::Stack + fn mem_to_stack_membase(&self, mem: Mem) -> MemBase { + match mem { + Mem { base: MemBase::Reg(reg_no), disp, num_bits } if NATIVE_BASE_PTR.unwrap_reg().reg_no == reg_no => { + let stack_idx = self.disp_to_stack_idx(disp); + MemBase::Stack { stack_idx, num_bits } + } + _ => unreachable!(), + } + } + + /// Convert MemBase::Stack to Mem + pub(super) fn stack_membase_to_mem(&self, membase: MemBase) -> Mem { + match membase { + MemBase::Stack { stack_idx, num_bits } => { + let disp = self.stack_idx_to_disp(stack_idx); + Mem { base: MemBase::Reg(NATIVE_BASE_PTR.unwrap_reg().reg_no), disp, num_bits } + } + _ => unreachable!(), + } + } +} + +/// RegisterPool manages which registers are used by which VReg +struct RegisterPool { + /// List of registers that can be allocated + regs: Vec<Reg>, + + /// Some(vreg_idx) if the register at the index in `pool` is used by the VReg. + /// None if the register is not in use. + pool: Vec<Option<usize>>, + + /// The number of live registers. + /// Provides a quick way to query `pool.filter(|r| r.is_some()).count()` + live_regs: usize, + + /// Fallback to let StackState allocate stack slots when RegisterPool runs out of registers. + stack_state: StackState, +} + +impl RegisterPool { + /// Initialize a register pool + fn new(regs: Vec<Reg>, stack_base_idx: usize) -> Self { + let pool = vec![None; regs.len()]; + RegisterPool { + regs, + pool, + live_regs: 0, + stack_state: StackState::new(stack_base_idx), + } + } + + /// Mutate the pool to indicate that the register at the index + /// has been allocated and is live. + fn alloc_opnd(&mut self, vreg_idx: usize) -> Opnd { + for (reg_idx, reg) in self.regs.iter().enumerate() { + if self.pool[reg_idx].is_none() { + self.pool[reg_idx] = Some(vreg_idx); + self.live_regs += 1; + return Opnd::Reg(*reg); + } + } + self.stack_state.alloc_stack(vreg_idx) + } + + /// Allocate a specific register + fn take_reg(&mut self, reg: &Reg, vreg_idx: usize) -> Opnd { + let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) + .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); + assert_eq!(self.pool[reg_idx], None, "register already allocated for VReg({:?})", self.pool[reg_idx]); + self.pool[reg_idx] = Some(vreg_idx); + self.live_regs += 1; + Opnd::Reg(*reg) + } + + // Mutate the pool to indicate that the given register is being returned + // as it is no longer used by the instruction that previously held it. + fn dealloc_opnd(&mut self, opnd: &Opnd) { + if let Opnd::Mem(Mem { disp, .. }) = *opnd { + return self.stack_state.dealloc_stack(disp); + } + + let reg = opnd.unwrap_reg(); + let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) + .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); + if self.pool[reg_idx].is_some() { + self.pool[reg_idx] = None; + self.live_regs -= 1; + } + } + + /// Return a list of (Reg, vreg_idx) tuples for all live registers + fn live_regs(&self) -> Vec<(Reg, usize)> { + let mut live_regs = Vec::with_capacity(self.live_regs); + for (reg_idx, ®) in self.regs.iter().enumerate() { + if let Some(vreg_idx) = self.pool[reg_idx] { + live_regs.push((reg, vreg_idx)); + } + } + live_regs + } + + /// Return vreg_idx if a given register is already in use + fn vreg_for(&self, reg: &Reg) -> Option<usize> { + let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no).unwrap(); + self.pool[reg_idx] + } + + /// Return true if no register is in use + fn is_empty(&self) -> bool { + self.live_regs == 0 + } +} + +/// Initial capacity for asm.insns vector +const ASSEMBLER_INSNS_CAPACITY: usize = 256; + +/// Object into which we assemble instructions to be +/// optimized and lowered +#[derive(Clone)] +pub struct Assembler { + pub basic_blocks: Vec<BasicBlock>, + + /// The block to which new instructions are added. Used during HIR to LIR lowering to + /// determine which LIR block we should add instructions to. Set by `set_current_block()` + /// and automatically set to new entry blocks created by `new_block()`. + current_block_id: BlockId, + + /// Live range for each VReg indexed by its `idx`` + pub(super) live_ranges: Vec<LiveRange>, + + /// Names of labels + pub(super) label_names: Vec<String>, + + /// If true, `push_insn` is allowed to use scratch registers. + /// On `compile`, it also disables the backend's use of them. + pub(super) accept_scratch_reg: bool, + + /// The Assembler can use NATIVE_BASE_PTR + stack_base_idx as the + /// first stack slot in case it needs to allocate memory. This is + /// equal to the number of spilled basic block arguments. + pub(super) stack_base_idx: usize, + + /// If Some, the next ccall should verify its leafness + leaf_ccall_stack_size: Option<usize>, + + /// Current instruction index, incremented for each instruction pushed + idx: usize, +} + +impl Assembler +{ + /// Create an Assembler with defaults + pub fn new() -> Self { + Self { + label_names: Vec::default(), + accept_scratch_reg: false, + stack_base_idx: 0, + leaf_ccall_stack_size: None, + basic_blocks: Vec::default(), + current_block_id: BlockId(0), + live_ranges: Vec::default(), + idx: 0, + } + } + + /// Create an Assembler, reserving a specified number of stack slots + pub fn new_with_stack_slots(stack_base_idx: usize) -> Self { + Self { stack_base_idx, ..Self::new() } + } + + /// Create an Assembler that allows the use of scratch registers. + /// This should be called only through [`Self::new_with_scratch_reg`]. + pub(super) fn new_with_accept_scratch_reg(accept_scratch_reg: bool) -> Self { + Self { accept_scratch_reg, ..Self::new() } + } + + /// Create an Assembler with parameters of another Assembler and empty instructions. + /// Compiler passes build a next Assembler with this API and insert new instructions to it. + pub(super) fn new_with_asm(old_asm: &Assembler) -> Self { + let mut asm = Self { + label_names: old_asm.label_names.clone(), + accept_scratch_reg: old_asm.accept_scratch_reg, + stack_base_idx: old_asm.stack_base_idx, + ..Self::new() + }; + + // Initialize basic blocks from the old assembler, preserving hir_block_id and entry flag + // but with empty instruction lists + for old_block in &old_asm.basic_blocks { + asm.new_block_from_old_block(&old_block); + } + + // Initialize live_ranges to match the old assembler's size + // This allows reusing VRegs from the old assembler + asm.live_ranges.resize(old_asm.live_ranges.len(), LiveRange { start: None, end: None }); + + asm + } + + // Create a new LIR basic block. Returns the newly created block ID + pub fn new_block(&mut self, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, hir_block_id, is_entry, rpo_index); + self.basic_blocks.push(lir_bb); + if is_entry { + self.set_current_block(bb_id); + } + bb_id + } + + // Create a new LIR basic block from an old one. This should only be used + // when creating new assemblers during passes when we want to translate + // one assembler to a new one. + pub fn new_block_from_old_block(&mut self, old_block: &BasicBlock) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, old_block.hir_block_id, old_block.is_entry, old_block.rpo_index); + self.basic_blocks.push(lir_bb); + bb_id + } + + // Create a LIR basic block without a valid HIR block ID (for testing or internal use). + pub fn new_block_without_id(&mut self) -> BlockId { + self.new_block(hir::BlockId(DUMMY_HIR_BLOCK_ID), true, DUMMY_RPO_INDEX) + } + + pub fn set_current_block(&mut self, block_id: BlockId) { + self.current_block_id = block_id; + } + + pub fn current_block(&mut self) -> &mut BasicBlock { + &mut self.basic_blocks[self.current_block_id.0] + } + + /// Return basic blocks sorted by RPO index, then by block ID. + /// TODO: Use a more advanced scheduling algorithm + pub fn sorted_blocks(&self) -> Vec<&BasicBlock> { + let mut sorted: Vec<&BasicBlock> = self.basic_blocks.iter().collect(); + sorted.sort_by_key(|block| block.sort_key()); + sorted + } + + /// Return true if `opnd` is or depends on `reg` + pub fn has_reg(opnd: Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(opnd_reg) => opnd_reg == reg, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => reg_no == reg.reg_no, + _ => false, + } + } + + pub fn instruction_iterator(&mut self) -> InsnIter { + let mut blocks = take(&mut self.basic_blocks); + blocks.sort_by_key(|block| block.sort_key()); + + let mut iter = InsnIter { + blocks, + current_block_idx: 0, + current_insn_iter: vec![].into_iter(), // Will be replaced immediately + peeked: None, + index: 0, + }; + + // Set up first block's iterator + if !iter.blocks.is_empty() { + iter.current_insn_iter = take(&mut iter.blocks[0].insns).into_iter(); + } + + iter + } + + /// Return an operand for a basic block argument at a given index. + /// To simplify the implementation, we allocate a fixed register or a stack slot + /// for each basic block argument. + pub fn param_opnd(idx: usize) -> Opnd { + use crate::backend::current::ALLOC_REGS; + use crate::cruby::SIZEOF_VALUE_I32; + + if idx < ALLOC_REGS.len() { + Opnd::Reg(ALLOC_REGS[idx]) + } else { + // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register. + // To avoid clobbering it, we need to start from the next slot, hence `+ 1` for the index. + Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) + } + } + + pub fn linearize_instructions(&self) -> Vec<Insn> { + // Emit instructions with labels, expanding branch parameters + let mut insns = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); + + for block in self.sorted_blocks() { + // Process each instruction, expanding branch params if needed + for insn in &block.insns { + self.expand_branch_insn(insn, &mut insns); + } + } + insns + } + + /// Expand and linearize a branch instruction: + /// 1. If the branch has Target::Block with arguments, insert a ParallelMov first + /// 2. Convert Target::Block to Target::Label + /// 3. Push the converted instruction + fn expand_branch_insn(&self, insn: &Insn, insns: &mut Vec<Insn>) { + // Helper to process branch arguments and return the label target + let mut process_edge = |edge: &BranchEdge| -> Label { + if !edge.args.is_empty() { + insns.push(Insn::ParallelMov { + moves: edge.args.iter().enumerate() + .map(|(idx, &arg)| (Assembler::param_opnd(idx), arg)) + .collect() + }); + } + self.block_label(edge.target) + }; + + // Convert Target::Block to Target::Label, processing args if needed + let stripped_insn = match insn { + Insn::Jmp(Target::Block(edge)) => Insn::Jmp(Target::Label(process_edge(edge))), + Insn::Jz(Target::Block(edge)) => Insn::Jz(Target::Label(process_edge(edge))), + Insn::Jnz(Target::Block(edge)) => Insn::Jnz(Target::Label(process_edge(edge))), + Insn::Je(Target::Block(edge)) => Insn::Je(Target::Label(process_edge(edge))), + Insn::Jne(Target::Block(edge)) => Insn::Jne(Target::Label(process_edge(edge))), + Insn::Jl(Target::Block(edge)) => Insn::Jl(Target::Label(process_edge(edge))), + Insn::Jg(Target::Block(edge)) => Insn::Jg(Target::Label(process_edge(edge))), + Insn::Jge(Target::Block(edge)) => Insn::Jge(Target::Label(process_edge(edge))), + Insn::Jbe(Target::Block(edge)) => Insn::Jbe(Target::Label(process_edge(edge))), + Insn::Jb(Target::Block(edge)) => Insn::Jb(Target::Label(process_edge(edge))), + Insn::Jo(Target::Block(edge)) => Insn::Jo(Target::Label(process_edge(edge))), + Insn::JoMul(Target::Block(edge)) => Insn::JoMul(Target::Label(process_edge(edge))), + Insn::Joz(opnd, Target::Block(edge)) => Insn::Joz(*opnd, Target::Label(process_edge(edge))), + Insn::Jonz(opnd, Target::Block(edge)) => Insn::Jonz(*opnd, Target::Label(process_edge(edge))), + _ => insn.clone() + }; + + // Push the stripped instruction + insns.push(stripped_insn); + } + + // Get the label for a given block by extracting it from the first instruction. + pub(super) fn block_label(&self, block_id: BlockId) -> Label { + let block = &self.basic_blocks[block_id.0]; + match block.insns.first() { + Some(Insn::Label(Target::Label(label))) => *label, + other => panic!("Expected first instruction of block {:?} to be a Label, but found: {:?}", block_id, other), + } + } + + pub fn expect_leaf_ccall(&mut self, stack_size: usize) { + self.leaf_ccall_stack_size = Some(stack_size); + } + + fn set_stack_canary(&mut self) -> Option<Opnd> { + if cfg!(feature = "runtime_checks") { + if let Some(stack_size) = self.leaf_ccall_stack_size.take() { + let canary_addr = self.lea(Opnd::mem(64, SP, (stack_size as i32) * SIZEOF_VALUE_I32)); + let canary_opnd = Opnd::mem(64, canary_addr, 0); + self.mov(canary_opnd, vm_stack_canary().into()); + return Some(canary_opnd) + } + } + None + } + + fn clear_stack_canary(&mut self, canary_opnd: Option<Opnd>){ + if let Some(canary_opnd) = canary_opnd { + self.store(canary_opnd, 0.into()); + }; + } + + /// Build an Opnd::VReg and initialize its LiveRange + pub(super) fn new_vreg(&mut self, num_bits: u8) -> Opnd { + let vreg = Opnd::VReg { idx: self.live_ranges.len(), num_bits }; + self.live_ranges.push(LiveRange { start: None, end: None }); + vreg + } + + /// Append an instruction onto the current list of instructions and update + /// the live ranges of any instructions whose outputs are being used as + /// operands to this instruction. + pub fn push_insn(&mut self, insn: Insn) { + // Index of this instruction + let insn_idx = self.idx; + + // Initialize the live range of the output VReg to insn_idx..=insn_idx + if let Some(Opnd::VReg { idx, .. }) = insn.out_opnd() { + assert!(*idx < self.live_ranges.len()); + assert_eq!(self.live_ranges[*idx], LiveRange { start: None, end: None }); + self.live_ranges[*idx] = LiveRange { start: Some(insn_idx), end: Some(insn_idx) }; + } + + // If we find any VReg from previous instructions, extend the live range to insn_idx + let opnd_iter = insn.opnd_iter(); + for opnd in opnd_iter { + match *opnd { + Opnd::VReg { idx, .. } | + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { + assert!(idx < self.live_ranges.len()); + assert_ne!(self.live_ranges[idx].end, None); + self.live_ranges[idx].end = Some(self.live_ranges[idx].end().max(insn_idx)); + } + _ => {} + } + } + + // If this Assembler should not accept scratch registers, assert no use of them. + if !self.accept_scratch_reg { + let opnd_iter = insn.opnd_iter(); + for opnd in opnd_iter { + assert!(!Self::has_scratch_reg(*opnd), "should not use scratch register: {opnd:?}"); + } + } + + self.idx += 1; + + self.current_block().push_insn(insn); + } + + /// Create a new label instance that we can jump to + pub fn new_label(&mut self, name: &str) -> Target + { + assert!(!name.contains(' '), "use underscores in label names, not spaces"); + + let label = Label(self.label_names.len()); + self.label_names.push(name.to_string()); + Target::Label(label) + } + + // Shuffle register moves, sometimes adding extra moves using scratch_reg, + // so that they will not rewrite each other before they are used. + pub fn resolve_parallel_moves(old_moves: &[(Opnd, Opnd)], scratch_opnd: Option<Opnd>) -> Option<Vec<(Opnd, Opnd)>> { + // Return the index of a move whose destination is not used as a source if any. + fn find_safe_move(moves: &[(Opnd, Opnd)]) -> Option<usize> { + moves.iter().enumerate().find(|&(_, &(dst, src))| { + // Check if `dst` is used in other moves. If `dst` is not used elsewhere, it's safe to write into `dst` now. + moves.iter().filter(|&&other_move| other_move != (dst, src)).all(|&(other_dst, other_src)| + match dst { + Opnd::Reg(reg) => !Assembler::has_reg(other_dst, reg) && !Assembler::has_reg(other_src, reg), + _ => other_dst != dst && other_src != dst, + } + ) + }).map(|(index, _)| index) + } + + // Remove moves whose source and destination are the same + let mut old_moves: Vec<(Opnd, Opnd)> = old_moves.iter().copied() + .filter(|&(dst, src)| dst != src).collect(); + + let mut new_moves = vec![]; + while !old_moves.is_empty() { + // Keep taking safe moves + while let Some(index) = find_safe_move(&old_moves) { + new_moves.push(old_moves.remove(index)); + } + + // No safe move. Load the source of one move into scratch_opnd, and + // then load scratch_opnd into the destination when it's safe. + if !old_moves.is_empty() { + // If scratch_opnd is None, return None and leave it to *_split_with_scratch_regs to resolve it. + let scratch_opnd = scratch_opnd?; + let scratch_reg = scratch_opnd.unwrap_reg(); + // Make sure it's safe to use scratch_reg + assert!(old_moves.iter().all(|&(dst, src)| !Self::has_reg(dst, scratch_reg) && !Self::has_reg(src, scratch_reg))); + + // Move scratch_opnd <- src, and delay dst <- scratch_opnd + let (dst, src) = old_moves.remove(0); + new_moves.push((scratch_opnd, src)); + old_moves.push((dst, scratch_opnd)); + } + } + Some(new_moves) + } + + + /// Sets the out field on the various instructions that require allocated + /// registers because their output is used as the operand on a subsequent + /// instruction. This is our implementation of the linear scan algorithm. + pub(super) fn alloc_regs(mut self, regs: Vec<Reg>) -> Result<Assembler, CompileError> { + // First, create the pool of registers. + let mut pool = RegisterPool::new(regs.clone(), self.stack_base_idx); + + // Mapping between VReg and register or stack slot for each VReg index. + // None if no register or stack slot has been allocated for the VReg. + let mut vreg_opnd: Vec<Option<Opnd>> = vec![None; self.live_ranges.len()]; + + // List of registers saved before a C call, paired with the VReg index. + let mut saved_regs: Vec<(Reg, usize)> = vec![]; + + // Remember the indexes of Insn::FrameSetup to update the stack size later + let mut frame_setup_idxs: Vec<(BlockId, usize)> = vec![]; + + // live_ranges is indexed by original `index` given by the iterator. + let mut asm_local = Assembler::new_with_asm(&self); + + let iterator = &mut self.instruction_iterator(); + + let asm = &mut asm_local; + + let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); + + while let Some((index, mut insn)) = iterator.next(asm) { + // Remember the index of FrameSetup to bump slot_count when we know the max number of spilled VRegs. + if let Insn::FrameSetup { .. } = insn { + assert!(asm.current_block().is_entry); + frame_setup_idxs.push((asm.current_block().id, asm.current_block().insns.len())); + } + + let before_ccall = match (&insn, iterator.peek().map(|(_, insn)| insn)) { + (Insn::ParallelMov { .. }, Some(Insn::CCall { .. })) | + (Insn::CCall { .. }, _) if !pool.is_empty() => { + // If C_RET_REG is in use, move it to another register. + // This must happen before last-use registers are deallocated. + if let Some(vreg_idx) = pool.vreg_for(&C_RET_REG) { + let new_opnd = pool.alloc_opnd(vreg_idx); + asm.mov(new_opnd, C_RET_OPND); + pool.dealloc_opnd(&Opnd::Reg(C_RET_REG)); + vreg_opnd[vreg_idx] = Some(new_opnd); + } + + true + }, + _ => false, + }; + + // Check if this is the last instruction that uses an operand that + // spans more than one instruction. In that case, return the + // allocated register to the pool. + for opnd in insn.opnd_iter() { + match *opnd { + Opnd::VReg { idx, .. } | + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { + // We're going to check if this is the last instruction that + // uses this operand. If it is, we can return the allocated + // register to the pool. + if live_ranges[idx].end() == index { + if let Some(opnd) = vreg_opnd[idx] { + pool.dealloc_opnd(&opnd); + } else { + unreachable!("no register allocated for insn {:?}", insn); + } + } + } + _ => {} + } + } + + // Save caller-saved registers on a C call. + if before_ccall { + // Find all live registers + saved_regs = pool.live_regs(); + + // Save live registers + for pair in saved_regs.chunks(2) { + match *pair { + [(reg0, _), (reg1, _)] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + pool.dealloc_opnd(&Opnd::Reg(reg0)); + pool.dealloc_opnd(&Opnd::Reg(reg1)); + } + [(reg, _)] => { + asm.cpush(Opnd::Reg(reg)); + pool.dealloc_opnd(&Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } + } + // On x86_64, maintain 16-byte stack alignment + if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { + asm.cpush(Opnd::Reg(saved_regs.last().unwrap().0)); + } + } + + // Allocate a register for the output operand if it exists + let vreg_idx = match insn.out_opnd() { + Some(Opnd::VReg { idx, .. }) => Some(*idx), + _ => None, + }; + if let Some(vreg_idx) = vreg_idx { + if live_ranges[vreg_idx].end() == index { + debug!("Allocating a register for VReg({}) at instruction index {} even though it does not live past this index", vreg_idx, index); + } + // This is going to be the output operand that we will set on the + // instruction. CCall and LiveReg need to use a specific register. + let mut out_reg = match insn { + Insn::CCall { .. } => { + Some(pool.take_reg(&C_RET_REG, vreg_idx)) + } + Insn::LiveReg { opnd, .. } => { + let reg = opnd.unwrap_reg(); + Some(pool.take_reg(®, vreg_idx)) + } + _ => None + }; + + // If this instruction's first operand maps to a register and + // this is the last use of the register, reuse the register + // We do this to improve register allocation on x86 + // e.g. out = add(reg0, reg1) + // reg0 = add(reg0, reg1) + if out_reg.is_none() { + let mut opnd_iter = insn.opnd_iter(); + + if let Some(Opnd::VReg{ idx, .. }) = opnd_iter.next() { + if live_ranges[*idx].end() == index { + if let Some(Opnd::Reg(reg)) = vreg_opnd[*idx] { + out_reg = Some(pool.take_reg(®, vreg_idx)); + } + } + } + } + + // Allocate a new register for this instruction if one is not + // already allocated. + let out_opnd = out_reg.unwrap_or_else(|| pool.alloc_opnd(vreg_idx)); + + // Set the output operand on the instruction + let out_num_bits = Opnd::match_num_bits_iter(insn.opnd_iter()); + + // If we have gotten to this point, then we're sure we have an + // output operand on this instruction because the live range + // extends beyond the index of the instruction. + let out = insn.out_opnd_mut().unwrap(); + let out_opnd = out_opnd.with_num_bits(out_num_bits); + vreg_opnd[out.vreg_idx()] = Some(out_opnd); + *out = out_opnd; + } + + // Replace VReg and Param operands by their corresponding register + let mut opnd_iter = insn.opnd_iter_mut(); + while let Some(opnd) = opnd_iter.next() { + match *opnd { + Opnd::VReg { idx, num_bits } => { + *opnd = vreg_opnd[idx].unwrap().with_num_bits(num_bits); + }, + Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { + *opnd = match vreg_opnd[idx].unwrap() { + Opnd::Reg(reg) => Opnd::Mem(Mem { base: MemBase::Reg(reg.reg_no), disp, num_bits }), + // If the base is spilled, lower it to MemBase::Stack, which scratch_split will lower to MemBase::Reg. + Opnd::Mem(mem) => Opnd::Mem(Mem { base: pool.stack_state.mem_to_stack_membase(mem), disp, num_bits }), + _ => unreachable!(), + } + } + _ => {}, + } + } + + // If we have an output that dies at its definition (it is unused), free up the + // register + if let Some(idx) = vreg_idx { + if live_ranges[idx].end() == index { + if let Some(opnd) = vreg_opnd[idx] { + pool.dealloc_opnd(&opnd); + } else { + unreachable!("no register allocated for insn {:?}", insn); + } + } + } + + // Push instruction(s) + let is_ccall = matches!(insn, Insn::CCall { .. }); + match insn { + Insn::CCall { opnds, fptr, start_marker, end_marker, out } => { + // Split start_marker and end_marker here to avoid inserting push/pop between them. + if let Some(start_marker) = start_marker { + asm.push_insn(Insn::PosMarker(start_marker)); + } + asm.push_insn(Insn::CCall { opnds, fptr, start_marker: None, end_marker: None, out }); + if let Some(end_marker) = end_marker { + asm.push_insn(Insn::PosMarker(end_marker)); + } + } + Insn::Mov { src, dest } | Insn::LoadInto { dest, opnd: src } if src == dest => { + // Remove no-op move now that VReg are resolved to physical Reg + } + _ => asm.push_insn(insn), + } + + // After a C call, restore caller-saved registers + if is_ccall { + // On x86_64, maintain 16-byte stack alignment + if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { + asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0)); + } + // Restore saved registers + for pair in saved_regs.chunks(2).rev() { + match *pair { + [(reg, vreg_idx)] => { + asm.cpop_into(Opnd::Reg(reg)); + pool.take_reg(®, vreg_idx); + } + [(reg0, vreg_idx0), (reg1, vreg_idx1)] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + pool.take_reg(®1, vreg_idx1); + pool.take_reg(®0, vreg_idx0); + } + _ => unreachable!("chunks(2)") + } + } + saved_regs.clear(); + } + } + + // Extend the stack space for spilled operands + for (block_id, frame_setup_idx) in frame_setup_idxs { + match &mut asm.basic_blocks[block_id.0].insns[frame_setup_idx] { + Insn::FrameSetup { slot_count, .. } => { + *slot_count += pool.stack_state.stack_size; + } + _ => unreachable!(), + } + } + + assert!(pool.is_empty(), "Expected all registers to be returned to the pool"); + Ok(asm_local) + } + + /// Compile the instructions down to machine code. + /// Can fail due to lack of code memory and inopportune code placement, among other reasons. + pub fn compile(self, cb: &mut CodeBlock) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + #[cfg(feature = "disasm")] + let start_addr = cb.get_write_ptr(); + let alloc_regs = Self::get_alloc_regs(); + let had_dropped_bytes = cb.has_dropped_bytes(); + let ret = self.compile_with_regs(cb, alloc_regs).inspect_err(|err| { + // If we use too much memory to compile the Assembler, it would set cb.dropped_bytes = true. + // To avoid failing future compilation by cb.has_dropped_bytes(), attempt to reset dropped_bytes with + // the current zjit_alloc_bytes() which may be decreased after self is dropped in compile_with_regs(). + if *err == CompileError::OutOfMemory && !had_dropped_bytes { + cb.update_dropped_bytes(); + } + }); + + #[cfg(feature = "disasm")] + if get_option!(dump_disasm) { + let end_addr = cb.get_write_ptr(); + let disasm = crate::disasm::disasm_addr_range(cb, start_addr.raw_ptr(cb) as usize, end_addr.raw_ptr(cb) as usize); + println!("{}", disasm); + } + ret + } + + /// Compile with a limited number of registers. Used only for unit tests. + #[cfg(test)] + pub fn compile_with_num_regs(self, cb: &mut CodeBlock, num_regs: usize) -> (CodePtr, Vec<CodePtr>) { + let mut alloc_regs = Self::get_alloc_regs(); + let alloc_regs = alloc_regs.drain(0..num_regs).collect(); + self.compile_with_regs(cb, alloc_regs).unwrap() + } + + /// Compile Target::SideExit and convert it into Target::CodePtr for all instructions + pub fn compile_exits(&mut self) { + /// Compile the main side-exit code. This function takes only SideExit so + /// that it can be safely deduplicated by using SideExit as a dedup key. + fn compile_exit(asm: &mut Assembler, exit: &SideExit) { + let SideExit { pc, stack, locals } = exit; + + asm_comment!(asm, "save cfp->pc"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); + + asm_comment!(asm, "save cfp->sp"); + asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + + if !stack.is_empty() { + asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); + for (idx, &opnd) in stack.iter().enumerate() { + asm.store(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + } + + if !locals.is_empty() { + asm_comment!(asm, "write locals: {}", join_opnds(&locals, ", ")); + for (idx, &opnd) in locals.iter().enumerate() { + asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); + } + } + + asm_comment!(asm, "exit to the interpreter"); + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(Opnd::UImm(Qundef.as_u64())); + } + + fn join_opnds(opnds: &Vec<Opnd>, delimiter: &str) -> String { + opnds.iter().map(|opnd| format!("{opnd}")).collect::<Vec<_>>().join(delimiter) + } + + // Extract targets first so that we can update instructions while referencing part of them. + let mut targets = HashMap::new(); + + for block in self.sorted_blocks().iter() { + for (idx, insn) in block.insns.iter().enumerate() { + if let Some(target @ Target::SideExit { .. }) = insn.target() { + targets.insert((block.id.0, idx), target.clone()); + } + } + } + + // Map from SideExit to compiled Label. This table is used to deduplicate side exit code. + let mut compiled_exits: HashMap<SideExit, Label> = HashMap::new(); + + for ((block_id, idx), target) in targets { + // Compile a side exit. Note that this is past the split pass and alloc_regs(), + // so you can't use an instruction that returns a VReg. + if let Target::SideExit { exit: exit @ SideExit { pc, .. }, reason } = target { + // Only record the exit if `trace_side_exits` is defined and the counter is either the one specified + let should_record_exit = get_option!(trace_side_exits).map(|trace| match trace { + TraceExits::All => true, + TraceExits::Counter(counter) if counter == side_exit_counter(reason) => true, + _ => false, + }).unwrap_or(false); + + // If enabled, instrument exits first, and then jump to a shared exit. + let counted_exit = if get_option!(stats) || should_record_exit { + let counted_exit = self.new_label("counted_exit"); + self.write_label(counted_exit.clone()); + asm_comment!(self, "Counted Exit: {reason}"); + + if get_option!(stats) { + asm_comment!(self, "increment a side exit counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr(reason)), 1.into()); + + if let SideExitReason::UnhandledYARVInsn(opcode) = reason { + asm_comment!(self, "increment an unhandled YARV insn counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr_for_opcode(opcode)), 1.into()); + } + } + + if should_record_exit { + asm_ccall!(self, rb_zjit_record_exit_stack, pc); + } + + // If the side exit has already been compiled, jump to it. + // Otherwise, let it fall through and compile the exit next. + if let Some(&exit_label) = compiled_exits.get(&exit) { + self.jmp(Target::Label(exit_label)); + } + Some(counted_exit) + } else { + None + }; + + // Compile the shared side exit if not compiled yet + let compiled_exit = if let Some(&compiled_exit) = compiled_exits.get(&exit) { + Target::Label(compiled_exit) + } else { + let new_exit = self.new_label("side_exit"); + self.write_label(new_exit.clone()); + asm_comment!(self, "Exit: {pc}"); + compile_exit(self, &exit); + compiled_exits.insert(exit, new_exit.unwrap_label()); + new_exit + }; + + *self.basic_blocks[block_id].insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); + } + } + } +} + +/// Return a result of fmt::Display for Assembler without escape sequence +pub fn lir_string(asm: &Assembler) -> String { + use crate::ttycolors::TTY_TERMINAL_COLOR; + format!("{asm}").replace(TTY_TERMINAL_COLOR.bold_begin, "").replace(TTY_TERMINAL_COLOR.bold_end, "") +} + +impl fmt::Display for Assembler { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Count the number of duplicated label names to disambiguate them if needed + let mut label_counts: HashMap<&String, usize> = HashMap::new(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + for label_name in self.label_names.iter() { + let counter = label_counts.entry(label_name).or_insert(0); + *counter += 1; + } + + /// Return a label name String. Suffix "_{label_idx}" if the label name is used multiple times. + fn label_name(asm: &Assembler, label_idx: usize, label_counts: &HashMap<&String, usize>) -> String { + let label_name = &asm.label_names[label_idx]; + let label_count = label_counts.get(&label_name).unwrap_or(&0); + if *label_count > 1 { + format!("{label_name}_{label_idx}") + } else { + label_name.to_string() + } + } + + for insn in self.linearize_instructions().iter() { + match insn { + Insn::Comment(comment) => { + writeln!(f, " {bold_begin}# {comment}{bold_end}")?; + } + Insn::Label(target) => { + let &Target::Label(Label(label_idx)) = target else { + panic!("unexpected target for Insn::Label: {target:?}"); + }; + writeln!(f, " {}:", label_name(self, label_idx, &label_counts))?; + } + _ => { + write!(f, " ")?; + + // Print output operand if any + if let Some(out) = insn.out_opnd() { + write!(f, "{out} = ")?; + } + + // Print the instruction name + write!(f, "{}", insn.op())?; + + // Show slot_count for FrameSetup + if let Insn::FrameSetup { slot_count, preserved } = insn { + write!(f, " {slot_count}")?; + if !preserved.is_empty() { + write!(f, ",")?; + } + } + + // Print target + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => write!(f, " {code_ptr:?}")?, + Target::Label(Label(label_idx)) => write!(f, " {}", label_name(self, *label_idx, &label_counts))?, + Target::SideExit { reason, .. } => write!(f, " Exit({reason})")?, + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, " bb{}", edge.target.0)?; + } else { + write!(f, " bb{}(", edge.target.0)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + write!(f, ")")?; + } + } + } + } + + // Print list of operands + if let Some(Target::SideExit { .. }) = insn.target() { + // If the instruction has a SideExit, avoid using opnd_iter(), which has stack/locals. + // Here, only handle instructions that have both Opnd and Target. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if let Some(Target::Block(_)) = insn.target() { + // If the instruction has a Block target, avoid using opnd_iter() for branch args + // since they're already printed inline with the target. Only print non-target operands. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if let Insn::ParallelMov { moves } = insn { + // Print operands with a special syntax for ParallelMov + moves.iter().try_fold(" ", |prefix, (dst, src)| write!(f, "{prefix}{dst} <- {src}").and(Ok(", ")))?; + } else if insn.opnd_iter().count() > 0 { + insn.opnd_iter().try_fold(" ", |prefix, opnd| write!(f, "{prefix}{opnd}").and(Ok(", ")))?; + } + + write!(f, "\n")?; + } + } + } + Ok(()) + } +} + +impl fmt::Debug for Assembler { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + writeln!(fmt, "Assembler")?; + + for (idx, insn) in self.linearize_instructions().iter().enumerate() { + writeln!(fmt, " {idx:03} {insn:?}")?; + } + + Ok(()) + } +} + +pub struct InsnIter { + blocks: Vec<BasicBlock>, + current_block_idx: usize, + current_insn_iter: std::vec::IntoIter<Insn>, + peeked: Option<(usize, Insn)>, + index: usize, +} + +impl InsnIter { + // We're implementing our own peek() because we don't want peek to + // cross basic blocks as we're iterating. + pub fn peek(&mut self) -> Option<&(usize, Insn)> { + // If we don't have a peeked value, get one + if self.peeked.is_none() { + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + self.peeked = Some((idx, insn)); + } + // Return a reference to the peeked value + self.peeked.as_ref() + } + + // Get the next instruction, advancing to the next block when current block is exhausted. + // Sets the current block on new_asm when moving to a new block. + pub fn next(&mut self, new_asm: &mut Assembler) -> Option<(usize, Insn)> { + // If we have a peeked value, return it + if let Some(item) = self.peeked.take() { + return Some(item); + } + + // Try to get the next instruction from current block + if let Some(insn) = self.current_insn_iter.next() { + let idx = self.index; + self.index += 1; + return Some((idx, insn)); + } + + // Current block is exhausted, move to next block + self.current_block_idx += 1; + if self.current_block_idx >= self.blocks.len() { + return None; + } + + // Set up the next block + let next_block = &mut self.blocks[self.current_block_idx]; + new_asm.set_current_block(next_block.id); + self.current_insn_iter = take(&mut next_block.insns).into_iter(); + + // Get first instruction from the new block + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + Some((idx, insn)) + } +} + +impl Assembler { + #[must_use] + pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Add { left, right, out }); + out + } + + pub fn add_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of add_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Add { left, right, out: left }); + } + + #[must_use] + pub fn and(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::And { left, right, out }); + out + } + + pub fn bake_string(&mut self, text: &str) { + self.push_insn(Insn::BakeString(text.to_string())); + } + + #[allow(dead_code)] + pub fn breakpoint(&mut self) { + self.push_insn(Insn::Breakpoint); + } + + /// Call a C function without PosMarkers + pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { + let canary_opnd = self.set_stack_canary(); + let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + let fptr = Opnd::const_ptr(fptr); + self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + self.clear_stack_canary(canary_opnd); + out + } + + /// Call a C function stored in a register + pub fn ccall_reg(&mut self, fptr: Opnd, num_bits: u8) -> Opnd { + assert!(matches!(fptr, Opnd::Reg(_)), "ccall_reg must be called with Opnd::Reg: {fptr:?}"); + let out = self.new_vreg(num_bits); + self.push_insn(Insn::CCall { fptr, opnds: vec![], start_marker: None, end_marker: None, out }); + out + } + + /// Call a C function with PosMarkers. This is used for recording the start and end + /// addresses of the C call and rewriting it with a different function address later. + pub fn ccall_with_pos_markers( + &mut self, + fptr: *const u8, + opnds: Vec<Opnd>, + start_marker: impl Fn(CodePtr, &CodeBlock) + 'static, + end_marker: impl Fn(CodePtr, &CodeBlock) + 'static, + ) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + self.push_insn(Insn::CCall { + fptr: Opnd::const_ptr(fptr), + opnds, + start_marker: Some(Rc::new(start_marker)), + end_marker: Some(Rc::new(end_marker)), + out, + }); + out + } + + pub fn count_call_to(&mut self, fn_name: &str) { + // We emit ccalls while initializing the JIT. Unfortunately, we skip those because + // otherwise we have no counter pointers to read. + if crate::state::ZJITState::has_instance() && get_option!(stats) { + let ccall_counter_pointers = crate::state::ZJITState::get_ccall_counter_pointers(); + let counter_ptr = ccall_counter_pointers.entry(fn_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + self.incr_counter(Opnd::const_ptr(counter_ptr), 1.into()); + } + } + + pub fn cmp(&mut self, left: Opnd, right: Opnd) { + self.push_insn(Insn::Cmp { left, right }); + } + + #[must_use] + pub fn cpop(&mut self) -> Opnd { + let out = self.new_vreg(Opnd::DEFAULT_NUM_BITS); + self.push_insn(Insn::CPop { out }); + out + } + + pub fn cpop_into(&mut self, opnd: Opnd) { + assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); + self.push_insn(Insn::CPopInto(opnd)); + } + + #[track_caller] + pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); + } + + pub fn cpush(&mut self, opnd: Opnd) { + self.push_insn(Insn::CPush(opnd)); + } + + #[track_caller] + pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPushPair(opnd0, opnd1)); + } + + pub fn cret(&mut self, opnd: Opnd) { + self.push_insn(Insn::CRet(opnd)); + } + + #[must_use] + pub fn csel_e(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_g(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelG { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_ge(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelGE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_l(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelL { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_le(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelLE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_ne(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelNE { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_nz(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelNZ { truthy, falsy, out }); + out + } + + #[must_use] + pub fn csel_z(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[truthy, falsy])); + self.push_insn(Insn::CSelZ { truthy, falsy, out }); + out + } + + pub fn frame_setup(&mut self, preserved_regs: &'static [Opnd]) { + let slot_count = self.stack_base_idx; + self.push_insn(Insn::FrameSetup { preserved: preserved_regs, slot_count }); + } + + /// The inverse of [Self::frame_setup] used before return. `reserve_bytes` + /// not necessary since we use a base pointer register. + pub fn frame_teardown(&mut self, preserved_regs: &'static [Opnd]) { + self.push_insn(Insn::FrameTeardown { preserved: preserved_regs }); + } + + pub fn incr_counter(&mut self, mem: Opnd, value: Opnd) { + self.push_insn(Insn::IncrCounter { mem, value }); + } + + pub fn jbe(&mut self, target: Target) { + self.push_insn(Insn::Jbe(target)); + } + + pub fn jb(&mut self, target: Target) { + self.push_insn(Insn::Jb(target)); + } + + pub fn je(&mut self, target: Target) { + self.push_insn(Insn::Je(target)); + } + + pub fn jl(&mut self, target: Target) { + self.push_insn(Insn::Jl(target)); + } + + #[allow(dead_code)] + pub fn jg(&mut self, target: Target) { + self.push_insn(Insn::Jg(target)); + } + + #[allow(dead_code)] + pub fn jge(&mut self, target: Target) { + self.push_insn(Insn::Jge(target)); + } + + pub fn jmp(&mut self, target: Target) { + self.push_insn(Insn::Jmp(target)); + } + + pub fn jmp_opnd(&mut self, opnd: Opnd) { + self.push_insn(Insn::JmpOpnd(opnd)); + } + + pub fn jne(&mut self, target: Target) { + self.push_insn(Insn::Jne(target)); + } + + pub fn jnz(&mut self, target: Target) { + self.push_insn(Insn::Jnz(target)); + } + + pub fn jo(&mut self, target: Target) { + self.push_insn(Insn::Jo(target)); + } + + pub fn jo_mul(&mut self, target: Target) { + self.push_insn(Insn::JoMul(target)); + } + + pub fn jz(&mut self, target: Target) { + self.push_insn(Insn::Jz(target)); + } + + #[must_use] + pub fn lea(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Lea { opnd, out }); + out + } + + pub fn lea_into(&mut self, out: Opnd, opnd: Opnd) { + assert!(matches!(out, Opnd::Reg(_) | Opnd::Mem(_)), "Destination of lea_into must be a register or memory, got: {out:?}"); + self.push_insn(Insn::Lea { opnd, out }); + } + + #[must_use] + pub fn lea_jump_target(&mut self, target: Target) -> Opnd { + let out = self.new_vreg(Opnd::DEFAULT_NUM_BITS); + self.push_insn(Insn::LeaJumpTarget { target, out }); + out + } + + #[must_use] + pub fn live_reg_opnd(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::LiveReg { opnd, out }); + out + } + + #[must_use] + pub fn load(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Load { opnd, out }); + out + } + + pub fn load_into(&mut self, dest: Opnd, opnd: Opnd) { + assert!(matches!(dest, Opnd::Reg(_)), "Destination of load_into must be a register, got: {dest:?}"); + match (dest, opnd) { + (Opnd::Reg(dest), Opnd::Reg(opnd)) if dest == opnd => {}, // skip if noop + _ => self.push_insn(Insn::LoadInto { dest, opnd }), + } + } + + #[must_use] + pub fn load_sext(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::LoadSExt { opnd, out }); + out + } + + #[must_use] + pub fn lshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::LShift { opnd, shift, out }); + out + } + + pub fn parallel_mov(&mut self, moves: Vec<(Opnd, Opnd)>) { + self.push_insn(Insn::ParallelMov { moves }); + } + + pub fn mov(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of mov must not be Opnd::VReg, got: {dest:?}"); + self.push_insn(Insn::Mov { dest, src }); + } + + #[must_use] + pub fn not(&mut self, opnd: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); + self.push_insn(Insn::Not { opnd, out }); + out + } + + #[must_use] + pub fn or(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Or { left, right, out }); + out + } + + pub fn patch_point(&mut self, target: Target, invariant: Invariant, version: IseqVersionRef) { + self.push_insn(Insn::PatchPoint { target, invariant, version }); + } + + pub fn pad_patch_point(&mut self) { + self.push_insn(Insn::PadPatchPoint); + } + + pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr, &CodeBlock) + 'static) { + self.push_insn(Insn::PosMarker(Rc::new(marker_fn))); + } + + #[must_use] + pub fn rshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::RShift { opnd, shift, out }); + out + } + + pub fn store(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of store must not be Opnd::VReg, got: {dest:?}"); + self.push_insn(Insn::Store { dest, src }); + } + + #[must_use] + pub fn sub(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Sub { left, right, out }); + out + } + + pub fn sub_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of sub_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Sub { left, right, out: left }); + } + + #[must_use] + pub fn mul(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Mul { left, right, out }); + out + } + + pub fn test(&mut self, left: Opnd, right: Opnd) { + self.push_insn(Insn::Test { left, right }); + } + + #[must_use] + #[allow(dead_code)] + pub fn urshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[opnd, shift])); + self.push_insn(Insn::URShift { opnd, shift, out }); + out + } + + /// Add a label at the current position + pub fn write_label(&mut self, target: Target) { + assert!(target.unwrap_label().0 < self.label_names.len()); + self.push_insn(Insn::Label(target)); + } + + #[must_use] + pub fn xor(&mut self, left: Opnd, right: Opnd) -> Opnd { + let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); + self.push_insn(Insn::Xor { left, right, out }); + out + } + + /// This is used for trampolines that don't allow scratch registers. + /// Linearizes all blocks into a single giant block. + pub fn resolve_parallel_mov_pass(self) -> Assembler { + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = self.accept_scratch_reg; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for insn in linearized_insns { + match insn { + Insn::ParallelMov { moves } => { + // Resolve parallel moves without scratch register + if let Some(resolved_moves) = Assembler::resolve_parallel_moves(&moves, None) { + for (dst, src) in resolved_moves { + asm_local.mov(dst, src); + } + } else { + unreachable!("ParallelMov requires scratch register but scratch_reg is not allowed"); + } + } + _ => { + asm_local.push_insn(insn); + } + } + } + + asm_local + } +} + +/// Macro to use format! for Insn::Comment, which skips a format! call +/// when not dumping disassembly. +macro_rules! asm_comment { + ($asm:expr, $($fmt:tt)*) => { + // If --zjit-dump-disasm or --zjit-dump-lir is given, enrich them with comments. + // Also allow --zjit-debug on dev builds to enable comments since dev builds dump LIR on panic. + let enable_comment = $crate::options::get_option!(dump_disasm) || + $crate::options::get_option!(dump_lir).is_some() || + (cfg!(debug_assertions) && $crate::options::get_option!(debug)); + if enable_comment { + $asm.push_insn(crate::backend::lir::Insn::Comment(format!($($fmt)*))); + } + }; +} +pub(crate) use asm_comment; + +/// Convenience macro over [`Assembler::ccall`] that also adds a comment with the function name. +macro_rules! asm_ccall { + [$asm: ident, $fn_name:ident, $($args:expr),* ] => {{ + $crate::backend::lir::asm_comment!($asm, concat!("call ", stringify!($fn_name))); + $asm.count_call_to(stringify!($fn_name)); + $asm.ccall($fn_name as *const u8, vec![$($args),*]) + }}; +} +pub(crate) use asm_ccall; + +// Allow moving Assembler to panic hooks. Since we take the VM lock on compilation, +// no other threads should reference the same Assembler instance. +unsafe impl Send for Insn {} +unsafe impl Sync for Insn {} + +/// Dump Assembler with insn_idx on panic. Restore the original panic hook on drop. +pub struct AssemblerPanicHook { + /// Original panic hook before AssemblerPanicHook is installed. + prev_hook: Box<dyn Fn(&panic::PanicHookInfo<'_>) + Sync + Send + 'static>, +} + +impl AssemblerPanicHook { + /// Maximum number of lines [`Self::dump_asm`] is allowed to dump by default. + /// When --zjit-dump-lir is given, this limit is ignored. + const MAX_DUMP_LINES: usize = 10; + + /// Install a panic hook to dump Assembler with insn_idx on dev builds. + /// This returns shared references to the previous hook and insn_idx. + /// It takes insn_idx as an argument so that you can manually use it + /// on non-emit passes that keep mutating the Assembler to be dumped. + pub fn new(asm: &Assembler, insn_idx: usize) -> (Option<Arc<Self>>, Option<Arc<Mutex<usize>>>) { + if cfg!(debug_assertions) { + // Wrap prev_hook with Arc to share it among the new hook and Self to be dropped. + let prev_hook = panic::take_hook(); + let panic_hook_ref = Arc::new(Self { prev_hook }); + let weak_hook = Arc::downgrade(&panic_hook_ref); + + // Wrap insn_idx with Arc to share it among the new hook and the caller mutating it. + let insn_idx = Arc::new(Mutex::new(insn_idx)); + let insn_idx_ref = insn_idx.clone(); + + // Install a new hook to dump Assembler with insn_idx + let asm = asm.clone(); + panic::set_hook(Box::new(move |panic_info| { + if let Some(panic_hook) = weak_hook.upgrade() { + if let Ok(insn_idx) = insn_idx_ref.lock() { + // Dump Assembler, highlighting the insn_idx line + Self::dump_asm(&asm, *insn_idx); + } + + // Call the previous panic hook + (panic_hook.prev_hook)(panic_info); + } + })); + + (Some(panic_hook_ref), Some(insn_idx)) + } else { + (None, None) + } + } + + /// Dump Assembler, highlighting the insn_idx line + fn dump_asm(asm: &Assembler, insn_idx: usize) { + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + let lir_string = lir_string(asm); + let lines: Vec<&str> = lir_string.split('\n').collect(); + + // By default, dump only MAX_DUMP_LINES lines. + // Ignore it if --zjit-dump-lir is given. + let (min_idx, max_idx) = if get_option!(dump_lir).is_some() { + (0, lines.len()) + } else { + (insn_idx.saturating_sub(Self::MAX_DUMP_LINES / 2), insn_idx.saturating_add(Self::MAX_DUMP_LINES / 2)) + }; + + eprintln!("Failed to compile LIR at insn_idx={insn_idx}:"); + for (idx, line) in lines.iter().enumerate().filter(|(idx, _)| (min_idx..=max_idx).contains(idx)) { + if idx == insn_idx && line.starts_with(" ") { + eprintln!("{bold_begin}=>{}{bold_end}", &line[" ".len()..]); + } else { + eprintln!("{line}"); + } + } + } +} + +impl Drop for AssemblerPanicHook { + fn drop(&mut self) { + // Restore the original hook + panic::set_hook(std::mem::replace(&mut self.prev_hook, Box::new(|_| {}))); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_reg() -> Opnd { + Assembler::new_with_scratch_reg().1 + } + + #[test] + fn test_opnd_iter() { + let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; + + let mut opnd_iter = insn.opnd_iter(); + assert!(matches!(opnd_iter.next(), Some(Opnd::None))); + assert!(matches!(opnd_iter.next(), Some(Opnd::None))); + + assert!(opnd_iter.next().is_none()); + } + + #[test] + fn test_opnd_iter_mut() { + let mut insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; + + let mut opnd_iter = insn.opnd_iter_mut(); + assert!(matches!(opnd_iter.next(), Some(Opnd::None))); + assert!(matches!(opnd_iter.next(), Some(Opnd::None))); + + assert!(opnd_iter.next().is_none()); + } + + #[test] + #[should_panic] + fn load_into_memory_is_invalid() { + let mut asm = Assembler::new(); + let mem = Opnd::mem(64, SP, 0); + asm.load_into(mem, mem); + } + + #[test] + fn test_resolve_parallel_moves_reorder_registers() { + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, Some(vec![ + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + fn test_resolve_parallel_moves_give_up_register_cycle() { + // If scratch_opnd is not given, it cannot break cycles. + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, None); + } + + #[test] + fn test_resolve_parallel_moves_break_register_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_memory_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_register_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_reorder_memory_destination() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_register() { + Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[0], CFP), + ], Some(scratch_reg())); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_memory() { + Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg())); + } +} diff --git a/zjit/src/backend/mod.rs b/zjit/src/backend/mod.rs new file mode 100644 index 0000000000..635acbf60c --- /dev/null +++ b/zjit/src/backend/mod.rs @@ -0,0 +1,18 @@ +//! A multi-platform assembler generation backend. + +#[cfg(target_arch = "x86_64")] +pub mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub mod arm64; + +#[cfg(target_arch = "x86_64")] +pub use x86_64 as current; + +#[cfg(target_arch = "aarch64")] +pub use arm64 as current; + +#[cfg(test)] +mod tests; + +pub mod lir; diff --git a/zjit/src/backend/tests.rs b/zjit/src/backend/tests.rs new file mode 100644 index 0000000000..701029b8ec --- /dev/null +++ b/zjit/src/backend/tests.rs @@ -0,0 +1,313 @@ +use crate::asm::CodeBlock; +use crate::backend::lir::*; +use crate::cruby::*; +use crate::codegen::c_callable; +use crate::options::rb_zjit_prepare_options; +use crate::hir; + +#[test] +fn test_add() { + let mut asm = Assembler::new(); + asm.new_block_without_id(); + let out = asm.add(SP, Opnd::UImm(1)); + let _ = asm.add(out, Opnd::UImm(2)); +} + +#[test] +fn test_alloc_regs() { + rb_zjit_prepare_options(); // for asm.alloc_regs + let mut asm = Assembler::new(); + asm.new_block_without_id(); + + // Get the first output that we're going to reuse later. + let out1 = asm.add(EC, Opnd::UImm(1)); + + // Pad some instructions in to make sure it can handle that. + let _ = asm.add(EC, Opnd::UImm(2)); + + // Get the second output we're going to reuse. + let out2 = asm.add(EC, Opnd::UImm(3)); + + // Pad another instruction. + let _ = asm.add(EC, Opnd::UImm(4)); + + // Reuse both the previously captured outputs. + let _ = asm.add(out1, out2); + + // Now get a third output to make sure that the pool has registers to + // allocate now that the previous ones have been returned. + let out3 = asm.add(EC, Opnd::UImm(5)); + let _ = asm.add(out3, Opnd::UImm(6)); + + // Here we're going to allocate the registers. + let result = &asm.alloc_regs(Assembler::get_alloc_regs()).unwrap().basic_blocks[0]; + + // Now we're going to verify that the out field has been appropriately + // updated for each of the instructions that needs it. + let regs = Assembler::get_alloc_regs(); + let reg0 = regs[0]; + let reg1 = regs[1]; + + match result.insns[0].out_opnd() { + Some(Opnd::Reg(value)) => assert_eq!(value, ®0), + val => panic!("Unexpected register value {:?}", val), + } + + match result.insns[2].out_opnd() { + Some(Opnd::Reg(value)) => assert_eq!(value, ®1), + val => panic!("Unexpected register value {:?}", val), + } + + match result.insns[5].out_opnd() { + Some(Opnd::Reg(value)) => assert_eq!(value, ®0), + val => panic!("Unexpected register value {:?}", val), + } +} + +fn setup_asm() -> (Assembler, CodeBlock) { + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) +} + +// Test full codegen pipeline +#[test] +fn test_compile() +{ + let (mut asm, mut cb) = setup_asm(); + let regs = Assembler::get_alloc_regs(); + + let out = asm.add(Opnd::Reg(regs[0]), Opnd::UImm(2)); + let out2 = asm.add(out, Opnd::UImm(2)); + asm.store(Opnd::mem(64, SP, 0), out2); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test memory-to-memory move +#[test] +fn test_mov_mem2mem() +{ + let (mut asm, mut cb) = setup_asm(); + + asm_comment!(asm, "check that comments work too"); + asm.mov(Opnd::mem(64, SP, 0), Opnd::mem(64, SP, 8)); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test load of register into new register +#[test] +fn test_load_reg() +{ + let (mut asm, mut cb) = setup_asm(); + + let out = asm.load(SP); + asm.mov(Opnd::mem(64, SP, 0), out); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Test load of a GC'd value +#[test] +fn test_load_value() +{ + let (mut asm, mut cb) = setup_asm(); + + let gcd_value = VALUE(0xFFFFFFFFFFFF00); + assert!(!gcd_value.special_const_p()); + + let out = asm.load(Opnd::Value(gcd_value)); + asm.mov(Opnd::mem(64, SP, 0), out); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Multiple registers needed and register reuse +#[test] +fn test_reuse_reg() +{ + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.add(Opnd::mem(64, SP, 0), Opnd::UImm(1)); + let v1 = asm.add(Opnd::mem(64, SP, 8), Opnd::UImm(1)); + + let v2 = asm.add(v1, Opnd::UImm(1)); // Reuse v1 register + let v3 = asm.add(v0, v2); + + asm.store(Opnd::mem(64, SP, 0), v2); + asm.store(Opnd::mem(64, SP, 8), v3); + + asm.compile_with_num_regs(&mut cb, 2); +} + +// 64-bit values can't be written directly to memory, +// need to be split into one or more register movs first +#[test] +fn test_store_u64() +{ + let (mut asm, mut cb) = setup_asm(); + asm.store(Opnd::mem(64, SP, 0), u64::MAX.into()); + + asm.compile_with_num_regs(&mut cb, 1); +} + +// Use instruction output as base register for memory operand +#[test] +fn test_base_insn_out() +{ + let (mut asm, mut cb) = setup_asm(); + + // Forced register to be reused + // This also causes the insn sequence to change length + asm.mov( + Opnd::mem(64, SP, 8), + Opnd::mem(64, SP, 0) + ); + + // Load the pointer into a register + let ptr_opnd = Opnd::const_ptr(4351776248 as *const u8); + + // Increment and store the updated value + asm.incr_counter(ptr_opnd, 1.into()); + + asm.compile_with_num_regs(&mut cb, 2); +} + +#[test] +fn test_c_call() +{ + c_callable! { + fn dummy_c_fun(_v0: usize, _v1: usize) {} + } + + let (mut asm, mut cb) = setup_asm(); + + let ret_val = asm.ccall( + dummy_c_fun as *const u8, + vec![Opnd::mem(64, SP, 0), Opnd::UImm(1)] + ); + + // Make sure that the call's return value is usable + asm.mov(Opnd::mem(64, SP, 0), ret_val); + + asm.compile(&mut cb).unwrap(); +} + +#[test] +fn test_alloc_ccall_regs() { + let (mut asm, mut cb) = setup_asm(); + let out1 = asm.ccall(std::ptr::null::<u8>(), vec![]); + let out2 = asm.ccall(std::ptr::null::<u8>(), vec![out1]); + asm.mov(EC, out2); + asm.compile_with_regs(&mut cb, Assembler::get_alloc_regs()).unwrap(); +} + +#[test] +fn test_lea_ret() +{ + let (mut asm, mut cb) = setup_asm(); + + let addr = asm.lea(Opnd::mem(64, SP, 0)); + asm.cret(addr); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_jcc_label() +{ + let (mut asm, mut cb) = setup_asm(); + + let label = asm.new_label("foo"); + asm.cmp(EC, EC); + asm.je(label.clone()); + asm.write_label(label); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_jcc_ptr() +{ + let (mut asm, mut cb) = setup_asm(); + + let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32)); + asm.test( + Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32), + not_mask, + ); + asm.jnz(side_exit); + + asm.compile_with_num_regs(&mut cb, 2); +} + +/// Direct jump to a stub e.g. for deferred compilation +#[test] +fn test_jmp_ptr() +{ + let (mut asm, mut cb) = setup_asm(); + + let stub = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + asm.jmp(stub); + + asm.compile_with_num_regs(&mut cb, 0); +} + +#[test] +fn test_jo() +{ + let (mut asm, mut cb) = setup_asm(); + + let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); + + let arg1 = Opnd::mem(64, SP, 0); + let arg0 = Opnd::mem(64, SP, 8); + + let arg0_untag = asm.sub(arg0, Opnd::Imm(1)); + let out_val = asm.add(arg0_untag, arg1); + asm.jo(side_exit); + + asm.mov(Opnd::mem(64, SP, 0), out_val); + + asm.compile_with_num_regs(&mut cb, 2); +} + +#[test] +fn test_bake_string() { + let (mut asm, mut cb) = setup_asm(); + + asm.bake_string("Hello, world!"); + asm.compile_with_num_regs(&mut cb, 0); +} + +#[test] +fn test_cmp_8_bit() { + let (mut asm, mut cb) = setup_asm(); + let reg = Assembler::get_alloc_regs()[0]; + asm.cmp(Opnd::Reg(reg).with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + + asm.compile_with_num_regs(&mut cb, 1); +} + +#[test] +fn test_no_pos_marker_callback_when_compile_fails() { + // When compilation fails (e.g. when out of memory), the code written out is malformed. + // We don't want to invoke the pos_marker callbacks with positions of malformed code. + let mut asm = Assembler::new(); + rb_zjit_prepare_options(); // for asm.compile + asm.new_block_without_id(); + + // Markers around code to exhaust memory limit + let fail_if_called = |_code_ptr, _cb: &_| panic!("pos_marker callback should not be called"); + asm.pos_marker(fail_if_called); + let zero = asm.load(0.into()); + let sum = asm.add(zero, 500.into()); + asm.store(Opnd::mem(64, SP, 8), sum); + asm.pos_marker(fail_if_called); + + let cb = &mut CodeBlock::new_dummy_sized(8); + assert!(asm.compile(cb).is_err(), "should fail due to tiny size limit"); +} diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs new file mode 100644 index 0000000000..a4cf8dfcc5 --- /dev/null +++ b/zjit/src/backend/x86_64/mod.rs @@ -0,0 +1,1959 @@ +use std::mem::{self, take}; + +use crate::asm::*; +use crate::asm::x86_64::*; +use crate::codegen::split_patch_point; +use crate::stats::CompileError; +use crate::virtualmem::CodePtr; +use crate::cruby::*; +use crate::backend::lir::*; +use crate::cast::*; +use crate::options::asm_dump; + +// Use the x86 register type for this platform +pub type Reg = X86Reg; + +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit GP register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_type: RegType::GP, reg_no } +} + +// Callee-saved registers +pub const CFP: Opnd = Opnd::Reg(R13_REG); +pub const EC: Opnd = Opnd::Reg(R12_REG); +pub const SP: Opnd = Opnd::Reg(RBX_REG); + +// C argument registers on this platform +pub const C_ARG_OPNDS: [Opnd; 6] = [ + Opnd::Reg(RDI_REG), + Opnd::Reg(RSI_REG), + Opnd::Reg(RDX_REG), + Opnd::Reg(RCX_REG), + Opnd::Reg(R8_REG), + Opnd::Reg(R9_REG) +]; + +// C return value register on this platform +pub const C_RET_REG: Reg = RAX_REG; +pub const C_RET_OPND: Opnd = Opnd::Reg(RAX_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(RSP_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(RBP_REG); + +impl CodeBlock { + // The number of bytes that are generated by jmp_ptr + pub fn jmp_ptr_bytes(&self) -> usize { 5 } +} + +/// Map Opnd to X86Opnd +impl From<Opnd> for X86Opnd { + fn from(opnd: Opnd) -> Self { + match opnd { + // NOTE: these operand types need to be lowered first + //Value(VALUE), // Immediate Ruby value, may be GC'd, movable + //VReg(usize), // Output of a preceding instruction in this block + + Opnd::VReg{..} => panic!("VReg operand made it past register allocation"), + + Opnd::UImm(val) => uimm_opnd(val), + Opnd::Imm(val) => imm_opnd(val), + Opnd::Value(VALUE(uimm)) => uimm_opnd(uimm as u64), + + // General-purpose register + Opnd::Reg(reg) => X86Opnd::Reg(reg), + + // Memory operand with displacement + Opnd::Mem(Mem{ base: MemBase::Reg(reg_no), num_bits, disp }) => { + let reg = X86Reg { + reg_no, + num_bits: 64, + reg_type: RegType::GP + }; + + mem_opnd(num_bits, X86Opnd::Reg(reg), disp) + } + + Opnd::None => panic!( + "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." + ), + + _ => panic!("unsupported x86 operand type: {opnd:?}") + } + } +} + +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for X86Opnd { + fn from(opnd: &Opnd) -> Self { + X86Opnd::from(*opnd) + } +} + +/// List of registers that can be used for register allocation. +/// This has the same number of registers for x86_64 and arm64. +/// SCRATCH0_OPND is excluded. +pub const ALLOC_REGS: &[Reg] = &[ + RDI_REG, + RSI_REG, + RDX_REG, + RCX_REG, + R8_REG, + R9_REG, + RAX_REG, +]; + +/// Special scratch register for intermediate processing. It should be used only by +/// [`Assembler::x86_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(R11_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(R10_REG); + +impl Assembler { + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } + + /// Get the list of registers from which we can allocate on this platform + pub fn get_alloc_regs() -> Vec<Reg> { + ALLOC_REGS.to_vec() + } + + /// Get a list of all of the caller-save registers + pub fn get_caller_save_regs() -> Vec<Reg> { + vec![RAX_REG, RCX_REG, RDX_REG, RSI_REG, RDI_REG, R8_REG, R9_REG, R10_REG, R11_REG] + } + + /// How many bytes a call and a bare bones [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + + // These are the callee-saved registers in the x86-64 SysV ABI + // RBX, RSP, RBP, and R12–R15 + + /// Split IR instructions for the x86 platform + fn x86_split(mut self) -> Assembler + { + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; + let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); + let mut iterator = self.instruction_iterator(); + + while let Some((index, mut insn)) = iterator.next(asm) { + let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); + let mut opnd_iter = insn.opnd_iter_mut(); + + while let Some(opnd) = opnd_iter.next() { + // Lower Opnd::Value to Opnd::VReg or Opnd::UImm + match opnd { + Opnd::Value(value) if !is_load => { + // Since mov(mem64, imm32) sign extends, as_i64() makes sure + // we split when the extended value is different. + *opnd = if !value.special_const_p() || imm_num_bits(value.as_i64()) > 32 { + asm.load(*opnd) + } else { + Opnd::UImm(value.as_u64()) + } + } + _ => {}, + }; + } + + // When we split an operand, we can create a new VReg not in `live_ranges`. + // So when we see a VReg with out-of-range index, it's created from splitting + // from the loop above and we know it doesn't outlive the current instruction. + let vreg_outlives_insn = |vreg_idx| { + live_ranges + .get(vreg_idx) + .is_some_and(|live_range: &LiveRange| live_range.end() > index) + }; + + // We are replacing instructions here so we know they are already + // being used. It is okay not to use their output here. + #[allow(unused_must_use)] + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::Mul { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + match (&left, &right, iterator.peek().map(|(_, insn)| insn)) { + // Merge this insn, e.g. `add REG, right -> out`, and `mov REG, out` if possible + (Opnd::Reg(_), Opnd::UImm(value), Some(Insn::Mov { dest, src })) + if out == src && left == dest && live_ranges[out.vreg_idx()].end() == index + 1 && uimm_num_bits(*value) <= 32 => { + *out = *dest; + asm.push_insn(insn); + iterator.next(asm); // Pop merged Insn::Mov + } + (Opnd::Reg(_), Opnd::Reg(_), Some(Insn::Mov { dest, src })) + if out == src && live_ranges[out.vreg_idx()].end() == index + 1 && *dest == *left => { + *out = *dest; + asm.push_insn(insn); + iterator.next(asm); // Pop merged Insn::Mov + } + _ => { + match (*left, *right) { + (Opnd::Mem(_), Opnd::Mem(_)) => { + *left = asm.load(*left); + *right = asm.load(*right); + }, + (Opnd::Mem(_), Opnd::UImm(_) | Opnd::Imm(_)) => { + *left = asm.load(*left); + }, + // Instruction output whose live range spans beyond this instruction + (Opnd::VReg { idx, .. }, _) => { + if vreg_outlives_insn(idx) { + *left = asm.load(*left); + } + }, + // We have to load memory operands to avoid corrupting them + (Opnd::Mem(_), _) => { + *left = asm.load(*left); + }, + // We have to load register operands to avoid corrupting them + (Opnd::Reg(_), _) => { + if *left != *out { + *left = asm.load(*left); + } + }, + // The first operand can't be an immediate value + (Opnd::UImm(_), _) => { + *left = asm.load(*left); + } + _ => {} + } + asm.push_insn(insn); + } + } + }, + Insn::Cmp { left, right } => { + // Replace `cmp REG, 0` (4 bytes) with `test REG, REG` (3 bytes) + // when next IR is `je`, `jne`, `csel_e`, or `csel_ne` + match (&left, &right, iterator.peek().map(|(_, insn)| insn)) { + (Opnd::VReg { .. }, + Opnd::UImm(0) | Opnd::Imm(0), + Some(Insn::Je(_) | Insn::Jne(_) | Insn::CSelE { .. } | Insn::CSelNE { .. })) => { + asm.push_insn(Insn::Test { left: *left, right: *left }); + } + _ => { + // Split the instruction if `cmp` can't be encoded with given operands + match (&left, &right) { + // One of the operands should not be a memory operand + (Opnd::Mem(_), Opnd::Mem(_)) => { + *right = asm.load(*right); + } + // The left operand needs to be either a register or a memory operand + (Opnd::UImm(_) | Opnd::Imm(_), _) => { + *left = asm.load(*left); + } + _ => {}, + } + asm.push_insn(insn); + } + } + }, + Insn::Test { left, right } => { + match (&left, &right) { + (Opnd::Mem(_), Opnd::Mem(_)) => { + *right = asm.load(*right); + } + // The first operand can't be an immediate value + (Opnd::UImm(_) | Opnd::Imm(_), _) => { + *left = asm.load(*left); + } + _ => {} + } + asm.push_insn(insn); + }, + // These instructions modify their input operand in-place, so we + // may need to load the input value to preserve it + Insn::LShift { opnd, .. } | + Insn::RShift { opnd, .. } | + Insn::URShift { opnd, .. } => { + match opnd { + // Instruction output whose live range spans beyond this instruction + Opnd::VReg { idx, .. } => { + if vreg_outlives_insn(*idx) { + *opnd = asm.load(*opnd); + } + }, + // We have to load non-reg operands to avoid corrupting them + Opnd::Mem(_) | Opnd::Reg(_) | Opnd::UImm(_) | Opnd::Imm(_) => { + *opnd = asm.load(*opnd); + }, + _ => {} + } + asm.push_insn(insn); + }, + Insn::CSelZ { truthy, falsy, .. } | + Insn::CSelNZ { truthy, falsy, .. } | + Insn::CSelE { truthy, falsy, .. } | + Insn::CSelNE { truthy, falsy, .. } | + Insn::CSelL { truthy, falsy, .. } | + Insn::CSelLE { truthy, falsy, .. } | + Insn::CSelG { truthy, falsy, .. } | + Insn::CSelGE { truthy, falsy, .. } => { + match *truthy { + // If we have an instruction output whose live range + // spans beyond this instruction, we have to load it. + Opnd::VReg { idx, .. } => { + if vreg_outlives_insn(idx) { + *truthy = asm.load(*truthy); + } + }, + Opnd::UImm(_) | Opnd::Imm(_) => { + *truthy = asm.load(*truthy); + }, + // Opnd::Value could have already been split + Opnd::Value(_) if !matches!(truthy, Opnd::VReg { .. }) => { + *truthy = asm.load(*truthy); + }, + _ => {} + } + + match falsy { + Opnd::UImm(_) | Opnd::Imm(_) => { + *falsy = asm.load(*falsy); + }, + _ => {} + } + + asm.push_insn(insn); + }, + Insn::Mov { dest, src } => { + if let Opnd::Mem(_) = dest { + asm.store(*dest, *src); + } else { + asm.mov(*dest, *src); + } + }, + Insn::Not { opnd, .. } => { + match *opnd { + // If we have an instruction output whose live range + // spans beyond this instruction, we have to load it. + Opnd::VReg { idx, .. } => { + if vreg_outlives_insn(idx) { + *opnd = asm.load(*opnd); + } + }, + // We have to load memory and register operands to avoid + // corrupting them. + Opnd::Mem(_) | Opnd::Reg(_) => { + *opnd = asm.load(*opnd); + }, + // Otherwise we can just reuse the existing operand. + _ => {}, + }; + asm.push_insn(insn); + }, + Insn::CCall { opnds, .. } => { + assert!(opnds.len() <= C_ARG_OPNDS.len()); + + // Load each operand into the corresponding argument register. + if !opnds.is_empty() { + let mut args: Vec<(Opnd, Opnd)> = vec![]; + for (idx, opnd) in opnds.iter_mut().enumerate() { + args.push((C_ARG_OPNDS[idx], *opnd)); + } + asm.parallel_mov(args); + } + + // Now we push the CCall without any arguments so that it + // just performs the call. + *opnds = vec![]; + asm.push_insn(insn); + }, + Insn::Lea { .. } => { + // Merge `lea` and `mov` into a single `lea` when possible + match (&insn, iterator.peek().map(|(_, insn)| insn)) { + (Insn::Lea { opnd, out }, Some(Insn::Mov { dest: Opnd::Reg(reg), src })) + if matches!(out, Opnd::VReg { .. }) && out == src && live_ranges[out.vreg_idx()].end() == index + 1 => { + asm.push_insn(Insn::Lea { opnd: *opnd, out: Opnd::Reg(*reg) }); + iterator.next(asm); // Pop merged Insn::Mov + } + _ => asm.push_insn(insn), + } + }, + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Split instructions using scratch registers. To maximize the use of the register pool + /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so + /// this splits them and uses scratch registers for it. + pub fn x86_scratch_split(self) -> Assembler { + /// For some instructions, we want to be able to lower a 64-bit operand + /// without requiring more registers to be available in the register + /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold + /// the value before we immediately use it. + fn split_64bit_immediate(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Imm(value) => { + // 32-bit values will be sign-extended + if imm_num_bits(value) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + }, + Opnd::UImm(value) => { + // 32-bit values will be sign-extended + if imm_num_bits(value as i64) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd + } else { + Opnd::Imm(value as i64) + } + }, + _ => opnd + } + } + + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg using a scratch regsiter. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp, num_bits }) = opnd { + let base = Opnd::Mem(stack_state.stack_membase_to_mem(stack_membase)); + asm.load_into(scratch_opnd, base); + Opnd::Mem(Mem { base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), disp, num_bits }) + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + /// If both opnd and other are Opnd::Mem, split opnd with scratch_opnd. + fn split_if_both_memory(asm: &mut Assembler, opnd: Opnd, other: Opnd, scratch_opnd: Opnd) -> Opnd { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (opnd, other) { + asm.load_into(scratch_opnd.with_num_bits(opnd.rm_num_bits()), opnd); + scratch_opnd.with_num_bits(opnd.rm_num_bits()) + } else { + opnd + } + } + + /// Move src to dst, splitting it with scratch_opnd if it's a Mem-to-Mem move. Skip it if dst == src. + fn asm_mov(asm: &mut Assembler, dst: Opnd, src: Opnd, scratch_opnd: Opnd) { + if dst != src { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (dst, src) { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else { + asm.mov(dst, src); + } + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + for insn in linearized_insns.iter() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + Insn::Mul { left, right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + // imul doesn't have (Mem, Reg) encoding. Swap left and right in that case. + if let (Opnd::Mem(_), Opnd::Reg(_)) = (&left, &right) { + mem::swap(left, right); + } + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + &mut Insn::Not { opnd, out } | + &mut Insn::LShift { opnd, out, .. } | + &mut Insn::RShift { opnd, out, .. } | + &mut Insn::URShift { opnd, out, .. } => { + asm.push_insn(insn); + asm_mov(asm, out, opnd, SCRATCH0_OPND); + } + Insn::Test { left, right } | + Insn::Cmp { left, right } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + + let num_bits = match right { + Opnd::Imm(value) => Some(imm_num_bits(*value)), + Opnd::UImm(value) => Some(uimm_num_bits(*value)), + _ => None + }; + + // If the immediate is less than 64 bits (like 32, 16, 8), and the operand + // sizes match, then we can represent it as an immediate in the instruction + // without moving it to a register first. + // IOW, 64 bit immediates must always be moved to a register + // before comparisons, where other sizes may be encoded + // directly in the instruction. + let use_imm = num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64; + if !use_imm { + *right = split_64bit_immediate(asm, *right, SCRATCH0_OPND); + } + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Lea { opnd, out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(_) = target { + asm.push_insn(Insn::LeaJumpTarget { out: SCRATCH0_OPND, target: target.clone() }); + asm.mov(*out, SCRATCH0_OPND); + } + } + Insn::Load { out, opnd } | + Insn::LoadInto { dest: out, opnd } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND.with_num_bits(mem_out.rm_num_bits())); + } + } + // Convert Opnd::const_ptr into Opnd::Mem. This split is done here to give + // a register for compile_exits. + &mut Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.incr_counter(Opnd::mem(64, SCRATCH0_OPND, 0), value); + } + &mut Insn::Mov { dest, src } => { + asm_mov(asm, dest, src, SCRATCH0_OPND); + } + // Resolve ParallelMov that couldn't be handled without a scratch register. + Insn::ParallelMov { moves } => { + for (dst, src) in Self::resolve_parallel_moves(&moves, Some(SCRATCH0_OPND)).unwrap() { + asm_mov(asm, dst, src, SCRATCH0_OPND); + } + } + // Handle various operand combinations for spills on compile_exits. + &mut Insn::Store { dest, src } => { + let num_bits = dest.rm_num_bits(); + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + + let src = match src { + Opnd::Reg(_) => src, + Opnd::Mem(_) => { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + Opnd::Imm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else if uimm_num_bits(imm as u64) <= num_bits { + // If the bit string is short enough for the destination, use the unsigned representation. + // Note that 64-bit and negative values are ruled out. + Opnd::UImm(imm as u64) + } else { + src + } + } + Opnd::UImm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm as i64) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else { + src.into() + } + } + Opnd::Value(_) => { + asm.load_into(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_scratch_split: {src:?}"), + }; + asm.store(dest, src); + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Emit platform-specific machine code + pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { + fn emit_csel( + cb: &mut CodeBlock, + truthy: Opnd, + falsy: Opnd, + out: Opnd, + cmov_fn: fn(&mut CodeBlock, X86Opnd, X86Opnd), + cmov_neg: fn(&mut CodeBlock, X86Opnd, X86Opnd)){ + + // Assert that output is a register + out.unwrap_reg(); + + // If the truthy value is a memory operand + if let Opnd::Mem(_) = truthy { + if out != falsy { + mov(cb, out.into(), falsy.into()); + } + + cmov_fn(cb, out.into(), truthy.into()); + } else { + if out != truthy { + mov(cb, out.into(), truthy.into()); + } + + cmov_neg(cb, out.into(), falsy.into()); + } + } + + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest_reg: X86Opnd, value: VALUE) { + // Using movabs because mov might write value in 32 bits + movabs(cb, dest_reg, value.0 as _); + // The pointer immediate is encoded as the last part of the mov written out + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } + + // List of GC offsets + let mut gc_offsets: Vec<CodePtr> = Vec::new(); + + // Buffered list of PosMarker callbacks to fire if codegen is successful + let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // Install a panic hook to dump Assembler with insn_idx on dev builds + let (_hook, mut hook_insn_idx) = AssemblerPanicHook::new(self, 0); + + // For each instruction + let mut insn_idx: usize = 0; + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { + // Update insn_idx that is shown on panic + hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); + + match insn { + Insn::Comment(text) => { + cb.add_comment(text); + }, + + // Write the label at the current position + Insn::Label(target) => { + cb.write_label(target.unwrap_label()); + }, + + // Report back the current position in the generated code + Insn::PosMarker(..) => { + pos_markers.push((insn_idx, cb.get_write_ptr())); + }, + + Insn::BakeString(text) => { + for byte in text.as_bytes() { + cb.write_byte(*byte); + } + + // Add a null-terminator byte for safety (in case we pass + // this to C code) + cb.write_byte(0); + }, + + // Set up RBP as frame pointer work with unwinding + // (e.g. with Linux `perf record --call-graph fp`) + // and to allow push and pops in the function. + &Insn::FrameSetup { preserved, mut slot_count } => { + // Bump slot_count for alignment if necessary + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + let total_slots = 2 /* rbp and return address*/ + slot_count + preserved.len(); + if total_slots % 2 == 1 { + slot_count += 1; + } + push(cb, RBP); + mov(cb, RBP, RSP); + for reg in preserved { + push(cb, reg.into()); + } + if slot_count > 0 { + sub(cb, RSP, uimm_opnd((slot_count * SIZEOF_VALUE) as u64)); + } + } + &Insn::FrameTeardown { preserved } => { + let mut preserved_offset = -8; + for reg in preserved { + mov(cb, reg.into(), mem_opnd(64, RBP, preserved_offset)); + preserved_offset -= 8; + } + mov(cb, RSP, RBP); + pop(cb, RBP); + } + + Insn::Add { left, right, .. } => { + add(cb, left.into(), right.into()); + }, + + Insn::Sub { left, right, .. } => { + sub(cb, left.into(), right.into()); + }, + + Insn::Mul { left, right, .. } => { + imul(cb, left.into(), right.into()); + }, + + Insn::And { left, right, .. } => { + and(cb, left.into(), right.into()); + }, + + Insn::Or { left, right, .. } => { + or(cb, left.into(), right.into()); + }, + + Insn::Xor { left, right, .. } => { + xor(cb, left.into(), right.into()); + }, + + Insn::Not { opnd, .. } => { + not(cb, opnd.into()); + }, + + Insn::LShift { opnd, shift , ..} => { + shl(cb, opnd.into(), shift.into()) + }, + + Insn::RShift { opnd, shift , ..} => { + sar(cb, opnd.into(), shift.into()) + }, + + Insn::URShift { opnd, shift, .. } => { + shr(cb, opnd.into(), shift.into()) + }, + + // This assumes only load instructions can contain references to GC'd Value operands + Insn::Load { opnd, out } | + Insn::LoadInto { dest: out, opnd } => { + match opnd { + Opnd::Value(val) if val.heap_object_p() => { + emit_load_gc_value(cb, &mut gc_offsets, out.into(), *val); + } + _ => mov(cb, out.into(), opnd.into()) + } + }, + + Insn::LoadSExt { opnd, out } => { + movsx(cb, out.into(), opnd.into()); + }, + + Insn::ParallelMov { .. } => unreachable!("{insn:?} should have been lowered at alloc_regs()"), + + Insn::Store { dest, src } | + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); + }, + + // Load effective address + Insn::Lea { opnd, out } => { + lea(cb, out.into(), opnd.into()); + }, + + // Load address of jump target + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(label) = target { + let out = *out; + // Set output to the raw address of the label + cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| { + let disp = dst_addr - src_addr; + lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap())); + Ok(()) + }); + } else { + // Set output to the jump target's raw address + let target_code = target.unwrap_code_ptr(); + let target_addr = target_code.raw_addr(cb).as_u64(); + // Constant encoded length important for patching + movabs(cb, out.into(), target_addr); + } + }, + + // Push and pop to/from the C stack + Insn::CPush(opnd) => { + push(cb, opnd.into()); + }, + Insn::CPushPair(opnd0, opnd1) => { + push(cb, opnd0.into()); + push(cb, opnd1.into()); + }, + Insn::CPop { out } => { + pop(cb, out.into()); + }, + Insn::CPopInto(opnd) => { + pop(cb, opnd.into()); + }, + Insn::CPopPairInto(opnd0, opnd1) => { + pop(cb, opnd0.into()); + pop(cb, opnd1.into()); + }, + + // C function call + Insn::CCall { fptr, .. } => { + match fptr { + Opnd::UImm(fptr) => { + call_ptr(cb, RAX, *fptr as *const u8); + } + Opnd::Reg(_) => { + call(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } + }, + + Insn::CRet(opnd) => { + // TODO: bias allocation towards return register + if *opnd != Opnd::Reg(C_RET_REG) { + mov(cb, RAX, opnd.into()); + } + + ret(cb); + }, + + // Compare + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); + } + + // Test and set flags + Insn::Test { left, right } => { + test(cb, left.into(), right.into()); + } + + Insn::JmpOpnd(opnd) => { + jmp_rm(cb, opnd.into()); + } + + // Conditional jump to a label + Insn::Jmp(target) => { + match *target { + Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), + Target::Label(label) => jmp_label(cb, label), + Target::Block(ref edge) => jmp_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Je(target) => { + match *target { + Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), + Target::Label(label) => je_label(cb, label), + Target::Block(ref edge) => je_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jne(target) => { + match *target { + Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), + Target::Label(label) => jne_label(cb, label), + Target::Block(ref edge) => jne_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jl(target) => { + match *target { + Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), + Target::Label(label) => jl_label(cb, label), + Target::Block(ref edge) => jl_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jg(target) => { + match *target { + Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr), + Target::Label(label) => jg_label(cb, label), + Target::Block(ref edge) => jg_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jge(target) => { + match *target { + Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr), + Target::Label(label) => jge_label(cb, label), + Target::Block(ref edge) => jge_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jbe(target) => { + match *target { + Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), + Target::Label(label) => jbe_label(cb, label), + Target::Block(ref edge) => jbe_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jb(target) => { + match *target { + Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr), + Target::Label(label) => jb_label(cb, label), + Target::Block(ref edge) => jb_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + }, + + Insn::Jz(target) => { + match *target { + Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), + Target::Label(label) => jz_label(cb, label), + Target::Block(ref edge) => jz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jnz(target) => { + match *target { + Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), + Target::Label(label) => jnz_label(cb, label), + Target::Block(ref edge) => jnz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Jo(target) | + Insn::JoMul(target) => { + match *target { + Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), + Target::Label(label) => jo_label(cb, label), + Target::Block(ref edge) => jo_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), + } + } + + Insn::Joz(..) | Insn::Jonz(..) => unreachable!("Joz/Jonz should be unused for now"), + + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in x86_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + let code_size = cb.get_write_pos().saturating_sub(last_patch_pos); + if code_size < cb.jmp_ptr_bytes() { + nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + + // Atomically increment a counter at a given memory location + Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::Mem(_))); + assert!(matches!(value, Opnd::UImm(_) | Opnd::Imm(_) ) ); + write_lock_prefix(cb); + add(cb, mem.into(), value.into()); + }, + + Insn::Breakpoint => int3(cb), + + Insn::CSelZ { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovz, cmovnz); + }, + Insn::CSelNZ { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovnz, cmovz); + }, + Insn::CSelE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmove, cmovne); + }, + Insn::CSelNE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovne, cmove); + }, + Insn::CSelL { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovl, cmovge); + }, + Insn::CSelLE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovle, cmovg); + }, + Insn::CSelG { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovg, cmovle); + }, + Insn::CSelGE { truthy, falsy, out } => { + emit_csel(cb, *truthy, *falsy, *out, cmovge, cmovl); + } + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + }; + + insn_idx += 1; + } + + // Error if we couldn't write out everything + if cb.has_dropped_bytes() { + None + } else { + // No bytes dropped, so the pos markers point to valid code + for (insn_idx, pos) in pos_markers { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { + callback(pos, cb); + } else { + panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); + } + } + + Some(gc_offsets) + } + } + + /// Optimize and compile the stored instructions + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_regs = !self.accept_scratch_reg; + asm_dump!(self, init); + + let asm = self.x86_split(); + asm_dump!(asm, split); + + let mut asm = asm.alloc_regs(regs)?; + asm_dump!(asm, alloc_regs); + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + asm.compile_exits(); + asm_dump!(asm, compile_exits); + + if use_scratch_regs { + asm = asm.x86_scratch_split(); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = asm.resolve_parallel_mov_pass(); + asm_dump!(asm, resolve_parallel_mov); + } + + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.x86_emit(cb); + + if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + Ok((start_ptr, gc_offsets)) + } else { + cb.clear_labels(); + Err(CompileError::OutOfMemory) + } + } +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + use crate::options::rb_zjit_prepare_options; + use super::*; + + const BOLD_BEGIN: &str = "\x1b[1m"; + const BOLD_END: &str = "\x1b[22m"; + + fn setup_asm() -> (Assembler, CodeBlock) { + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id(); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.parallel_mov(vec![(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)), (C_ARG_OPNDS[1], Opnd::mem(64, SP, -8))]); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.je(label); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @r" + bb0: + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, r13, rbx, r12 + v0 = Add r13, 0x40 + Store [rbx + 0x10], v0 + Joz Exit(Interrupt), v0 + ParallelMov rdi <- eax, rsi <- [rbx - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[r12 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown r13, rbx, r12 + "); + } + + #[test] + #[ignore] + fn test_emit_add_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: add rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_add_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: add rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c01d8"); + } + + #[test] + #[ignore] + fn test_emit_and_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: and rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_and_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: and rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c21d8"); + } + + #[test] + fn test_emit_cmp_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"4881f8ff000000"); + } + + #[test] + fn test_emit_cmp_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); + } + + #[test] + fn test_emit_cmp_64_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, -1"); + assert_snapshot!(cb.hexdump(), @"4883f8ff"); + } + + #[test] + fn test_emit_cmp_mem_16_bits_with_imm_16() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(16, Opnd::Reg(RAX_REG), 6); + + asm.cmp(shape_opnd, Opnd::UImm(0xF000)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp word ptr [rax + 6], 0xf000"); + assert_snapshot!(cb.hexdump(), @"6681780600f0"); + } + + #[test] + fn test_emit_cmp_mem_32_bits_with_imm_32() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, Opnd::Reg(RAX_REG), 4); + + asm.cmp(shape_opnd, Opnd::UImm(0xF000_0000)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp dword ptr [rax + 4], 0xf0000000"); + assert_snapshot!(cb.hexdump(), @"817804000000f0"); + } + + #[test] + #[ignore] + fn test_emit_or_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: or rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c8ff000000"); + } + + #[test] + #[ignore] + fn test_emit_or_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: or rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c09d8"); + } + + #[test] + #[ignore] + fn test_emit_sub_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: sub rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e8ff000000"); + } + + #[test] + #[ignore] + fn test_emit_sub_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: sub rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c29d8"); + } + + #[test] + fn test_emit_test_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: test rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"48f7c0ff000000"); + } + + #[test] + fn test_emit_test_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: test rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c85d8"); + } + + #[test] + #[ignore] + fn test_emit_xor_lt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: xor rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881f0ff000000"); + } + + #[test] + #[ignore] + fn test_emit_xor_gt_32_bits() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: xor rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c31d8"); + } + + #[test] + fn test_merge_lea_reg() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.lea(Opnd::mem(64, SP, 8)); + asm.mov(SP, sp); // should be merged to lea + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lea rbx, [rbx + 8]"); + assert_snapshot!(cb.hexdump(), @"488d5b08"); + } + + #[test] + #[ignore] + fn test_merge_lea_mem() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.lea(Opnd::mem(64, SP, 8)); + asm.mov(Opnd::mem(64, SP, 0), sp); // should NOT be merged to lea + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); + } + + #[test] + #[ignore] + fn test_replace_cmp_0() { + let (mut asm, mut cb) = setup_asm(); + + let val = asm.load(Opnd::mem(64, SP, 8)); + asm.cmp(val, 0.into()); + let result = asm.csel_e(Qtrue.into(), Qfalse.into()); + asm.mov(Opnd::Reg(RAX_REG), result); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, qword ptr [rbx + 8] + 0x4: test rax, rax + 0x7: mov eax, 0x14 + 0xc: mov ecx, 0 + 0x11: cmovne rax, rcx + 0x15: mov rax, rax + "); + assert_snapshot!(cb.hexdump(), @"488b43084885c0b814000000b900000000480f45c14889c0"); + } + + #[test] + fn test_merge_add_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.add(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); + } + + #[test] + fn test_add_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.add_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); + } + + #[test] + fn test_merge_sub_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.sub(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); + } + + #[test] + fn test_sub_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.sub_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); + } + + #[test] + fn test_merge_and_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.and(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983e540"); + } + + #[test] + fn test_merge_or_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.or(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983cd40"); + } + + #[test] + fn test_merge_xor_mov() { + let (mut asm, mut cb) = setup_asm(); + + let sp = asm.xor(CFP, Opnd::UImm(0x40)); + asm.mov(CFP, sp); // should be merged to add + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983f540"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_no_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov rdi, rdi (optimized away) + C_ARG_OPNDS[1], // mov rsi, rsi (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov eax, 0 + 0x5: call rax + "); + assert_snapshot!(cb.hexdump(), @"b800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_single_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi and rsi form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[0], // mov rsi, rdi + C_ARG_OPNDS[2], // mov rdx, rdx (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rsi + 0x3: mov rsi, rdi + 0x6: mov rdi, r11 + 0x9: mov eax, 0 + 0xe: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89dfb800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_two_cycles() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi and rsi form a cycle, and rdx and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[0], // mov rsi, rdi + C_ARG_OPNDS[3], // mov rdx, rcx + C_ARG_OPNDS[2], // mov rcx, rdx + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rsi + 0x3: mov rsi, rdi + 0x6: mov rdi, r11 + 0x9: mov r11, rcx + 0xc: mov rcx, rdx + 0xf: mov rdx, r11 + 0x12: mov eax, 0 + 0x17: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89df4989cb4889d14c89dab800000000ffd0"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_large_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // rdi, rsi, and rdx form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov rdi, rsi + C_ARG_OPNDS[2], // mov rsi, rdx + C_ARG_OPNDS[0], // mov rdx, rdi + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rsi + 0x3: mov rsi, rdx + 0x6: mov rdx, rdi + 0x9: mov rdi, r11 + 0xc: mov eax, 0 + 0x11: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989f34889d64889fa4c89dfb800000000ffd0"); + } + + #[test] + #[ignore] + fn test_ccall_resolve_parallel_moves_with_insn_out() { + let (mut asm, mut cb) = setup_asm(); + + let rax = asm.load(Opnd::UImm(1)); + let rcx = asm.load(Opnd::UImm(2)); + let rdx = asm.load(Opnd::UImm(3)); + // rcx and rdx form a cycle + asm.ccall(0 as _, vec![ + rax, // mov rdi, rax + rcx, // mov rsi, rcx + rcx, // mov rdx, rcx + rdx, // mov rcx, rdx + ]); + asm.compile_with_num_regs(&mut cb, 3); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov eax, 1 + 0x5: mov ecx, 2 + 0xa: mov edx, 3 + 0xf: mov rdi, rax + 0x12: mov rsi, rcx + 0x15: mov r11, rcx + 0x18: mov rcx, rdx + 0x1b: mov rdx, r11 + 0x1e: mov eax, 0 + 0x23: call rax + "); + assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: push rdi + 0x15: push rsi + 0x16: push rdx + 0x17: push rcx + 0x18: mov eax, 0 + 0x1d: call rax + 0x1f: pop rcx + 0x20: pop rdx + 0x21: pop rsi + 0x22: pop rdi + 0x23: add rdi, rsi + 0x26: add rdx, rcx + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74801ca"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: mov r8d, 5 + 0x1a: push rdi + 0x1b: push rsi + 0x1c: push rdx + 0x1d: push rcx + 0x1e: push r8 + 0x20: push r8 + 0x22: mov eax, 0 + 0x27: call rax + 0x29: pop r8 + 0x2b: pop r8 + 0x2d: pop rcx + 0x2e: pop rdx + 0x2f: pop rsi + 0x30: pop rdi + 0x31: add rdi, rsi + 0x34: mov rdi, rdx + 0x37: add rdi, rcx + 0x3a: add rdx, r8 + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141504150b800000000ffd041584158595a5e5f4801f74889d74801cf4c01c2"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: push rdi + 0xb: push rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: pop rdi + 0xb: pop rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e"); + } + + #[test] + fn test_cmov_mem() { + let (mut asm, mut cb) = setup_asm(); + + let top = Opnd::mem(64, SP, 0); + let ary_opnd = SP; + let array_len_opnd = Opnd::mem(64, SP, 16); + + asm.cmp(array_len_opnd, 1.into()); + let elem_opnd = asm.csel_g(Opnd::mem(64, ary_opnd, 0), Qnil.into()); + asm.mov(top, elem_opnd); + + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cmp qword ptr [rbx + 0x10], 1 + 0x5: mov edi, 4 + 0xa: cmovg rdi, qword ptr [rbx] + 0xe: mov qword ptr [rbx], rdi + "); + assert_snapshot!(cb.hexdump(), @"48837b1001bf04000000480f4f3b48893b"); + } + + #[test] + #[ignore] + fn test_csel_split() { + let (mut asm, mut cb) = setup_asm(); + + let stack_top = Opnd::mem(64, SP, 0); + let elem_opnd = asm.csel_ne(VALUE(0x7f22c88d1930).into(), Qnil.into()); + asm.mov(stack_top, elem_opnd); + + asm.compile_with_num_regs(&mut cb, 3); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs rax, 0x7f22c88d1930 + 0xa: mov ecx, 4 + 0xf: cmove rax, rcx + 0x13: mov qword ptr [rbx], rax + "); + assert_snapshot!(cb.hexdump(), @"48b830198dc8227f0000b904000000480f44c1488903"); + } + + #[test] + fn test_mov_m32_imm32() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, C_RET_OPND, 0); + asm.mov(shape_opnd, Opnd::UImm(0x8000_0001)); + asm.mov(shape_opnd, Opnd::Imm(0x8000_0001)); + + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov dword ptr [rax], 0x80000001 + 0x6: mov dword ptr [rax], 0x80000001 + "); + assert_snapshot!(cb.hexdump(), @"c70001000080c70001000080"); + } + + #[test] + fn frame_setup_teardown_preserved_regs() { + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(JIT_PRESERVED_REGS); + asm.frame_teardown(JIT_PRESERVED_REGS); + asm.cret(C_RET_OPND); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: push r13 + 0x6: push rbx + 0x7: push r12 + 0x9: sub rsp, 8 + 0xd: mov r13, qword ptr [rbp - 8] + 0x11: mov rbx, qword ptr [rbp - 0x10] + 0x15: mov r12, qword ptr [rbp - 0x18] + 0x19: mov rsp, rbp + 0x1c: pop rbp + 0x1d: ret + "); + assert_snapshot!(cb.hexdump(), @"554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3"); + } + + #[test] + fn frame_setup_teardown_stack_base_idx() { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 5; + asm.frame_setup(&[]); + asm.frame_teardown(&[]); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: sub rsp, 0x30 + 0x8: mov rsp, rbp + 0xb: pop rbp + "); + assert_snapshot!(cb.hexdump(), @"554889e54883ec304889ec5d"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + asm = asm.x86_scratch_split(); + let gc_offsets = asm.x86_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0x1000 + 0xa: mov qword ptr [rbx], r11 + "); + assert_snapshot!(cb.hexdump(), @"49bb00100000000000004c891b"); + } + + #[test] + fn test_csel_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let left = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let right = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 1, num_bits: 64 }, disp: 2, num_bits: 64 }); + let _ = asm.csel_e(left, right); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [rbp - 8] + 0x4: mov r11, qword ptr [rbp - 0x10] + 0x8: mov r11, qword ptr [r11 + 2] + 0xc: cmove r11, qword ptr [r10] + 0x10: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b55f84c8b5df04d8b5b024d0f441a4c895df8"); + } + + #[test] + fn test_lea_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let _ = asm.lea(opnd); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 8] + 0x4: lea r11, [r11] + 0x7: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b5df84d8d1b4c895df8"); + } +} diff --git a/zjit/src/bitset.rs b/zjit/src/bitset.rs new file mode 100644 index 0000000000..b5b69abdee --- /dev/null +++ b/zjit/src/bitset.rs @@ -0,0 +1,126 @@ +//! Optimized bitset implementation. + +type Entry = u128; + +const ENTRY_NUM_BITS: usize = Entry::BITS as usize; + +// TODO(max): Make a `SmallBitSet` and `LargeBitSet` and switch between them if `num_bits` fits in +// `Entry`. +#[derive(Clone)] +pub struct BitSet<T: Into<usize> + Copy> { + entries: Vec<Entry>, + num_bits: usize, + phantom: std::marker::PhantomData<T>, +} + +impl<T: Into<usize> + Copy> BitSet<T> { + pub fn with_capacity(num_bits: usize) -> Self { + let num_entries = num_bits.div_ceil(ENTRY_NUM_BITS); + Self { entries: vec![0; num_entries], num_bits, phantom: Default::default() } + } + + /// Returns whether the value was newly inserted: true if the set did not originally contain + /// the bit, and false otherwise. + pub fn insert(&mut self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + let newly_inserted = (self.entries[entry_idx] & (1 << bit_idx)) == 0; + self.entries[entry_idx] |= 1 << bit_idx; + newly_inserted + } + + /// Set all bits to 1. + pub fn insert_all(&mut self) { + for i in 0..self.entries.len() { + self.entries[i] = !0; + } + } + + pub fn get(&self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + (self.entries[entry_idx] & (1 << bit_idx)) != 0 + } + + /// Modify `self` to only have bits set if they are also set in `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn intersect_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] &= other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } +} + +#[cfg(test)] +mod tests { + use super::BitSet; + + #[test] + #[should_panic] + fn get_over_capacity_panics() { + let set = BitSet::with_capacity(0); + assert!(!set.get(0usize)); + } + + #[test] + fn with_capacity_defaults_to_zero() { + let set = BitSet::with_capacity(4); + assert!(!set.get(0usize)); + assert!(!set.get(1usize)); + assert!(!set.get(2usize)); + assert!(!set.get(3usize)); + } + + #[test] + fn insert_sets_bit() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(set.get(1usize)); + } + + #[test] + fn insert_with_set_bit_returns_false() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(!set.insert(1usize)); + } + + #[test] + fn insert_all_sets_all_bits() { + let mut set = BitSet::with_capacity(4); + set.insert_all(); + assert!(set.get(0usize)); + assert!(set.get(1usize)); + assert!(set.get(2usize)); + assert!(set.get(3usize)); + } + + #[test] + #[should_panic] + fn intersect_with_panics_with_different_num_bits() { + let mut left: BitSet<usize> = BitSet::with_capacity(3); + let right = BitSet::with_capacity(4); + left.intersect_with(&right); + } + #[test] + fn intersect_with_keeps_only_common_bits() { + let mut left = BitSet::with_capacity(3); + let mut right = BitSet::with_capacity(3); + left.insert(0usize); + left.insert(1usize); + right.insert(1usize); + right.insert(2usize); + left.intersect_with(&right); + assert!(!left.get(0usize)); + assert!(left.get(1usize)); + assert!(!left.get(2usize)); + } +} diff --git a/zjit/src/cast.rs b/zjit/src/cast.rs new file mode 100644 index 0000000000..52e2078cde --- /dev/null +++ b/zjit/src/cast.rs @@ -0,0 +1,64 @@ +//! Optimized [usize] casting trait. + +#![allow(clippy::wrong_self_convention)] + +/// Trait for casting to [usize] that allows you to say `.as_usize()`. +/// Implementation conditional on the cast preserving the numeric value on +/// all inputs and being inexpensive. +/// +/// [usize] is only guaranteed to be more than 16-bit wide, so we can't use +/// `.into()` to cast an `u32` or an `u64` to a `usize` even though in all +/// the platforms we support these casts are pretty much no-ops. +/// We could say `as usize` or `.try_convert().unwrap()` everywhere for these +/// casts but both options have undesirable consequences if and when we decide to +/// support 32-bit platforms. Unfortunately we can't implement [::core::convert::From] +/// for [usize] since both the trait and the type are external. Also, naming +/// the method `into()` also causes a name conflict. +pub(crate) trait IntoUsize { + /// Convert to usize. Implementation conditional on width of [usize]. + fn to_usize(self) -> usize; +} + +#[cfg(target_pointer_width = "64")] +impl IntoUsize for u64 { + fn to_usize(self) -> usize { + self as usize + } +} + +#[cfg(target_pointer_width = "64")] +impl IntoUsize for u32 { + fn to_usize(self) -> usize { + self as usize + } +} + +impl IntoUsize for u16 { + /// Alias for `.into()`. For convenience so you could use the trait for + /// all unsgined types. + fn to_usize(self) -> usize { + self.into() + } +} + +impl IntoUsize for u8 { + /// Alias for `.into()`. For convenience so you could use the trait for + /// all unsgined types. + fn to_usize(self) -> usize { + self.into() + } +} + +/// The `Into<u64>` Rust does not provide. +/// Convert to u64 with assurance that the value is preserved. +/// Currently, `usize::BITS == 64` holds for all platforms we support. +pub(crate) trait IntoU64 { + fn as_u64(self) -> u64; +} + +#[cfg(target_pointer_width = "64")] +impl IntoU64 for usize { + fn as_u64(self) -> u64 { + self as u64 + } +} diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs new file mode 100644 index 0000000000..0030493ddf --- /dev/null +++ b/zjit/src/codegen.rs @@ -0,0 +1,2992 @@ +//! This module is for native code generation. + +#![allow(clippy::let_and_return)] + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::ffi::{c_int, c_long, c_void}; +use std::slice; + +use crate::backend::current::ALLOC_REGS; +use crate::invariants::{ + track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, + track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption +}; +use crate::gc::append_gc_offsets; +use crate::payload::{get_or_create_iseq_payload, IseqCodePtrs, IseqVersion, IseqVersionRef, IseqStatus}; +use crate::state::ZJITState; +use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; +use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; +use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_STACK_PTR, Opnd, SP, SideExit, Target, asm_ccall, asm_comment}; +use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; +use crate::hir::{Const, FrameState, Function, Insn, InsnId, SendFallbackReason}; +use crate::hir_type::{types, Type}; +use crate::options::get_option; +use crate::cast::IntoUsize; + +/// At the moment, we support recompiling each ISEQ only once. +pub const MAX_ISEQ_VERSIONS: usize = 2; + +/// Sentinel program counter stored in C frames when runtime checks are enabled. +const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { + Some(usize::MAX as *const VALUE) +} else { + None +}; + +/// Ephemeral code generation state +struct JITState { + /// Instruction sequence for the method being compiled + iseq: IseqPtr, + + /// ISEQ version that is being compiled, which will be used by PatchPoint + version: IseqVersionRef, + + /// Low-level IR Operands indexed by High-level IR's Instruction ID + opnds: Vec<Option<Opnd>>, + + /// Labels for each basic block indexed by the BlockId + labels: Vec<Option<Target>>, + + /// JIT entry point for the `iseq` + jit_entries: Vec<Rc<RefCell<JITEntry>>>, + + /// ISEQ calls that need to be compiled later + iseq_calls: Vec<IseqCallRef>, +} + +impl JITState { + /// Create a new JITState instance + fn new(iseq: IseqPtr, version: IseqVersionRef, num_insns: usize, num_blocks: usize) -> Self { + JITState { + iseq, + version, + opnds: vec![None; num_insns], + labels: vec![None; num_blocks], + jit_entries: Vec::default(), + iseq_calls: Vec::default(), + } + } + + /// Retrieve the output of a given instruction that has been compiled + fn get_opnd(&self, insn_id: InsnId) -> lir::Opnd { + self.opnds[insn_id.0].unwrap_or_else(|| panic!("Failed to get_opnd({insn_id})")) + } + + /// Find or create a label for a given BlockId + fn get_label(&mut self, asm: &mut Assembler, lir_block_id: lir::BlockId, hir_block_id: BlockId) -> Target { + // Extend labels vector if the requested index is out of bounds + if lir_block_id.0 >= self.labels.len() { + self.labels.resize(lir_block_id.0 + 1, None); + } + + match &self.labels[lir_block_id.0] { + Some(label) => label.clone(), + None => { + let label = asm.new_label(&format!("{hir_block_id}_{lir_block_id}")); + self.labels[lir_block_id.0] = Some(label.clone()); + label + } + } + } +} + +/// CRuby API to compile a given ISEQ. +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See jit_compile_exception() for details. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, jit_exception: bool) -> *const u8 { + // Take a lock to avoid writing to ISEQ in parallel with Ractors. + // with_vm_lock() does nothing if the program doesn't use Ractors. + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let mut code_ptr = with_time_stat(compile_time_ns, || gen_iseq_entry_point(cb, iseq, jit_exception)); + + if let Err(err) = &code_ptr { + // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled. + // We assert only `jit_exception: false` cases until we support exception handlers. + if ZJITState::assert_compiles_enabled() && !jit_exception { + let iseq_location = iseq_get_location(iseq, 0); + panic!("Failed to compile: {iseq_location}"); + } + + // For --zjit-stats, generate an entry that just increments exit_compilation_failure and exits + if get_option!(stats) { + code_ptr = gen_compile_error_counter(cb, err); + } + } + + // Always mark the code region executable if asm.compile() has been used. + // We need to do this even if code_ptr is None because gen_iseq() may have already used asm.compile(). + cb.mark_all_executable(); + + code_ptr.map_or(std::ptr::null(), |ptr| ptr.raw_ptr(cb)) + }) +} + +/// Compile an entry point for a given ISEQ +fn gen_iseq_entry_point(cb: &mut CodeBlock, iseq: IseqPtr, jit_exception: bool) -> Result<CodePtr, CompileError> { + // We don't support exception handlers yet + if jit_exception { + return Err(CompileError::ExceptionHandler); + } + + // Compile ISEQ into High-level IR + let function = compile_iseq(iseq).inspect_err(|_| { + incr_counter!(failed_iseq_count); + })?; + + // Compile the High-level IR + let IseqCodePtrs { start_ptr, .. } = gen_iseq(cb, iseq, Some(&function)).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq, 0)); + })?; + + Ok(start_ptr) +} + +/// Stub a branch for a JIT-to-JIT call +pub fn gen_iseq_call(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<(), CompileError> { + // Compile a function stub + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the JIT-to-JIT call to call the stub + let stub_addr = stub_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); + asm.ccall(stub_addr, vec![]); + }); + Ok(()) +} + +/// Write an entry to the perf map in /tmp +fn register_with_perf(iseq_name: String, start_ptr: usize, code_size: usize) { + use std::io::Write; + let perf_map = format!("/tmp/perf-{}.map", std::process::id()); + let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(&perf_map) else { + debug!("Failed to open perf map file: {perf_map}"); + return; + }; + let mut file = std::io::BufWriter::new(file); + let Ok(_) = writeln!(file, "{start_ptr:#x} {code_size:#x} zjit::{iseq_name}") else { + debug!("Failed to write {iseq_name} to perf map file: {perf_map}"); + return; + }; +} + +/// Compile a shared JIT entry trampoline +pub fn gen_entry_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + // Set up registers for CFP, EC, SP, and basic block arguments + let mut asm = Assembler::new(); + asm.new_block_without_id(); + gen_entry_prologue(&mut asm); + + // Jump to the first block using a call instruction. This trampoline is used + // as rb_zjit_func_t in jit_exec(), which takes (EC, CFP, rb_jit_func_t). + // So C_ARG_OPNDS[2] is rb_jit_func_t, which is (EC, CFP) -> VALUE. + asm.ccall_reg(C_ARG_OPNDS[2], VALUE_BITS); + + // Restore registers for CFP, EC, and SP after use + asm_comment!(asm, "return to the interpreter"); + asm.frame_teardown(lir::JIT_PRESERVED_REGS); + asm.cret(C_RET_OPND); + + let (code_ptr, gc_offsets) = asm.compile(cb)?; + assert!(gc_offsets.is_empty()); + if get_option!(perf) { + let start_ptr = code_ptr.raw_addr(cb); + let end_ptr = cb.get_write_ptr().raw_addr(cb); + let code_size = end_ptr - start_ptr; + register_with_perf("ZJIT entry trampoline".into(), start_ptr, code_size); + } + Ok(code_ptr) +} + +/// Compile an ISEQ into machine code if not compiled yet +fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // Return an existing pointer if it's already compiled + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + match last_status { + Some(IseqStatus::Compiled(code_ptrs)) => return Ok(code_ptrs.clone()), + Some(IseqStatus::CantCompile(err)) => return Err(err.clone()), + _ => {}, + } + // If the ISEQ already hax MAX_ISEQ_VERSIONS, do not compile a new version. + if payload.versions.len() == MAX_ISEQ_VERSIONS { + return Err(CompileError::IseqVersionLimitReached); + } + + // Compile the ISEQ + let mut version = IseqVersion::new(iseq); + let code_ptrs = gen_iseq_body(cb, iseq, version, function); + match &code_ptrs { + Ok(code_ptrs) => { + unsafe { version.as_mut() }.status = IseqStatus::Compiled(code_ptrs.clone()); + incr_counter!(compiled_iseq_count); + } + Err(err) => { + unsafe { version.as_mut() }.status = IseqStatus::CantCompile(err.clone()); + incr_counter!(failed_iseq_count); + } + } + payload.versions.push(version); + code_ptrs +} + +/// Compile an ISEQ into machine code +fn gen_iseq_body(cb: &mut CodeBlock, iseq: IseqPtr, mut version: IseqVersionRef, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // If we ran out of code region, we shouldn't attempt to generate new code. + if cb.has_dropped_bytes() { + return Err(CompileError::OutOfMemory); + } + + // Convert ISEQ into optimized High-level IR if not given + let function = match function { + Some(function) => function, + None => &compile_iseq(iseq)?, + }; + + // Compile the High-level IR + let (iseq_code_ptrs, gc_offsets, iseq_calls) = gen_function(cb, iseq, version, function)?; + + // Stub callee ISEQs for JIT-to-JIT calls + for iseq_call in iseq_calls.iter() { + gen_iseq_call(cb, iseq_call)?; + } + + // Prepare for GC + unsafe { version.as_mut() }.outgoing.extend(iseq_calls); + append_gc_offsets(iseq, version, &gc_offsets); + Ok(iseq_code_ptrs) +} + +/// Compile a function +fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, function: &Function) -> Result<(IseqCodePtrs, Vec<CodePtr>, Vec<IseqCallRef>), CompileError> { + let num_spilled_params = max_num_params(function).saturating_sub(ALLOC_REGS.len()); + let mut jit = JITState::new(iseq, version, function.num_insns(), function.num_blocks()); + let mut asm = Assembler::new_with_stack_slots(num_spilled_params); + + // Mapping from HIR block IDs to LIR block IDs. + // This is is a one-to-one mapping from HIR to LIR blocks used for finding + // jump targets in LIR (LIR should always jump to the head of an HIR block) + let mut hir_to_lir: Vec<Option<lir::BlockId>> = vec![None; function.num_blocks()]; + + let reverse_post_order = function.rpo(); + + // Create all LIR basic blocks corresponding to HIR basic blocks + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + let lir_block_id = asm.new_block(block_id, function.is_entry_block(block_id), rpo_idx); + hir_to_lir[block_id.0] = Some(lir_block_id); + } + + // Compile each basic block + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + // Set the current block to the LIR block that corresponds to this + // HIR block. + let lir_block_id = hir_to_lir[block_id.0].unwrap(); + asm.set_current_block(lir_block_id); + + // Write a label to jump to the basic block + let label = jit.get_label(&mut asm, lir_block_id, block_id); + asm.write_label(label); + + let block = function.block(block_id); + asm_comment!( + asm, "{block_id}({}): {}", + block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", "), + iseq_get_location(iseq, block.insn_idx), + ); + + // Compile all parameters + for (idx, &insn_id) in block.params().enumerate() { + match function.find(insn_id) { + Insn::Param => { + jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx)); + }, + insn => unreachable!("Non-param insn found in block.params: {insn:?}"), + } + } + + // Compile all instructions + for &insn_id in block.insns() { + let insn = function.find(insn_id); + match insn { + Insn::IfFalse { val, target } => { + let val_opnd = jit.get_opnd(val); + + let lir_target = hir_to_lir[target.target.0].unwrap(); + + let fall_through_target = asm.new_block(block_id, false, rpo_idx); + + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![] + }; + + gen_if_false(&mut asm, val_opnd, branch_edge, fall_through_edge); + asm.set_current_block(fall_through_target); + + let label = jit.get_label(&mut asm, fall_through_target, block_id); + asm.write_label(label); + }, + Insn::IfTrue { val, target } => { + let val_opnd = jit.get_opnd(val); + + let lir_target = hir_to_lir[target.target.0].unwrap(); + + let fall_through_target = asm.new_block(block_id, false, rpo_idx); + + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![] + }; + + gen_if_true(&mut asm, val_opnd, branch_edge, fall_through_edge); + asm.set_current_block(fall_through_target); + + let label = jit.get_label(&mut asm, fall_through_target, block_id); + asm.write_label(label); + } + Insn::Jump(target) => { + let lir_target = hir_to_lir[target.target.0].unwrap(); + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + gen_jump(&mut asm, branch_edge); + }, + _ => { + if let Err(last_snapshot) = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn) { + debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); + gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); + gen_side_exit(&mut jit, &mut asm, &SideExitReason::UnhandledHIRInsn(insn_id), &function.frame_state(last_snapshot)); + // Don't bother generating code after a side-exit. We won't run it. + // TODO(max): Generate ud2 or equivalent. + break; + }; + // It's fine; we generated the instruction + } + } + } + // Make sure the last patch point has enough space to insert a jump + asm.pad_patch_point(); + } + + // Generate code if everything can be compiled + let result = asm.compile(cb); + if let Ok((start_ptr, _)) = result { + if get_option!(perf) { + let start_usize = start_ptr.raw_addr(cb); + let end_usize = cb.get_write_ptr().raw_addr(cb); + let code_size = end_usize - start_usize; + let iseq_name = iseq_get_location(iseq, 0); + register_with_perf(iseq_name, start_usize, code_size); + } + if ZJITState::should_log_compiled_iseqs() { + let iseq_name = iseq_get_location(iseq, 0); + ZJITState::log_compile(iseq_name); + } + } + result.map(|(start_ptr, gc_offsets)| { + // Make sure jit_entry_ptrs can be used as a parallel vector to jit_entry_insns() + jit.jit_entries.sort_by_key(|jit_entry| jit_entry.borrow().jit_entry_idx); + + let jit_entry_ptrs = jit.jit_entries.iter().map(|jit_entry| + jit_entry.borrow().start_addr.get().expect("start_addr should have been set by pos_marker in gen_entry_point") + ).collect(); + (IseqCodePtrs { start_ptr, jit_entry_ptrs }, gc_offsets, jit.iseq_calls) + }) +} + +/// Compile an instruction +fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Result<(), InsnId> { + // Convert InsnId to lir::Opnd + macro_rules! opnd { + ($insn_id:ident) => { + jit.get_opnd($insn_id.clone()) + }; + } + + macro_rules! opnds { + ($insn_ids:ident) => { + { + $insn_ids.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect::<Vec<_>>() + } + }; + } + + macro_rules! no_output { + ($call:expr) => { + { let () = $call; return Ok(()); } + }; + } + + if !matches!(*insn, Insn::Snapshot { .. }) { + asm_comment!(asm, "Insn: {insn_id} {insn}"); + } + + let out_opnd = match insn { + &Insn::Const { val: Const::Value(val) } => gen_const_value(val), + &Insn::Const { val: Const::CPtr(val) } => gen_const_cptr(val), + &Insn::Const { val: Const::CInt64(val) } => gen_const_long(val), + &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val), + &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val), + &Insn::Const { val: Const::CShape(val) } => { + assert_eq!(SHAPE_ID_NUM_BITS, 32); + gen_const_uint32(val.0) + } + Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), + Insn::NewArray { elements, state } => gen_new_array(asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), + Insn::ArrayAref { array, index, .. } => gen_array_aref(asm, opnd!(array), opnd!(index)), + Insn::ArrayAset { array, index, val } => { + no_output!(gen_array_aset(asm, opnd!(array), opnd!(index), opnd!(val))) + } + Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), + Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), + Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), + // concatstrings shouldn't have 0 strings + // If it happens we abort the compilation for now + Insn::StringConcat { strings, state, .. } if strings.is_empty() => return Err(*state), + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), + Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), + Insn::Param => unreachable!("block.insns should not have Insn::Param"), + Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment + &Insn::Send { cd, blockiseq, state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), + Insn::SendWithoutBlockDirect { cme, iseq, recv, args, kw_bits, state, .. } => gen_send_iseq_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), *kw_bits, &function.frame_state(*state), None), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), + // Ensure we have enough room fit ec, self, and arguments + // TODO remove this check when we have stack args (we can use Time.new to test it) + Insn::InvokeBuiltin { bf, state, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return Err(*state), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, *leaf, opnds!(args)), + &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), + Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right)), + Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right)), + Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right)), + Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right)), + Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right)), + Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right)), + Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), + Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), + Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + &Insn::FixnumLShift { left, right, state } => { + // We only create FixnumLShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + } + &Insn::FixnumRShift { left, right } => { + // We only create FixnumRShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_rshift(asm, opnd!(left), shift_amount) + } + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), + Insn::IsNil { val } => gen_isnil(asm, opnd!(val)), + &Insn::IsMethodCfunc { val, cd, cfunc, state: _ } => gen_is_method_cfunc(jit, asm, opnd!(val), cd, cfunc), + &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), + &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), + &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), + Insn::Test { val } => gen_test(asm, opnd!(val)), + Insn::RefineType { val, .. } => opnd!(val), + Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + Insn::GuardTypeNot { val, guard_type, state } => gen_guard_type_not(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + &Insn::GuardBitEquals { val, expected, reason, state } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, &function.frame_state(state)), + &Insn::GuardBlockParamProxy { level, state } => no_output!(gen_guard_block_param_proxy(jit, asm, level, &function.frame_state(state))), + Insn::GuardNotFrozen { recv, state } => gen_guard_not_frozen(jit, asm, opnd!(recv), &function.frame_state(*state)), + Insn::GuardNotShared { recv, state } => gen_guard_not_shared(jit, asm, opnd!(recv), &function.frame_state(*state)), + &Insn::GuardLess { left, right, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::GuardSuperMethodEntry { lep, cme, state } => no_output!(gen_guard_super_method_entry(jit, asm, opnd!(lep), cme, &function.frame_state(state))), + Insn::GetBlockHandler { lep } => gen_get_block_handler(asm, opnd!(lep)), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), + // Give up CCallWithFrame for 7+ args since asm.ccall() supports at most 6 args (recv + args). + // There's no test case for this because no core cfuncs have this many parameters. But C extensions could have such methods. + Insn::CCallWithFrame { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => + gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::CCallWithFrameTooManyArgs), + Insn::CCallWithFrame { cfunc, recv, name, args, cme, state, blockiseq, .. } => + gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *blockiseq, &function.frame_state(*state)), + Insn::CCallVariadic { cfunc, recv, name, args, cme, state, blockiseq, return_type: _, elidable: _ } => { + gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *blockiseq, &function.frame_state(*state)) + } + Insn::GetIvar { self_val, id, ic, state: _ } => gen_getivar(jit, asm, opnd!(self_val), *id, *ic), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + &Insn::GetLocal { ep_offset, level, use_sp, .. } => gen_getlocal(asm, ep_offset, level, use_sp), + &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), + Insn::SideExit { state, reason } => no_output!(gen_side_exit(jit, asm, reason, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type), + Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state)), + Insn::GetSpecialSymbol { symbol_type, state: _ } => gen_getspecial_symbol(asm, *symbol_type), + Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), + &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), + Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), + Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)), + &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), + &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + &Insn::GuardShape { val, shape, state } => gen_guard_shape(jit, asm, opnd!(val), shape, &function.frame_state(state)), + Insn::LoadPC => gen_load_pc(asm), + Insn::LoadEC => gen_load_ec(), + Insn::GetLEP => gen_get_lep(jit, asm), + Insn::LoadSelf => gen_load_self(), + &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), + &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), + &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(asm, opnd!(recv), opnd!(val), function.type_of(val))), + &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), opnd!(buffer), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + &Insn::IsA { val, class } => gen_is_a(asm, opnd!(val), opnd!(class)), + &Insn::ArrayMax { state, .. } + | &Insn::Throw { state, .. } + => return Err(state), + &Insn::IfFalse { .. } | Insn::IfTrue { .. } + | &Insn::Jump { .. } => unreachable!(), + }; + + assert!(insn.has_output(), "Cannot write LIR output of HIR instruction with no output: {insn}"); + + // If the instruction has an output, remember it in jit.opnds + jit.opnds[insn_id.0] = Some(out_opnd); + + Ok(()) +} + +/// Gets the EP of the ISeq of the containing method, or "local level". +/// Equivalent of GET_LEP() macro. +fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd { + // Equivalent of get_lvar_level() in compile.c + fn get_lvar_level(mut iseq: IseqPtr) -> u32 { + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let mut level = 0; + while iseq != local_iseq { + iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; + level += 1; + } + + level + } + + let level = get_lvar_level(jit.iseq); + gen_get_ep(asm, level) +} + +// Get EP at `level` from CFP +fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { + // Load environment pointer EP from CFP into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let mut ep_opnd = asm.load(ep_opnd); + + for _ in 0..level { + // Get the previous EP from the current EP + // See GET_PREV_EP(ep) macro + // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03)) + const UNTAGGING_MASK: Opnd = Opnd::Imm(!0x03); + let offset = SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL; + ep_opnd = asm.load(Opnd::mem(64, ep_opnd, offset)); + ep_opnd = asm.and(ep_opnd, UNTAGGING_MASK); + } + + ep_opnd +} + +fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + // TODO: Specialize for immediate types + // Call rb_vm_objtostring(iseq, recv, cd) + let ret = asm_ccall!(asm, rb_vm_objtostring, VALUE::from(jit.iseq).into(), val, Opnd::const_ptr(cd)); + + // TODO: Call `to_s` on the receiver if rb_vm_objtostring returns Qundef + // Need to replicate what CALL_SIMPLE_METHOD does + asm_comment!(asm, "side-exit if rb_vm_objtostring returns Qundef"); + asm.cmp(ret, Qundef.into()); + asm.je(side_exit(jit, state, ObjToStringFallback)); + + ret +} + +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Opnd { + match op_type as defined_type { + DEFINED_YIELD => { + // `yield` goes to the block handler stowed in the "local" iseq which is + // the current iseq or a parent. Only the "method" iseq type can be passed a + // block handler. (e.g. `yield` in the top level script is a syntax error.) + // + // Similar to gen_is_block_given + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + let lep = gen_get_lep(jit, asm); + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let pushval = asm.load(pushval.into()); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qnil.into(), pushval) + } else { + Qnil.into() + } + } + _ => { + // Save the PC and SP because the callee may allocate or call #respond_to? + gen_prepare_non_leaf_call(jit, asm, state); + + // TODO: Inline the cases for each op_type + // Call vm_defined(ec, reg_cfp, op_type, obj, v) + let def_result = asm_ccall!(asm, rb_vm_defined, EC, CFP, op_type.into(), obj.into(), tested_value); + + asm.cmp(def_result.with_num_bits(8), 0.into()); + asm.csel_ne(pushval.into(), Qnil.into()) + } + } +} + +/// Similar to gen_defined for DEFINED_YIELD +fn gen_is_block_given(asm: &mut Assembler, lep: Opnd) -> Opnd { + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qfalse.into(), Qtrue.into()) +} + +fn gen_unbox_fixnum(asm: &mut Assembler, val: Opnd) -> Opnd { + asm.rshift(val, Opnd::UImm(1)) +} + +/// Get a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. +/// We generate this instruction with level=0 only when the local variable is on the heap, so we +/// can't optimize the level=0 case using the SP register. +fn gen_getlocal(asm: &mut Assembler, local_ep_offset: u32, level: u32, use_sp: bool) -> lir::Opnd { + let local_ep_offset = i32::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_read_from_parent_iseq_local_count); + } + let local = if use_sp { + assert_eq!(level, 0, "use_sp optimization should be used only for level=0 locals"); + let offset = -(SIZEOF_VALUE_I32 * (local_ep_offset + 1)); + Opnd::mem(64, SP, offset) + } else { + let ep = gen_get_ep(asm, level); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + Opnd::mem(64, ep, offset) + }; + asm.load(local) +} + +/// Set a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. +/// We generate this instruction with level=0 only when the local variable is on the heap, so we +/// can't optimize the level=0 case using the SP register. +fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { + let local_ep_offset = c_int::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_write_to_parent_iseq_local_count); + } + let ep = gen_get_ep(asm, level); + + // When we've proved that we're writing an immediate, + // we can skip the write barrier. + if val_type.is_immediate() { + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(64, ep, offset), val); + } else { + // We're potentially writing a reference to an IMEMO/env object, + // so take care of the write barrier with a function. + let local_index = -local_ep_offset; + asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), val); + } +} + +fn gen_guard_block_param_proxy(jit: &JITState, asm: &mut Assembler, level: u32, state: &FrameState) { + // Bail out if the `&block` local variable has been modified + let ep = gen_get_ep(asm, level); + let flags = Opnd::mem(64, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.jnz(side_exit(jit, state, SideExitReason::BlockParamProxyModified)); + + // This handles two cases which are nearly identical + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + // Bail ouf if the block handler is neither ISEQ nor ifunc + let block_handler = asm.load(Opnd::mem(64, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.test(block_handler, 0x1.into()); + asm.jz(side_exit(jit, state, SideExitReason::BlockParamProxyNotIseqOrIfunc)); +} + +fn gen_guard_not_frozen(jit: &JITState, asm: &mut Assembler, recv: Opnd, state: &FrameState) -> Opnd { + let recv = asm.load(recv); + // It's a heap object, so check the frozen flag + let flags = asm.load(Opnd::mem(64, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_FL_FREEZE as u64).into()); + // Side-exit if frozen + asm.jnz(side_exit(jit, state, GuardNotFrozen)); + recv +} + +fn gen_guard_not_shared(jit: &JITState, asm: &mut Assembler, recv: Opnd, state: &FrameState) -> Opnd { + let recv = asm.load(recv); + // It's a heap object, so check the shared flag + let flags = asm.load(Opnd::mem(VALUE_BITS, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_ELTS_SHARED as u64).into()); + asm.jnz(side_exit(jit, state, SideExitReason::GuardNotShared)); + recv +} + +fn gen_guard_less(jit: &JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jge(side_exit(jit, state, SideExitReason::GuardLess)); + left +} + +fn gen_guard_greater_eq(jit: &JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jl(side_exit(jit, state, SideExitReason::GuardGreaterEq)); + left +} + +/// Guard that the method entry at ep[VM_ENV_DATA_INDEX_ME_CREF] matches the expected CME. +/// This ensures we're calling super from the expected method context. +fn gen_guard_super_method_entry( + jit: &JITState, + asm: &mut Assembler, + lep: Opnd, + cme: *const rb_callable_method_entry_t, + state: &FrameState, +) { + asm_comment!(asm, "guard super method entry"); + let ep_me_opnd = Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF); + let ep_me = asm.load(ep_me_opnd); + asm.cmp(ep_me, Opnd::UImm(cme as u64)); + asm.jne(side_exit(jit, state, SideExitReason::GuardSuperMethodEntry)); +} + +/// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). +fn gen_get_block_handler(asm: &mut Assembler, lep: Opnd) -> Opnd { + asm_comment!(asm, "get block handler from LEP"); + asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)) +} + +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { + unsafe extern "C" { + fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; + } + + // Anything could be called on const_missing + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) +} + +fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { + let bit_test: u64 = 0x01 << (index + 1); + asm.test(val, bit_test.into()); + asm.csel_z(Qtrue.into(), Qfalse.into()) +} + +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec<Opnd>) -> lir::Opnd { + assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, + "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", + unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }, + bf.argc); + if leaf { + gen_prepare_leaf_call_with_gc(asm, state); + } else { + // Anything can happen inside builtin functions + gen_prepare_non_leaf_call(jit, asm, state); + } + + let mut cargs = vec![EC]; + cargs.extend(args); + + asm.count_call_to(unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }); + asm.ccall(bf.func_ptr as *const u8, cargs) +} + +/// Record a patch point that should be invalidated on a given invariant +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { + let invariant = *invariant; + let exit = build_side_exit(jit, state); + + // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. + asm.patch_point(Target::SideExit { exit, reason: PatchPoint(invariant) }, invariant, jit.version); +} + +/// This is used by scratch_split to lower PatchPoint into PadPatchPoint and PosMarker. +/// It's called at scratch_split so that we can use the Label after side-exit deduplication in compile_exits. +pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invariant, version: IseqVersionRef) { + let Target::Label(exit_label) = *target else { + unreachable!("PatchPoint's target should have been lowered to Target::Label by compile_exits: {target:?}"); + }; + + // Fill nop instructions if the last patch point is too close. + asm.pad_patch_point(); + + // Remember the current address as a patch point + asm.pos_marker(move |code_ptr, cb| { + let side_exit_ptr = cb.resolve_label(exit_label); + match invariant { + Invariant::BOPRedefined { klass, bop } => { + track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, version); + } + Invariant::MethodRedefined { klass: _, method: _, cme } => { + track_cme_assumption(cme, code_ptr, side_exit_ptr, version); + } + Invariant::StableConstantNames { idlist } => { + track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, version); + } + Invariant::NoTracePoint => { + track_no_trace_point_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoEPEscape(iseq) => { + track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, version); + } + Invariant::SingleRactorMode => { + track_single_ractor_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoSingletonClass { klass } => { + track_no_singleton_class_assumption(klass, code_ptr, side_exit_ptr, version); + } + } + }); +} + +/// Generate code for a C function call that pushes a frame +fn gen_ccall_with_frame( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + blockiseq: Option<IseqPtr>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + let caller_stack_size = state.stack().len() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (and block arguments if any) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(block_iseq) = blockiseq { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + pc: PC_POISON, + specval: block_handler_specval, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, cfunc_args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know +/// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. +fn gen_ccall(asm: &mut Assembler, cfunc: *const u8, name: ID, recv: Opnd, args: Vec<Opnd>) -> lir::Opnd { + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + asm.ccall(cfunc, cfunc_args) +} + +/// Generate code for a variadic C function call +/// func(int argc, VALUE *argv, VALUE recv) +fn gen_ccall_variadic( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + blockiseq: Option<IseqPtr>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + + // Compute the caller's stack size after consuming recv and args. + // state.stack() includes recv + args, so subtract both. + let caller_stack_size = state.stack_size() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (like gen_ccall_with_frame does) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(block_iseq) = blockiseq { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + pc: PC_POISON, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let argv_ptr = gen_push_opnds(asm, &args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, vec![args.len().into(), argv_ptr, recv]); + gen_pop_opnds(asm, &args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Emit an uncached instance variable lookup +fn gen_getivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry) -> Opnd { + if ic.is_null() { + asm_ccall!(asm, rb_ivar_get, recv, id.0.into()) + } else { + let iseq = Opnd::Value(jit.iseq.into()); + asm_ccall!(asm, rb_vm_getinstancevariable, iseq, recv, id.0.into(), Opnd::const_ptr(ic)) + } +} + +/// Emit an uncached instance variable store +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { + // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. + gen_prepare_non_leaf_call(jit, asm, state); + if ic.is_null() { + asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); + } else { + let iseq = Opnd::Value(jit.iseq.into()); + asm_ccall!(asm, rb_vm_setinstancevariable, iseq, recv, id.0.into(), val, Opnd::const_ptr(ic)); + } +} + +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(jit.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) +} + +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(jit.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); +} + +/// Look up global variables +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { + // `Warning` module's method `warn` can be called when reading certain global variables + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_get, id.0.into()) +} + +/// Intern a string +fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_str_intern, val) +} + +/// Set global variables +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { + // When trace_var is used, setting a global variable can cause exceptions + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_set, id.0.into(), val); +} + +/// Side-exit into the interpreter +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) { + asm.jmp(side_exit(jit, state, *reason)); +} + +/// Emit a special object lookup +fn gen_putspecialobject(asm: &mut Assembler, value_type: SpecialObjectType) -> Opnd { + // Get the EP of the current CFP and load it into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let ep_reg = asm.load(ep_opnd); + + asm_ccall!(asm, rb_vm_get_special_object, ep_reg, Opnd::UImm(u64::from(value_type))) +} + +fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol) -> Opnd { + // Fetch a "special" backref based on the symbol type + + let backref = asm_ccall!(asm, rb_backref_get,); + + match symbol_type { + SpecialBackrefSymbol::LastMatch => { + asm_ccall!(asm, rb_reg_last_match, backref) + } + SpecialBackrefSymbol::PreMatch => { + asm_ccall!(asm, rb_reg_match_pre, backref) + } + SpecialBackrefSymbol::PostMatch => { + asm_ccall!(asm, rb_reg_match_post, backref) + } + SpecialBackrefSymbol::LastGroup => { + asm_ccall!(asm, rb_reg_match_last, backref) + } + } +} + +fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> Opnd { + // Fetch the N-th match from the last backref based on type shifted by 1 + + let backref = asm_ccall!(asm, rb_backref_get,); + + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) +} + +fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameState) { + // Check for interrupts + // see RUBY_VM_CHECK_INTS(ec) macro + asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); + // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, + // signal_exec, or rb_postponed_job_flush. + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); + asm.test(interrupt_flag, interrupt_flag); + asm.jnz(side_exit(jit, state, SideExitReason::Interrupt)); +} + +fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_resurrect, val) +} + +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aref, hash, key) +} + +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aset, hash, key, val); +} + +fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_push, array, val); +} + +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) +} + +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) +} + +fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) -> lir::Opnd { + asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) +} + +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_ary_concat, left, right); +} + +fn gen_guard_shape(jit: &mut JITState, asm: &mut Assembler, val: Opnd, shape: ShapeId, state: &FrameState) -> Opnd { + gen_incr_counter(asm, Counter::guard_shape_count); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + let val = asm.load(val); + let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, val, shape_id_offset); + asm.cmp(shape_opnd, Opnd::UImm(shape.0 as u64)); + asm.jne(side_exit(jit, state, SideExitReason::GuardShape(shape))); + val +} + +fn gen_load_pc(asm: &mut Assembler) -> Opnd { + asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC)) +} + +fn gen_load_ec() -> Opnd { + EC +} + +fn gen_load_self() -> Opnd { + Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF) +} + +fn gen_load_field(asm: &mut Assembler, recv: Opnd, id: ID, offset: i32, return_type: Type) -> Opnd { + asm_comment!(asm, "Load field id={} offset={}", id.contents_lossy(), offset); + let recv = asm.load(recv); + asm.load(Opnd::mem(return_type.num_bits(), recv, offset)) +} + +fn gen_store_field(asm: &mut Assembler, recv: Opnd, id: ID, offset: i32, val: Opnd, val_type: Type) { + asm_comment!(asm, "Store field id={} offset={}", id.contents_lossy(), offset); + let recv = asm.load(recv); + asm.store(Opnd::mem(val_type.num_bits(), recv, offset), val); +} + +fn gen_write_barrier(asm: &mut Assembler, recv: Opnd, val: Opnd, val_type: Type) { + // See RB_OBJ_WRITE/rb_obj_write: it's just assignment and rb_obj_written(). + // rb_obj_written() does: if (!RB_SPECIAL_CONST_P(val)) { rb_gc_writebarrier(recv, val); } + if !val_type.is_immediate() { + asm_comment!(asm, "Write barrier"); + let recv = asm.load(recv); + asm_ccall!(asm, rb_zjit_writebarrier_check_immediate, recv, val); + } +} + +/// Compile an interpreter entry block to be inserted into an ISEQ +fn gen_entry_prologue(asm: &mut Assembler) { + asm_comment!(asm, "ZJIT entry trampoline"); + // Save the registers we'll use for CFP, EP, SP + asm.frame_setup(lir::JIT_PRESERVED_REGS); + + // EC and CFP are passed as arguments + asm.mov(EC, C_ARG_OPNDS[0]); + asm.mov(CFP, C_ARG_OPNDS[1]); + + // Load the current SP from the CFP into REG_SP + asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +} + +/// Compile a constant +fn gen_const_value(val: VALUE) -> lir::Opnd { + // Just propagate the constant value and generate nothing + Opnd::Value(val) +} + +/// Compile Const::CPtr +fn gen_const_cptr(val: *const u8) -> lir::Opnd { + Opnd::const_ptr(val) +} + +fn gen_const_long(val: i64) -> lir::Opnd { + Opnd::Imm(val) +} + +fn gen_const_uint16(val: u16) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +fn gen_const_uint32(val: u32) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +/// Compile a basic block argument +fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { + // Allocate a register or a stack slot + match Assembler::param_opnd(idx) { + // If it's a register, insert LiveReg instruction to reserve the register + // in the register pool for register allocation. + param @ Opnd::Reg(_) => asm.live_reg_opnd(param), + param => param, + } +} + +/// Compile a jump to a basic block +fn gen_jump(asm: &mut Assembler, branch: lir::BranchEdge) { + // Jump to the basic block + asm.jmp(Target::Block(branch)); +} + +/// Compile a conditional branch to a basic block +fn gen_if_true(asm: &mut Assembler, val: lir::Opnd, branch: lir::BranchEdge, fall_through: lir::BranchEdge) { + // If val is zero, move on to the next instruction. + asm.test(val, val); + asm.jz(Target::Block(fall_through)); + asm.jmp(Target::Block(branch)); +} + +/// Compile a conditional branch to a basic block +fn gen_if_false(asm: &mut Assembler, val: lir::Opnd, branch: lir::BranchEdge, fall_through: lir::BranchEdge) { + // If val is not zero, move on to the next instruction. + asm.test(val, val); + asm.jnz(Target::Block(fall_through)); + asm.jmp(Target::Block(branch)); +} + +/// Compile a dynamic dispatch with block +fn gen_send( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_send, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch with `...` +fn gen_send_forward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_sendforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch without block +fn gen_send_without_block( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_send_without_block, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +/// Compile a direct call to an ISEQ method. +/// If `block_handler` is provided, it's used as the specval for the new frame (for forwarding blocks). +/// Otherwise, `VM_BLOCK_HANDLER_NONE` is used. +fn gen_send_iseq_direct( + cb: &mut CodeBlock, + jit: &mut JITState, + asm: &mut Assembler, + cme: *const rb_callable_method_entry_t, + iseq: IseqPtr, + recv: Opnd, + args: Vec<Opnd>, + kw_bits: u32, + state: &FrameState, + block_handler: Option<Opnd>, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::iseq_optimized_send_count); + + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); + let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); + gen_stack_overflow_check(jit, asm, state, stack_growth); + + // Save cfp->pc and cfp->sp for the caller frame + // Can't use gen_prepare_non_leaf_call because we need special SP math. + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver + + gen_spill_locals(jit, asm, state); + gen_spill_stack(jit, asm, state); + + let (frame_type, specval) = if VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(cme) } { + // Extract EP from the Proc instance + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let bmethod_frame_type = VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA; + // Tag the captured EP like VM_GUARDED_PREV_EP() in vm_call_iseq_bmethod() + let bmethod_specval = (capture.ep.addr() | 1).into(); + (bmethod_frame_type, bmethod_specval) + } else { + let specval = block_handler.unwrap_or_else(|| VM_BLOCK_HANDLER_NONE.into()); + (VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, specval) + }; + + // Set up the new frame + // TODO: Lazily materialize caller frames on side exits or when needed + gen_push_frame(asm, args.len(), state, ControlFrame { + recv, + iseq: Some(iseq), + cme, + frame_type, + pc: None, + specval, + }); + + // Write "keyword_bits" to the callee's frame if the callee accepts keywords. + // This is a synthetic local/parameter that the callee reads via checkkeyword to determine + // which optional keyword arguments need their defaults evaluated. + // We write this to the local table slot at bits_start so that: + // 1. The interpreter can read it via checkkeyword if we side-exit + // 2. The JIT entry can read it via GetLocal + if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + let bits_start = unsafe { (*keyword).bits_start } as usize; + let unspecified_bits = VALUE::fixnum_from_usize(kw_bits as usize); + let bits_offset = (state.stack().len() - args.len() + bits_start) * SIZEOF_VALUE; + asm_comment!(asm, "write keyword bits to callee frame"); + asm.store(Opnd::mem(64, SP, bits_offset as i32), unspecified_bits.into()); + } + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (state.stack().len() + local_size - args.len() + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + // Set up arguments + let mut c_args = vec![recv]; + c_args.extend(&args); + + let params = unsafe { iseq.params() }; + let num_optionals_passed = if params.flags.has_opt() != 0 { + // See vm_call_iseq_setup_normal_opt_start in vm_inshelper.c + let lead_num = params.lead_num as u32; + let opt_num = params.opt_num as u32; + let keyword = params.keyword; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } } as u32; + assert!(args.len() as u32 <= lead_num + opt_num + kw_total_num); + // For computing optional positional entry point, only count positional args + let positional_argc = args.len() as u32 - kw_total_num; + let num_optionals_passed = positional_argc.saturating_sub(lead_num); + num_optionals_passed + } else { + 0 + }; + + // Fill non-parameter locals with nil (they may be read by eval before being written) + let num_params = params.size.to_usize(); + if local_size > num_params { + asm_comment!(asm, "initialize non-parameter locals to nil"); + for local_idx in num_params..local_size { + let offset = local_size_and_idx_to_bp_offset(local_size, local_idx); + asm.store(Opnd::mem(64, SP, -offset * SIZEOF_VALUE_I32), Qnil.into()); + } + } + + // Make a method call. The target address will be rewritten once compiled. + let iseq_call = IseqCall::new(iseq, num_optionals_passed); + let dummy_ptr = cb.get_write_ptr().raw_ptr(cb); + jit.iseq_calls.push(iseq_call.clone()); + let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call); + + // If a callee side-exits, i.e. returns Qundef, propagate the return value to the caller. + // The caller will side-exit the callee into the interpreter. + // TODO: Let side exit code pop all JIT frames to optimize away this cmp + je. + asm_comment!(asm, "side-exit if callee side-exits"); + asm.cmp(ret, Qundef.into()); + // Restore the C stack pointer on exit + asm.je(ZJITState::get_exit_trampoline().into()); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + ret +} + +/// Compile for invokeblock +fn gen_invokeblock( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeblock"); + unsafe extern "C" { + fn rb_vm_invokeblock(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokeblock, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +fn gen_invokeproc( + jit: &mut JITState, + asm: &mut Assembler, + recv: Opnd, + args: Vec<Opnd>, + kw_splat: bool, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeproc"); + + let argv_ptr = gen_push_opnds(asm, &args); + let kw_splat_opnd = Opnd::Imm(i64::from(kw_splat)); + let result = asm_ccall!( + asm, + rb_optimized_call, + recv, + EC, + args.len().into(), + argv_ptr, + kw_splat_opnd, + VM_BLOCK_HANDLER_NONE.into() + ); + gen_pop_opnds(asm, &args); + + result +} + +/// Compile a dynamic dispatch for `super` +fn gen_invokesuper( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch"); + unsafe extern "C" { + fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuper, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a string resurrection +fn gen_string_copy(asm: &mut Assembler, recv: Opnd, chilled: bool, state: &FrameState) -> Opnd { + // TODO: split rb_ec_str_resurrect into separate functions + gen_prepare_leaf_call_with_gc(asm, state); + let chilled = if chilled { Opnd::Imm(1) } else { Opnd::Imm(0) }; + asm_ccall!(asm, rb_ec_str_resurrect, EC, recv, chilled) +} + +/// Compile an array duplication instruction +fn gen_array_dup( + asm: &mut Assembler, + val: lir::Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_ary_resurrect, val) +} + +/// Compile a new array instruction +fn gen_new_array( + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + let num: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + if elements.is_empty() { + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, 0i64.into(), Opnd::UImm(0)) + } else { + let argv = gen_push_opnds(asm, &elements); + let new_array = asm_ccall!(asm, rb_ec_ary_new_from_values, EC, num.into(), argv); + gen_pop_opnds(asm, &elements); + new_array + } +} + +/// Compile array access (`array[index]`) +fn gen_array_aref( + asm: &mut Assembler, + array: Opnd, + index: Opnd, +) -> lir::Opnd { + let unboxed_idx = asm.load(index); + let array = asm.load(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.load(Opnd::mem(VALUE_BITS, elem_ptr, 0)) +} + +fn gen_array_aset( + asm: &mut Assembler, + array: Opnd, + index: Opnd, + val: Opnd, +) { + let unboxed_idx = asm.load(index); + let array = asm.load(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.store(Opnd::mem(VALUE_BITS, elem_ptr, 0), val); +} + +fn gen_array_pop(asm: &mut Assembler, array: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_pop, array) +} + +fn gen_array_length(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let array = asm.load(array); + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + let embedded_len = asm.and(flags, (RARRAY_EMBED_LEN_MASK as u64).into()); + let embedded_len = asm.rshift(embedded_len, (RARRAY_EMBED_LEN_SHIFT as u64).into()); + // cmov between the embedded length and heap length depending on the embed flag + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_len = Opnd::mem(c_long::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_LEN); + asm.csel_nz(embedded_len, heap_len) +} + +fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_ptr = Opnd::mem(usize::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_PTR); + let embedded_ptr = asm.lea(Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RARRAY_AS_ARY)); + asm.csel_nz(embedded_ptr, heap_ptr) +} + +/// Compile opt_newarray_hash - create a hash from array elements +fn gen_opt_newarray_hash( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + // `Array#hash` will hash the elements of the array. + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_hash(ec: EcPtr, array_len: u32, elts: *const VALUE) -> VALUE; + } + + asm.ccall( + rb_vm_opt_newarray_hash as *const u8, + vec![EC, (array_len as u32).into(), elements_ptr], + ) +} + +fn gen_array_include( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the target. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 1; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_include_p(ec: EcPtr, num: c_long, elts: *const VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_include_p, + EC, array_len.into(), elements_ptr, target + ) +} + +fn gen_array_pack_buffer( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + fmt: Opnd, + buffer: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the fmt, followed by the buffer. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 2; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_pack_buffer(ec: EcPtr, num: c_long, elts: *const VALUE, fmt: VALUE, buffer: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_pack_buffer, + EC, array_len.into(), elements_ptr, fmt, buffer + ) +} + +fn gen_dup_array_include( + jit: &JITState, + asm: &mut Assembler, + ary: VALUE, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + unsafe extern "C" { + fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_duparray_include_p, + EC, ary.into(), target + ) +} + +fn gen_is_a(asm: &mut Assembler, obj: Opnd, class: Opnd) -> lir::Opnd { + let builtin_type = match class { + Opnd::Value(value) if value == unsafe { rb_cString } => Some(RUBY_T_STRING), + Opnd::Value(value) if value == unsafe { rb_cArray } => Some(RUBY_T_ARRAY), + Opnd::Value(value) if value == unsafe { rb_cHash } => Some(RUBY_T_HASH), + _ => None + }; + + if let Some(builtin_type) = builtin_type { + asm_comment!(asm, "IsA by matching builtin type"); + let ret_label = asm.new_label("is_a_ret"); + let false_label = asm.new_label("is_a_false"); + + let val = match obj { + Opnd::Reg(_) | Opnd::VReg { .. } => obj, + _ => asm.load(obj), + }; + + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(ret_label.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(false_label.clone()); + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let obj_builtin_type = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(obj_builtin_type, Opnd::UImm(builtin_type as u64)); + asm.jmp(ret_label.clone()); + + // If we get here then the value was false, unset the Z flag + // so that csel_e will select false instead of true + asm.write_label(false_label); + asm.test(Opnd::UImm(1), Opnd::UImm(1)); + + asm.write_label(ret_label); + asm.csel_e(Qtrue.into(), Qfalse.into()) + } else { + asm_ccall!(asm, rb_obj_is_kind_of, obj, class) + } +} + +/// Compile a new hash instruction +fn gen_new_hash( + jit: &mut JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let cap: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + let new_hash = asm_ccall!(asm, rb_hash_new_with_size, lir::Opnd::Imm(cap)); + + if !elements.is_empty() { + let argv = gen_push_opnds(asm, &elements); + asm_ccall!(asm, rb_hash_bulk_insert, elements.len().into(), argv, new_hash); + + gen_pop_opnds(asm, &elements); + } + + new_hash +} + +/// Compile a new range instruction +fn gen_new_range( + jit: &JITState, + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + // Sometimes calls `low.<=>(high)` + gen_prepare_non_leaf_call(jit, asm, state); + + // Call rb_range_new(low, high, flag) + asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) +} + +fn gen_new_range_fixnum( + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) +} + +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_obj_alloc, val) +} + +fn gen_object_alloc_class(asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { + // Allocating an object for a known class with default allocator is leaf; see doc for + // `ObjectAllocClass`. + gen_prepare_leaf_call_with_gc(asm, state); + if unsafe { rb_zjit_class_has_default_allocator(class) } { + // TODO(max): inline code to allocate an instance + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + } else { + assert!(class_has_leaf_allocator(class), "class passed to ObjectAllocClass must have a leaf allocator"); + let alloc_func = unsafe { rb_zjit_class_get_alloc_func(class) }; + assert!(alloc_func.is_some(), "class {} passed to ObjectAllocClass must have an allocator", get_class_name(class)); + asm_comment!(asm, "call allocator for class {}", get_class_name(class)); + asm.count_call_to(&format!("{}::allocator", get_class_name(class))); + asm.ccall(alloc_func.unwrap() as *const u8, vec![class.into()]) + } +} + +/// Compile a frame setup. If jit_entry_idx is Some, remember the address of it as a JIT entry. +fn gen_entry_point(jit: &mut JITState, asm: &mut Assembler, jit_entry_idx: Option<usize>) { + if let Some(jit_entry_idx) = jit_entry_idx { + let jit_entry = JITEntry::new(jit_entry_idx); + jit.jit_entries.push(jit_entry.clone()); + asm.pos_marker(move |code_ptr, _| { + jit_entry.borrow_mut().start_addr.set(Some(code_ptr)); + }); + } + asm.frame_setup(&[]); +} + +/// Compile code that exits from JIT code with a return value +fn gen_return(asm: &mut Assembler, val: lir::Opnd) { + // Pop the current frame (ec->cfp++) + // Note: the return PC is already in the previous CFP + asm_comment!(asm, "pop stack frame"); + let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, incr_cfp); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + // Order here is important. Because we're about to tear down the frame, + // we need to load the return value, which might be part of the frame. + asm.load_into(C_RET_OPND, val); + + // Return from the function + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(C_RET_OPND); +} + +/// Compile Fixnum + Fixnum +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Add left + right and test for overflow + let left_untag = asm.sub(left, Opnd::Imm(1)); + let out_val = asm.add(left_untag, right); + asm.jo(side_exit(jit, state, FixnumAddOverflow)); + + out_val +} + +/// Compile Fixnum - Fixnum +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Subtract left - right and test for overflow + let val_untag = asm.sub(left, right); + asm.jo(side_exit(jit, state, FixnumSubOverflow)); + asm.add(val_untag, Opnd::Imm(1)) +} + +/// Compile Fixnum * Fixnum +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Do some bitwise gymnastics to handle tag bits + // x * y is translated to (x >> 1) * (y - 1) + 1 + let left_untag = asm.rshift(left, Opnd::UImm(1)); + let right_untag = asm.sub(right, Opnd::UImm(1)); + let out_val = asm.mul(left_untag, right_untag); + + // Test for overflow + asm.jo_mul(side_exit(jit, state, FixnumMultOverflow)); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum / Fixnum +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + // Side exit if rhs is 0 + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(side_exit(jit, state, FixnumDivByZero)); + asm_ccall!(asm, rb_jit_fix_div_fix, left, right) +} + +/// Compile Fixnum == Fixnum +fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_e(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum != Fixnum +fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum < Fixnum +fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_l(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum <= Fixnum +fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_le(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum > Fixnum +fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_g(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum >= Fixnum +fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ge(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum & Fixnum +fn gen_fixnum_and(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.and(left, right) +} + +/// Compile Fixnum | Fixnum +fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) +} + +/// Compile Fixnum ^ Fixnum +fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + // XOR and then re-tag the resulting fixnum + let out_val = asm.xor(left, right); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum << Fixnum +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit + let out_val = asm.lshift(in_val, shift_amount.into()); + let unshifted = asm.rshift(out_val, shift_amount.into()); + asm.cmp(in_val, unshifted); + asm.jne(side_exit(jit, state, FixnumLShiftOverflow)); + // Re-tag the output value + let out_val = asm.add(out_val, 1.into()); + out_val +} + +/// Compile Fixnum >> Fixnum +fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let result = asm.rshift(left, shift_amount.into()); + // Re-tag the output value + asm.or(result, 1.into()) +} + +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Check for left % 0, which raises ZeroDivisionError + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(side_exit(jit, state, FixnumModByZero)); + asm_ccall!(asm, rb_fix_mod_fix, left, right) +} + +fn gen_fixnum_aref(asm: &mut Assembler, recv: lir::Opnd, index: lir::Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_fix_aref, recv, index) +} + +// Compile val == nil +fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.cmp(val, Qnil.into()); + // TODO: Implement and use setcc + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_method_cfunc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, cd: *const rb_call_data, cfunc: *const u8) -> lir::Opnd { + unsafe extern "C" { + fn rb_vm_method_cfunc_is(iseq: IseqPtr, cd: *const rb_call_data, recv: VALUE, cfunc: *const u8) -> VALUE; + } + asm_ccall!(asm, rb_vm_method_cfunc_is, VALUE::from(jit.iseq).into(), Opnd::const_ptr(cd), val, Opnd::const_ptr(cfunc)) +} + +fn gen_is_bit_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_bit_not_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.test(val, val); + asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) +} + +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Load the value, then test for overflow and tag it + let val = asm.load(val); + let shifted = asm.lshift(val, Opnd::UImm(1)); + asm.jo(side_exit(jit, state, BoxFixnumOverflow)); + asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) +} + +fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_obj_as_string_result, str, val) +} + +/// Evaluate if a value is truthy +/// Produces a CBool type (0 or 1) +/// In Ruby, only nil and false are falsy +/// Everything else evaluates to true +fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + // Test if any bit (outside of the Qnil bit) is on + // See RB_TEST(), include/ruby/internal/special_consts.h + asm.test(val, Opnd::Imm(!Qnil.as_i64())); + asm.csel_e(0.into(), 1.into()) +} + +/// Compile a type check with a side exit +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { + gen_incr_counter(asm, Counter::guard_type_count); + if guard_type.is_subtype(types::Fixnum) { + asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + asm.jz(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::Flonum) { + // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG + let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); + asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::StaticSymbol) { + // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG + // Use 8-bit comparison like YJIT does. GuardType should not be used + // for a known VALUE, which with_num_bits() does not support. + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::NilClass) { + asm.cmp(val, Qnil.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::TrueClass) { + asm.cmp(val, Qtrue.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::FalseClass) { + asm.cmp(val, Qfalse.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {guard_type}"); + } else if let Some(expected_class) = guard_type.runtime_exact_ruby_class() { + asm_comment!(asm, "guard exact class for non-immediate types"); + + // If val isn't in a register, load it to use it as the base of Opnd::mem later. + // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + // Check if it's a special constant + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(side_exit.clone()); + + // Check if it's false + asm.cmp(val, Qfalse.into()); + asm.je(side_exit.clone()); + + // Load the class from the object's klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); + + asm.cmp(klass, Opnd::Value(expected_class)); + asm.jne(side_exit); + } else if guard_type.is_subtype(types::String) { + let side = side_exit(jit, state, GuardType(guard_type)); + + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(side.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(side.clone()); + + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(RUBY_T_STRING as u64)); + asm.jne(side); + } else if guard_type.bit_equal(types::HeapBasicObject) { + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.cmp(val, Opnd::Value(Qfalse)); + asm.je(side_exit.clone()); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(side_exit); + } else { + unimplemented!("unsupported type: {guard_type}"); + } + val +} + +fn gen_guard_type_not(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { + if guard_type.is_subtype(types::String) { + // We only exit if val *is* a String. Otherwise we fall through. + let cont = asm.new_label("guard_type_not_string_cont"); + let side = side_exit(jit, state, GuardTypeNot(guard_type)); + + // Continue if special constant (not string) + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(cont.clone()); + + // Continue if false (not string) + asm.cmp(val, Qfalse.into()); + asm.je(cont.clone()); + + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(RUBY_T_STRING as u64)); + asm.je(side); + + // Otherwise (non-string heap object), continue. + asm.write_label(cont); + } else { + unimplemented!("unsupported type: {guard_type}"); + } + val +} + +/// Compile an identity check with a side exit +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let expected_opnd: Opnd = match expected { + crate::hir::Const::Value(v) => { Opnd::Value(v) } + crate::hir::Const::CInt64(v) => { v.into() } + crate::hir::Const::CShape(v) => { Opnd::UImm(v.0 as u64) } + _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), + }; + asm.cmp(val, expected_opnd); + asm.jnz(side_exit(jit, state, reason)); + val +} + +/// Generate code that records unoptimized C functions if --zjit-stats is enabled +fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) { + if get_option!(stats) { + asm.incr_counter(Opnd::const_ptr(counter_ptr as *const u8), Opnd::UImm(1)); + } +} + +/// Generate code that increments a counter if --zjit-stats +fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { + if get_option!(stats) { + let ptr = counter_ptr(counter); + gen_incr_counter_ptr(asm, ptr); + } +} + +/// Increment a counter for each DynamicSendReason. If the variant has +/// a counter prefix to break down the details, increment that as well. +fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReason) { + gen_incr_counter(asm, send_fallback_counter(reason)); + + use SendFallbackReason::*; + match reason { + Uncategorized(opcode) => { + gen_incr_counter_ptr(asm, send_fallback_counter_ptr_for_opcode(opcode)); + } + SendWithoutBlockNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_method_type(method_type)); + } + SendWithoutBlockNotOptimizedMethodTypeOptimized(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_optimized_method_type(method_type)); + } + SendNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_method_type(method_type)); + } + SuperNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_super_method_type(method_type)); + } + _ => {} + } +} + +/// Save only the PC to CFP. Use this when you need to call gen_save_sp() +/// immediately after with a custom stack size (e.g., gen_ccall_with_frame +/// adjusts SP to exclude receiver and arguments). +fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { + let opcode: usize = state.get_opcode().try_into().unwrap(); + let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; + + gen_incr_counter(asm, Counter::vm_write_pc_count); + asm_comment!(asm, "save PC to CFP"); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); +} + +/// Save the current PC on the CFP as a preparation for calling a C function +/// that may allocate objects and trigger GC. Use gen_prepare_non_leaf_call() +/// if it may raise exceptions or call arbitrary methods. +/// +/// Unlike YJIT, we don't need to save the stack slots to protect them from GC +/// because the backend spills all live registers onto the C stack on CCall. +/// However, to avoid marking uninitialized stack slots, this also updates SP, +/// which may have cfp->sp for a past frame or a past non-leaf call. +fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState, leaf: bool) { + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack_size()); + if leaf { + asm.expect_leaf_ccall(state.stack_size()); + } +} + +fn gen_prepare_leaf_call_with_gc(asm: &mut Assembler, state: &FrameState) { + // In gen_prepare_call_with_gc(), we update cfp->sp for leaf calls too. + // + // Here, cfp->sp may be pointing to either of the following: + // 1. cfp->sp for a past frame, which gen_push_frame() skips to initialize + // 2. cfp->sp set by gen_prepare_non_leaf_call() for the current frame + // + // When (1), to avoid marking dead objects, we need to set cfp->sp for the current frame. + // When (2), setting cfp->sp at gen_push_frame() and not updating cfp->sp here could lead to + // keeping objects longer than it should, so we set cfp->sp at every call of this function. + // + // We use state.without_stack() to pass stack_size=0 to gen_save_sp() because we don't write + // VM stack slots on leaf calls, which leaves those stack slots uninitialized. ZJIT keeps + // live objects on the C stack, so they are protected from GC properly. + gen_prepare_call_with_gc(asm, &state.without_stack(), true); +} + +/// Save the current SP on the CFP +fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { + // Update cfp->sp which will be read by the interpreter. We also have the SP register in JIT + // code, and ZJIT's codegen currently assumes the SP register doesn't move, e.g. gen_param(). + // So we don't update the SP register here. We could update the SP register to avoid using + // an extra register for asm.lea(), but you'll need to manage the SP offset like YJIT does. + gen_incr_counter(asm, Counter::vm_write_sp_count); + asm_comment!(asm, "save SP to CFP: {}", stack_size); + let sp_addr = asm.lea(Opnd::mem(64, SP, stack_size as i32 * SIZEOF_VALUE_I32)); + let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); + asm.mov(cfp_sp, sp_addr); +} + +/// Spill locals onto the stack. +fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Avoid spilling locals that have been spilled before and not changed. + gen_incr_counter(asm, Counter::vm_write_locals_count); + asm_comment!(asm, "spill locals"); + for (idx, &insn_id) in state.locals().enumerate() { + asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(jit.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Spill the virtual stack onto the stack. +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // This function does not call gen_save_sp() at the moment because + // gen_send_without_block_direct() spills stack slots above SP for arguments. + gen_incr_counter(asm, Counter::vm_write_stack_count); + asm_comment!(asm, "spill stack"); + for (idx, &insn_id) in state.stack().enumerate() { + asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Prepare for calling a C function that may call an arbitrary method. +/// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Lazily materialize caller frames when needed + // Save PC for backtraces and allocation tracing + // and SP to avoid marking uninitialized stack slots + gen_prepare_call_with_gc(asm, state, false); + + // Spill the virtual stack in case it raises an exception + // and the interpreter uses the stack for handling the exception + gen_spill_stack(jit, asm, state); + + // Spill locals in case the method looks at caller Bindings + gen_spill_locals(jit, asm, state); +} + +/// Frame metadata written by gen_push_frame() +struct ControlFrame { + recv: Opnd, + iseq: Option<IseqPtr>, + cme: *const rb_callable_method_entry_t, + frame_type: u32, + /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. + /// For the type of frames we push, block handler or the parent EP. + specval: lir::Opnd, + pc: Option<*const VALUE>, +} + +/// Compile an interpreter frame +fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: ControlFrame) { + // Locals are written by the callee frame on side-exits or non-leaf calls + + // See vm_push_frame() for details + asm_comment!(asm, "push cme, specval, frame type"); + // ep[-2]: cref of cme + let local_size = if let Some(iseq) = frame.iseq { + (unsafe { get_iseq_body_local_table_size(iseq) }) as i32 + } else { + 0 + }; + let ep_offset = state.stack().len() as i32 + local_size - argc as i32 + VM_ENV_DATA_SIZE as i32 - 1; + // ep[-2]: CME + asm.store(Opnd::mem(64, SP, (ep_offset - 2) * SIZEOF_VALUE_I32), VALUE::from(frame.cme).into()); + // ep[-1]: specval + asm.store(Opnd::mem(64, SP, (ep_offset - 1) * SIZEOF_VALUE_I32), frame.specval); + // ep[0]: ENV_FLAGS + asm.store(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32), frame.frame_type.into()); + + // Write to the callee CFP + fn cfp_opnd(offset: i32) -> Opnd { + Opnd::mem(64, CFP, offset - (RUBY_SIZEOF_CONTROL_FRAME as i32)) + } + + asm_comment!(asm, "push callee control frame"); + + if let Some(iseq) = frame.iseq { + // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits, non-leaf calls, or calls with GC + // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits, non-leaf calls, or calls with GC + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + } else { + // C frames don't have a PC and ISEQ in normal operation. + // When runtime checks are enabled we poison the PC so accidental reads stand out. + if let Some(pc) = frame.pc { + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); + } + + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); + let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); +} + +/// Stack overflow check: fails if CFP<=SP at any point in the callee. +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { + asm_comment!(asm, "stack overflow check"); + // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 + // adds to the margin another control frame with `&bounds[1]`. + const { assert!(RUBY_SIZEOF_CONTROL_FRAME % SIZEOF_VALUE == 0, "sizeof(rb_control_frame_t) is a multiple of sizeof(VALUE)"); } + let cfp_growth = 2 * (RUBY_SIZEOF_CONTROL_FRAME / SIZEOF_VALUE); + let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; + let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); + asm.cmp(CFP, stack_limit); + asm.jbe(side_exit(jit, state, StackOverflow)); +} + + +/// Inverse of ep_offset_to_local_idx(). See ep_offset_to_local_idx() for details. +pub fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }; + local_size_and_idx_to_ep_offset(local_size.to_usize(), local_idx) +} + +/// Convert the number of locals and a local index to an offset from the EP +pub fn local_size_and_idx_to_ep_offset(local_size: usize, local_idx: usize) -> i32 { + local_size as i32 - local_idx as i32 - 1 + VM_ENV_DATA_SIZE as i32 +} + +/// Convert the number of locals and a local index to an offset from the BP. +/// We don't move the SP register after entry, so we often use SP as BP. +pub fn local_size_and_idx_to_bp_offset(local_size: usize, local_idx: usize) -> i32 { + local_size_and_idx_to_ep_offset(local_size, local_idx) + 1 +} + +/// Convert ISEQ into High-level IR +fn compile_iseq(iseq: IseqPtr) -> Result<Function, CompileError> { + // Convert ZJIT instructions back to bare instructions + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + + // Reject ISEQs with very large temp stacks. + // We cannot encode too large offsets to access locals in arm64. + let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; + if stack_max >= i8::MAX as u32 { + debug!("ISEQ stack too large: {stack_max}"); + return Err(CompileError::IseqStackTooLarge); + } + + let mut function = match iseq_to_hir(iseq) { + Ok(function) => function, + Err(err) => { + debug!("ZJIT: iseq_to_hir: {err:?}: {}", iseq_get_location(iseq, 0)); + return Err(CompileError::ParseError(err)); + } + }; + if !get_option!(disable_hir_opt) { + function.optimize(); + } + function.dump_hir(); + Ok(function) +} + +/// Build a Target::SideExit +fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, state); + Target::SideExit { exit, reason } +} + +/// Build a side-exit context +fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { + let mut stack = Vec::new(); + for &insn_id in state.stack() { + stack.push(jit.get_opnd(insn_id)); + } + + let mut locals = Vec::new(); + for &insn_id in state.locals() { + locals.push(jit.get_opnd(insn_id)); + } + + SideExit{ + pc: Opnd::const_ptr(state.pc), + stack, + locals, + } +} + +/// Returne the maximum number of arguments for a block in a given function +fn max_num_params(function: &Function) -> usize { + let reverse_post_order = function.rpo(); + reverse_post_order.iter().map(|&block_id| { + let block = function.block(block_id); + block.params().len() + }).max().unwrap_or(0) +} + +#[cfg(target_arch = "x86_64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + extern "sysv64" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(target_arch = "aarch64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + extern "C" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(test)] +pub(crate) use c_callable; + +c_callable! { + /// Generated code calls this function with the SysV calling convention. See [gen_function_stub]. + /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. + /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported + /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { + with_vm_lock(src_loc!(), || { + // gen_push_frame() doesn't set PC, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; + unsafe { rb_set_cfp_pc(cfp, pc) }; + + // Successful JIT-to-JIT calls fill nils to non-parameter locals in generated code. + // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. + fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { + unsafe { + // Set SP which gen_push_frame() doesn't set + rb_set_cfp_sp(cfp, sp); + + // Fill nils to uninitialized (non-argument) locals + let local_size = get_iseq_body_local_table_size(iseq).to_usize(); + let num_params = iseq.params().size.to_usize(); + let base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, num_params) as isize); + slice::from_raw_parts_mut(base, local_size - num_params).fill(Qnil); + } + + // Increment a compile error counter for --zjit-stats + if get_option!(stats) { + incr_counter_by(exit_counter_for_compile_error(compile_error), 1); + } + } + + // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). + // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole + // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. + let cb = ZJITState::get_code_block(); + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + let compile_error = match last_status { + Some(IseqStatus::CantCompile(err)) => Some(err), + _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ => None, + }; + if let Some(compile_error) = compile_error { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, compile_error); + return ZJITState::get_exit_trampoline_with_counter().raw_ptr(cb); + } + + // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point. + let code_ptr = with_time_stat(compile_time_ns, || function_stub_hit_body(cb, &iseq_call)); + if code_ptr.is_ok() { + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.incoming.push(iseq_call); + } + } + let code_ptr = code_ptr.unwrap_or_else(|compile_error| { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, &compile_error); + ZJITState::get_exit_trampoline_with_counter() + }); + cb.mark_all_executable(); + code_ptr.raw_ptr(cb) + }) + } +} + +/// Compile an ISEQ for a function stub +fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<CodePtr, CompileError> { + // Compile the stubbed ISEQ + let IseqCodePtrs { jit_entry_ptrs, .. } = gen_iseq(cb, iseq_call.iseq.get(), None).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the stub to call the code pointer + let jit_entry_ptr = jit_entry_ptrs[iseq_call.jit_entry_idx.to_usize()]; + let code_addr = jit_entry_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); + asm.ccall(code_addr, vec![]); + }); + + Ok(jit_entry_ptr) +} + +/// Compile a stub for an ISEQ called by SendWithoutBlockDirect +fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); + asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + + // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. + // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. + asm.load_into(scratch_reg, Opnd::const_ptr(Rc::into_raw(iseq_call))); + asm.jmp(ZJITState::get_function_stub_hit_trampoline().into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function stub is called. +/// See [gen_function_stub] for how it's used. +pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); + asm_comment!(asm, "function_stub_hit trampoline"); + + // Maintain alignment for x86_64, and set up a frame for arm64 properly + asm.frame_setup(&[]); + + asm_comment!(asm, "preserve argument registers"); + + for pair in ALLOC_REGS.chunks(2) { + match *pair { + [reg0, reg1] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + } + [reg] => { + asm.cpush(Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } + } + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 + } + + // Compile the stubbed ISEQ + let jump_addr = asm_ccall!(asm, function_stub_hit, scratch_reg, CFP, SP); + asm.mov(scratch_reg, jump_addr); + + asm_comment!(asm, "restore argument registers"); + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpop_into(Opnd::Reg(ALLOC_REGS[0])); + } + + for pair in ALLOC_REGS.chunks(2).rev() { + match *pair { + [reg] => { + asm.cpop_into(Opnd::Reg(reg)); + } + [reg0, reg1] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + } + _ => unreachable!("chunks(2)") + } + } + + // Discard the current frame since the JIT function will set it up again + asm.frame_teardown(&[]); + + // Jump to scratch_reg so that cpop_into() doesn't clobber it + asm.jmp_opnd(scratch_reg); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function exits without restoring PC and the stack +pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id(); + + asm_comment!(asm, "side-exit trampoline"); + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that increments exit_compilation_failure and jumps to exit_trampoline. +pub fn gen_exit_trampoline_with_counter(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id(); + + asm_comment!(asm, "function stub exit trampoline"); + gen_incr_counter(&mut asm, exit_compile_error); + asm.jmp(Target::CodePtr(exit_trampoline)); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +fn gen_push_opnds(asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { + let n = opnds.len(); + let allocation_size = aligned_stack_bytes(n); + + // Bump the stack pointer to reserve the space for opnds + if n != 0 { + asm_comment!(asm, "allocate {} bytes on C stack for {} values", allocation_size, n); + asm.sub_into(NATIVE_STACK_PTR, allocation_size.into()); + } else { + asm_comment!(asm, "no opnds to allocate"); + } + + // Load NATIVE_STACK_PTR to get the address of a returned array + // to allow the backend to move it for its own use. + let argv = asm.load(NATIVE_STACK_PTR); + for (idx, &opnd) in opnds.iter().enumerate() { + asm.mov(Opnd::mem(VALUE_BITS, argv, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + + argv +} + +fn gen_pop_opnds(asm: &mut Assembler, opnds: &[Opnd]) { + if opnds.is_empty() { + asm_comment!(asm, "no opnds to restore"); + return + } + + asm_comment!(asm, "restore C stack pointer"); + let allocation_size = aligned_stack_bytes(opnds.len()); + asm.add_into(NATIVE_STACK_PTR, allocation_size.into()); +} + +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_opnd_ptr = gen_push_opnds(asm, &values); + + let tmp_ary = asm_ccall!(asm, rb_ary_tmp_new_from_values, Opnd::Imm(0), values.len().into(), first_opnd_ptr); + let result = asm_ccall!(asm, rb_reg_new_ary, tmp_ary, opt.into()); + asm_ccall!(asm, rb_ary_clear, tmp_ary); + + gen_pop_opnds(asm, &values); + + result +} + +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_string_ptr = gen_push_opnds(asm, &strings); + let result = asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr); + gen_pop_opnds(asm, &strings); + + result +} + +// Generate RSTRING_PTR +fn get_string_ptr(asm: &mut Assembler, string: Opnd) -> Opnd { + asm_comment!(asm, "get string pointer for embedded or heap"); + let string = asm.load(string); + let flags = Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RSTRING_NOEMBED as u64).into()); + let heap_ptr = asm.load(Opnd::mem( + usize::BITS as u8, + string, + RUBY_OFFSET_RSTRING_AS_HEAP_PTR, + )); + // Load the address of the embedded array + // (struct RString *)(obj)->as.ary + let ary = asm.lea(Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RSTRING_AS_ARY)); + asm.csel_nz(heap_ptr, ary) +} + +fn gen_string_getbyte(asm: &mut Assembler, string: Opnd, index: Opnd) -> Opnd { + let string_ptr = get_string_ptr(asm, string); + // TODO(max): Use SIB indexing here once the backend supports it + let string_ptr = asm.add(string_ptr, index); + let byte = asm.load(Opnd::mem(8, string_ptr, 0)); + // Zero-extend the byte to 64 bits + let byte = byte.with_num_bits(64); + let byte = asm.and(byte, 0xFF.into()); + // Tag the byte + let byte = asm.lshift(byte, Opnd::UImm(1)); + asm.or(byte, Opnd::UImm(1)) +} + +fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, value: Opnd) -> Opnd { + // rb_str_setbyte is not leaf, but we guard types and index ranges in HIR + asm_ccall!(asm, rb_str_setbyte, string, index, value) +} + +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_str_buf_append, string, val) +} + +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) +} + +/// Generate a JIT entry that just increments exit_compilation_failure and exits +fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + gen_incr_counter(&mut asm, exit_compile_error); + gen_incr_counter(&mut asm, exit_counter_for_compile_error(compile_error)); + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(0, gc_offsets.len()); + code_ptr + }) +} + +/// Given the number of spill slots needed for a function, return the number of bytes +/// the function needs to allocate on the stack for the stack frame. +fn aligned_stack_bytes(num_slots: usize) -> usize { + // Both x86_64 and arm64 require the stack to be aligned to 16 bytes. + // Since SIZEOF_VALUE is 8 bytes, we need to round up the size to the nearest even number. + let num_slots = num_slots + (num_slots % 2); + num_slots * SIZEOF_VALUE +} + +impl Assembler { + /// Make a C call while marking the start and end positions for IseqCall + fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &IseqCallRef) -> Opnd { + // We need to create our own branch rc objects so that we can move the closure below + let start_iseq_call = iseq_call.clone(); + let end_iseq_call = iseq_call.clone(); + + self.ccall_with_pos_markers( + fptr, + opnds, + move |code_ptr, _| { + start_iseq_call.start_addr.set(Some(code_ptr)); + }, + move |code_ptr, _| { + end_iseq_call.end_addr.set(Some(code_ptr)); + }, + ) + } +} + +/// Store info about a JIT entry point +pub struct JITEntry { + /// Index that corresponds to [crate::hir::jit_entry_insns] + jit_entry_idx: usize, + /// Position where the entry point starts + start_addr: Cell<Option<CodePtr>>, +} + +impl JITEntry { + /// Allocate a new JITEntry + fn new(jit_entry_idx: usize) -> Rc<RefCell<Self>> { + let jit_entry = JITEntry { + jit_entry_idx, + start_addr: Cell::new(None), + }; + Rc::new(RefCell::new(jit_entry)) + } +} + +/// Store info about a JIT-to-JIT call +#[derive(Debug)] +pub struct IseqCall { + /// Callee ISEQ that start_addr jumps to + pub iseq: Cell<IseqPtr>, + + /// Index that corresponds to [crate::hir::jit_entry_insns] + jit_entry_idx: u32, + + /// Position where the call instruction starts + start_addr: Cell<Option<CodePtr>>, + + /// Position where the call instruction ends (exclusive) + end_addr: Cell<Option<CodePtr>>, +} + +pub type IseqCallRef = Rc<IseqCall>; + +impl IseqCall { + /// Allocate a new IseqCall + fn new(iseq: IseqPtr, jit_entry_idx: u32) -> IseqCallRef { + let iseq_call = IseqCall { + iseq: Cell::new(iseq), + start_addr: Cell::new(None), + end_addr: Cell::new(None), + jit_entry_idx, + }; + Rc::new(iseq_call) + } + + /// Regenerate a IseqCall with a given callback + fn regenerate(&self, cb: &mut CodeBlock, callback: impl Fn(&mut Assembler)) { + cb.with_write_ptr(self.start_addr.get().unwrap(), |cb| { + let mut asm = Assembler::new(); + asm.new_block_without_id(); + callback(&mut asm); + asm.compile(cb).unwrap(); + assert_eq!(self.end_addr.get().unwrap(), cb.get_write_ptr()); + }); + } +} + +#[cfg(test)] +mod tests { + use crate::codegen::MAX_ISEQ_VERSIONS; + use crate::cruby::test_utils::*; + use crate::payload::*; + + #[test] + fn test_max_iseq_versions() { + eval(&format!(" + TEST = -1 + def test = TEST + + # compile and invalidate MAX+1 times + i = 0 + while i < {MAX_ISEQ_VERSIONS} + 1 + test; test # compile a version + + Object.send(:remove_const, :TEST) + TEST = i + + i += 1 + end + ")); + + // It should not exceed MAX_ISEQ_VERSIONS + let iseq = get_method_iseq("self", "test"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(payload.versions.len(), MAX_ISEQ_VERSIONS); + + // The last call should not discard the JIT code + assert!(matches!(unsafe { payload.versions.last().unwrap().as_ref() }.status, IseqStatus::Compiled(_))); + } +} diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs new file mode 100644 index 0000000000..51faaab9c2 --- /dev/null +++ b/zjit/src/cruby.rs @@ -0,0 +1,1411 @@ +//! This module deals with making relevant C functions available to Rust ZJIT. +//! Some C functions we use we maintain, some are public C extension APIs, +//! some are internal CRuby APIs. +//! +//! ## General notes about linking +//! +//! The ZJIT crate compiles to a native static library, which for our purposes +//! we can understand as a collection of object files. On ELF platforms at least, +//! object files can refer to "external symbols" which we could take some +//! liberty and understand as assembly labels that refer to code defined in other +//! object files resolved when linking. When we are linking, say to produce miniruby, +//! the linker resolves and put concrete addresses for each usage of C function in +//! the Rust static library. +//! +//! By declaring external functions and using them, we are asserting the symbols +//! we use have definition in one of the object files we pass to the linker. Declaring +//! a function here that has no definition anywhere causes a linking error. +//! +//! There are more things going on during linking and this section makes a lot of +//! simplifications but hopefully this gives a good enough working mental model. +//! +//! ## Difference from example in the Rustonomicon +//! +//! You might be wondering about why this is different from the [FFI example] +//! in the Nomicon, an official book about Unsafe Rust. +//! +//! There is no `#[link]` attribute because we are not linking against an external +//! library, but rather implicitly asserting that we'll supply a concrete definition +//! for all C functions we call, similar to how pure C projects put functions +//! across different compilation units and link them together. +//! +//! TODO(alan): is the model different enough on Windows that this setup is unworkable? +//! Seems prudent to at least learn more about Windows binary tooling before +//! committing to a design. +//! +//! Alan recommends reading the Nomicon cover to cover as he thinks the book is +//! not very long in general and especially for something that can save hours of +//! debugging Undefined Behavior (UB) down the road. +//! +//! UBs can cause Safe Rust to crash, at which point it's hard to tell which +//! usage of `unsafe` in the codebase invokes UB. Providing safe Rust interface +//! wrapping `unsafe` Rust is a good technique, but requires practice and knowledge +//! about what's well defined and what's undefined. +//! +//! For an extremely advanced example of building safe primitives using Unsafe Rust, +//! see the [GhostCell] paper. Some parts of the paper assume less background knowledge +//! than other parts, so there should be learning opportunities in it for all experience +//! levels. +//! +//! ## Binding generation +//! +//! For the moment declarations on the Rust side are hand written. The code is boilerplate +//! and could be generated automatically with a custom tooling that depend on +//! rust-lang/rust-bindgen. The output Rust code could be checked in to version control +//! and verified on CI like `make update-deps`. +//! +//! Upsides for this design: +//! - the ZJIT static lib that links with miniruby and friends will not need bindgen +//! as a dependency at all. This is an important property so Ruby end users can +//! build a ZJIT enabled Ruby with no internet connection using a release tarball +//! - Less hand-typed boilerplate +//! - Helps reduce risk of C definitions and Rust declaration going out of sync since +//! CI verifies synchronicity +//! +//! Downsides and known unknowns: +//! - Using rust-bindgen this way seems unusual. We might be depending on parts +//! that the project is not committed to maintaining +//! - This setup assumes rust-bindgen gives deterministic output, which can't be taken +//! for granted +//! - ZJIT contributors will need to install libclang on their system to get rust-bindgen +//! to work if they want to run the generation tool locally +//! +//! The elephant in the room is that we'll still need to use Unsafe Rust to call C functions, +//! and the binding generation can't magically save us from learning Unsafe Rust. +//! +//! +//! [FFI example]: https://doc.rust-lang.org/nomicon/ffi.html +//! [GhostCell]: http://plv.mpi-sws.org/rustbelt/ghostcell/ + +// CRuby types use snake_case. Allow them so we use one name across languages. +#![allow(non_camel_case_types)] +// A lot of imported CRuby globals aren't all-caps +#![allow(non_upper_case_globals)] +#![allow(clippy::upper_case_acronyms)] + +// Some of this code may not be used yet +#![allow(dead_code)] +#![allow(unused_macros)] +#![allow(unused_imports)] + +use std::convert::From; +use std::ffi::{c_void, CString, CStr}; +use std::fmt::{Debug, Display, Formatter}; +use std::os::raw::{c_char, c_int, c_uint}; +use std::panic::{catch_unwind, UnwindSafe}; + +// We check that we can do this with the configure script and a couple of +// static asserts. u64 and not usize to play nice with lowering to x86. +pub type size_t = u64; + +/// A type alias for the redefinition flags coming from CRuby. These are just +/// shifted 1s but not explicitly an enum. +pub type RedefinitionFlag = u32; + +#[allow(unsafe_op_in_unsafe_fn)] +#[allow(dead_code)] +#[allow(clippy::all)] // warning meant to help with reading; not useful for generated code +mod autogened { + use super::*; + // Textually include output from rust-bindgen as suggested by its user guide. + include!("cruby_bindings.inc.rs"); +} +pub use autogened::*; + +// TODO: For #defines that affect memory layout, we need to check for them +// on build and fail if they're wrong. e.g. USE_FLONUM *must* be true. + +// These are functions we expose from C files, not in any header. +// Parsing it would result in a lot of duplicate definitions. +// Use bindgen for functions that are defined in headers or in zjit.c. +#[cfg_attr(test, allow(unused))] // We don't link against C code when testing +unsafe extern "C" { + pub fn rb_check_overloaded_cme( + me: *const rb_callable_method_entry_t, + ci: *const rb_callinfo, + ) -> *const rb_callable_method_entry_t; + + // Floats within range will be encoded without creating objects in the heap. + // (Range is 0x3000000000000001 to 0x4fffffffffffffff (1.7272337110188893E-77 to 2.3158417847463237E+77). + pub fn rb_float_new(d: f64) -> VALUE; + + pub fn rb_hash_empty_p(hash: VALUE) -> VALUE; + pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE; + pub fn rb_str_getbyte(str: VALUE, index: VALUE) -> VALUE; + pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE; + pub fn rb_jit_fix_mod_fix(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE; + pub fn rb_vm_get_special_object(reg_ep: *const VALUE, value_type: vm_special_object_type) -> VALUE; + pub fn rb_vm_concat_to_array(ary1: VALUE, ary2st: VALUE) -> VALUE; + pub fn rb_vm_defined( + ec: EcPtr, + reg_cfp: CfpPtr, + op_type: rb_num_t, + obj: VALUE, + v: VALUE, + ) -> bool; + pub fn rb_vm_set_ivar_id(obj: VALUE, idx: u32, val: VALUE) -> VALUE; + pub fn rb_vm_setinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, val: VALUE, ic: IVC); + pub fn rb_vm_getinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, ic: IVC) -> VALUE; + pub fn rb_aliased_callable_method_entry( + me: *const rb_callable_method_entry_t, + ) -> *const rb_callable_method_entry_t; + pub fn rb_vm_getclassvariable(iseq: IseqPtr, cfp: CfpPtr, id: ID, ic: ICVARC) -> VALUE; + pub fn rb_vm_setclassvariable( + iseq: IseqPtr, + cfp: CfpPtr, + id: ID, + val: VALUE, + ic: ICVARC, + ) -> VALUE; + pub fn rb_vm_ic_hit_p(ic: IC, reg_ep: *const VALUE) -> bool; + pub fn rb_vm_stack_canary() -> VALUE; + pub fn rb_vm_push_cfunc_frame(cme: *const rb_callable_method_entry_t, recv_idx: c_int); + pub fn rb_obj_class(klass: VALUE) -> VALUE; + pub fn rb_vm_objtostring(iseq: IseqPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; +} + +// Renames +pub use rb_insn_name as raw_insn_name; +pub use rb_get_ec_cfp as get_ec_cfp; +pub use rb_get_cfp_iseq as get_cfp_iseq; +pub use rb_get_cfp_pc as get_cfp_pc; +pub use rb_get_cfp_sp as get_cfp_sp; +pub use rb_get_cfp_self as get_cfp_self; +pub use rb_get_cfp_ep as get_cfp_ep; +pub use rb_get_cfp_ep_level as get_cfp_ep_level; +pub use rb_vm_base_ptr as get_cfp_bp; +pub use rb_get_cme_def_type as get_cme_def_type; +pub use rb_get_cme_def_body_attr_id as get_cme_def_body_attr_id; +pub use rb_get_cme_def_body_optimized_type as get_cme_def_body_optimized_type; +pub use rb_get_cme_def_body_optimized_index as get_cme_def_body_optimized_index; +pub use rb_get_cme_def_body_cfunc as get_cme_def_body_cfunc; +pub use rb_get_def_method_serial as get_def_method_serial; +pub use rb_get_def_original_id as get_def_original_id; +pub use rb_get_mct_argc as get_mct_argc; +pub use rb_get_mct_func as get_mct_func; +pub use rb_get_def_iseq_ptr as get_def_iseq_ptr; +pub use rb_iseq_encoded_size as get_iseq_encoded_size; +pub use rb_get_iseq_body_local_iseq as get_iseq_body_local_iseq; +pub use rb_get_iseq_body_iseq_encoded as get_iseq_body_iseq_encoded; +pub use rb_get_iseq_body_stack_max as get_iseq_body_stack_max; +pub use rb_get_iseq_body_type as get_iseq_body_type; +pub use rb_get_iseq_body_local_table_size as get_iseq_body_local_table_size; +pub use rb_get_cikw_keyword_len as get_cikw_keyword_len; +pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx; +pub use rb_get_call_data_ci as get_call_data_ci; +pub use rb_FL_TEST as FL_TEST; +pub use rb_FL_TEST_RAW as FL_TEST_RAW; +pub use rb_RB_TYPE_P as RB_TYPE_P; +pub use rb_BASIC_OP_UNREDEFINED_P as BASIC_OP_UNREDEFINED_P; +pub use rb_vm_ci_argc as vm_ci_argc; +pub use rb_vm_ci_mid as vm_ci_mid; +pub use rb_vm_ci_flag as vm_ci_flag; +pub use rb_vm_ci_kwarg as vm_ci_kwarg; +pub use rb_METHOD_ENTRY_VISI as METHOD_ENTRY_VISI; +pub use rb_RCLASS_ORIGIN as RCLASS_ORIGIN; +pub use rb_vm_get_special_object as vm_get_special_object; +pub use rb_jit_fix_mod_fix as rb_fix_mod_fix; + +/// Helper so we can get a Rust string for insn_name() +pub fn insn_name(opcode: usize) -> String { + if opcode >= VM_INSTRUCTION_SIZE.try_into().unwrap() { + return "<unknown>".into(); + } + unsafe { + // Look up Ruby's NULL-terminated insn name string + let op_name = raw_insn_name(VALUE(opcode)); + + // Convert the op name C string to a Rust string and concat + let op_name = CStr::from_ptr(op_name).to_str().unwrap(); + + // Convert into an owned string + op_name.to_string() + } +} + +pub fn insn_len(opcode: usize) -> u32 { + unsafe { + rb_insn_len(VALUE(opcode)).try_into().unwrap() + } +} + +/// We avoid using bindgen for `rb_iseq_constant_body` since its definition changes depending +/// on build configuration while we need one bindgen file that works for all configurations. +/// Use an opaque type for it instead. +/// See: <https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs> +#[repr(C)] +pub struct rb_iseq_constant_body { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// An object handle similar to VALUE in the C code. Our methods assume +/// that this is a handle. Sometimes the C code briefly uses VALUE as +/// an unsigned integer type and don't necessarily store valid handles but +/// thankfully those cases are rare and don't cross the FFI boundary. +#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)] +#[repr(transparent)] // same size and alignment as simply `usize` +pub struct VALUE(pub usize); + +/// An interned string. See [ids] and methods this type. +/// `0` is a sentinal value for IDs. +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub struct ID(pub ::std::os::raw::c_ulong); + +/// Pointer to an ISEQ +pub type IseqPtr = *const rb_iseq_t; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ShapeId(pub u32); + +pub const INVALID_SHAPE_ID: ShapeId = ShapeId(rb_invalid_shape_id); + +impl ShapeId { + pub fn is_valid(self) -> bool { + self != INVALID_SHAPE_ID + } + + pub fn is_too_complex(self) -> bool { + unsafe { rb_jit_shape_too_complex_p(self.0) } + } + + pub fn is_frozen(self) -> bool { + (self.0 & SHAPE_ID_FL_FROZEN) != 0 + } +} + +// Given an ISEQ pointer, convert PC to insn_idx +pub fn iseq_pc_to_insn_idx(iseq: IseqPtr, pc: *mut VALUE) -> Option<u16> { + let pc_zero = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + unsafe { pc.offset_from(pc_zero) }.try_into().ok() +} + +/// Given an ISEQ pointer and an instruction index, return an opcode. +pub fn iseq_opcode_at_idx(iseq: IseqPtr, insn_idx: u32) -> u32 { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + unsafe { rb_iseq_opcode_at_pc(iseq, pc) as u32 } +} + +/// Return true if a given ISEQ is known to escape EP to the heap on entry. +/// +/// As of vm_push_frame(), EP is always equal to BP. However, after pushing +/// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + match unsafe { get_iseq_body_type(iseq) } { + // The EP of the <main> frame points to TOPLEVEL_BINDING + ISEQ_TYPE_MAIN | + // eval frames point to the EP of another frame or scope + ISEQ_TYPE_EVAL => true, + _ => false, + } +} + +/// Index of the local variable that has a rest parameter if any +pub fn iseq_rest_param_idx(params: &IseqParameters) -> Option<i32> { + // TODO(alan): replace with `params.rest_start` + if params.flags.has_rest() != 0 { + Some(params.opt_num + params.lead_num) + } else { + None + } +} + +/// Iterate over all existing ISEQs +pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { + unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { + // SAFETY: points to the local below + let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) }; + callback(iseq); + } + let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; +} + +/// Return a poison value to be set above the stack top to verify leafness. +#[cfg(not(test))] +pub fn vm_stack_canary() -> u64 { + unsafe { rb_vm_stack_canary() }.as_u64() +} + +/// Avoid linking the C function in `cargo test` +#[cfg(test)] +pub fn vm_stack_canary() -> u64 { + 0 +} + +/// Opaque execution-context type from vm_core.h +#[repr(C)] +pub struct rb_execution_context_struct { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} +/// Alias for rb_execution_context_struct used by CRuby sometimes +pub type rb_execution_context_t = rb_execution_context_struct; + +/// Pointer to an execution context (rb_execution_context_struct) +pub type EcPtr = *const rb_execution_context_struct; + +// From method.h +#[repr(C)] +pub struct rb_method_definition_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} +type rb_method_definition_struct = rb_method_definition_t; + +/// Opaque cfunc type from method.h +#[repr(C)] +pub struct rb_method_cfunc_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Opaque call-cache type from vm_callinfo.h +#[repr(C)] +pub struct rb_callcache { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Opaque control_frame (CFP) struct from vm_core.h +#[repr(C)] +pub struct rb_control_frame_struct { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Pointer to a control frame pointer (CFP) +pub type CfpPtr = *mut rb_control_frame_struct; + +/// Opaque struct from vm_core.h +#[repr(C)] +pub struct rb_cref_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +#[derive(PartialEq)] +pub enum ClassRelationship { + Subclass, + Superclass, + NoRelation, +} + +/// A print adapator for debug info about a [VALUE]. Includes info +/// the GC knows about the handle. Example: `println!("{}", value.obj_info());`. +pub struct ObjInfoPrinter(VALUE); + +impl Display for ObjInfoPrinter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use std::mem::MaybeUninit; + const BUFFER_SIZE: usize = 0x100; + let mut buffer: MaybeUninit<[c_char; BUFFER_SIZE]> = MaybeUninit::uninit(); + let info = unsafe { + rb_raw_obj_info(buffer.as_mut_ptr().cast(), BUFFER_SIZE, self.0); + CStr::from_ptr(buffer.as_ptr().cast()).to_string_lossy() + }; + write!(f, "{info}") + } +} + +impl VALUE { + /// Get a printer for raw debug info from `rb_obj_info()` about the value. + pub fn obj_info(self) -> ObjInfoPrinter { + ObjInfoPrinter(self) + } + + /// Return whether the value is truthy or falsy in Ruby -- only nil and false are falsy. + pub fn test(self) -> bool { + let VALUE(cval) = self; + let VALUE(qnilval) = Qnil; + (cval & !qnilval) != 0 + } + + /// Return true if the number is an immediate integer, flonum or static symbol + fn immediate_p(self) -> bool { + let VALUE(cval) = self; + let mask = RUBY_IMMEDIATE_MASK as usize; + (cval & mask) != 0 + } + + /// Return true if the value is a Ruby immediate integer, flonum, static symbol, nil or false + pub fn special_const_p(self) -> bool { + self.immediate_p() || !self.test() + } + + /// Return true if the value is a heap object + pub fn heap_object_p(self) -> bool { + !self.special_const_p() + } + + /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`. + pub fn shareable_p(self) -> bool { + (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0 + } + + /// Return true if the value is a Ruby Fixnum (immediate-size integer) + pub fn fixnum_p(self) -> bool { + let VALUE(cval) = self; + let flag = RUBY_FIXNUM_FLAG as usize; + (cval & flag) == flag + } + + /// Return true if the value is an immediate Ruby floating-point number (flonum) + pub fn flonum_p(self) -> bool { + let VALUE(cval) = self; + let mask = RUBY_FLONUM_MASK as usize; + let flag = RUBY_FLONUM_FLAG as usize; + (cval & mask) == flag + } + + /// Return true if the value is a Ruby symbol (RB_SYMBOL_P) + pub fn symbol_p(self) -> bool { + self.static_sym_p() || self.dynamic_sym_p() + } + + pub fn instance_can_have_singleton_class(self) -> bool { + if self == unsafe { rb_cInteger } || self == unsafe { rb_cFloat } || + self == unsafe { rb_cSymbol } || self == unsafe { rb_cNilClass } || + self == unsafe { rb_cTrueClass } || self == unsafe { rb_cFalseClass } { + + return false + } + true + } + + /// Return true for a static (non-heap) Ruby symbol (RB_STATIC_SYM_P) + pub fn static_sym_p(self) -> bool { + let VALUE(cval) = self; + let flag = RUBY_SYMBOL_FLAG as usize; + (cval & 0xff) == flag + } + + /// Return true for a dynamic Ruby symbol (RB_DYNAMIC_SYM_P) + fn dynamic_sym_p(self) -> bool { + if self.special_const_p() { + false + } else { + self.builtin_type() == RUBY_T_SYMBOL + } + } + + /// Returns true if the value is T_HASH + pub fn hash_p(self) -> bool { + !self.special_const_p() && self.builtin_type() == RUBY_T_HASH + } + + /// Returns true or false depending on whether the value is nil + pub fn nil_p(self) -> bool { + self == Qnil + } + + pub fn string_p(self) -> bool { + self.class_of() == unsafe { rb_cString } + } + + /// Read the flags bits from the RBasic object, then return a Ruby type enum (e.g. RUBY_T_ARRAY) + pub fn builtin_type(self) -> ruby_value_type { + (self.builtin_flags() & (RUBY_T_MASK as usize)) as ruby_value_type + } + + pub fn builtin_flags(self) -> usize { + assert!(!self.special_const_p()); + + let VALUE(cval) = self; + let rbasic_ptr = cval as *const RBasic; + let flags_bits: usize = unsafe { (*rbasic_ptr).flags }.as_usize(); + flags_bits + } + + pub fn class_of(self) -> VALUE { + if !self.special_const_p() { + let builtin_type = self.builtin_type(); + assert!( + builtin_type != RUBY_T_NONE && builtin_type != RUBY_T_MOVED, + "ZJIT saw a dead object. T_type={builtin_type}, {}", + self.obj_info() + ); + } + + unsafe { rb_yarv_class_of(self) } + } + + /// Check if `self` is a subclass of `other`. Assumes both `self` and `other` are class + /// objects. Returns [`ClassRelationship::Subclass`] if `self <= other`, + /// [`ClassRelationship::Superclass`] if `other < self`, and [`ClassRelationship::NoRelation`] + /// otherwise. + pub fn is_subclass_of(self, other: VALUE) -> ClassRelationship { + assert!(unsafe { RB_TYPE_P(self, RUBY_T_CLASS) }); + assert!(unsafe { RB_TYPE_P(other, RUBY_T_CLASS) }); + match unsafe { rb_class_inherited_p(self, other) } { + Qtrue => ClassRelationship::Subclass, + Qfalse => ClassRelationship::Superclass, + Qnil => ClassRelationship::NoRelation, + // The API specifies that it will return Qnil in this case + _ => panic!("Unexpected return value from rb_class_inherited_p"), + } + } + + /// Borrow the string contents of `self`. Rust unsafe because of possible mutation and GC + /// interactions. + pub unsafe fn as_rstring_byte_slice<'a>(self) -> Option<&'a [u8]> { + if !unsafe { RB_TYPE_P(self, RUBY_T_STRING) } { + None + } else { + let str_ptr = unsafe { rb_RSTRING_PTR(self) } as *const u8; + let str_len: usize = unsafe { rb_RSTRING_LEN(self) }.try_into().ok()?; + Some(unsafe { std::slice::from_raw_parts(str_ptr, str_len) }) + } + } + + pub fn is_frozen(self) -> bool { + unsafe { rb_obj_frozen_p(self) != VALUE(0) } + } + + pub fn shape_id_of(self) -> ShapeId { + ShapeId(unsafe { rb_obj_shape_id(self) }) + } + + pub fn embedded_p(self) -> bool { + unsafe { + FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0) + } + } + + pub fn struct_embedded_p(self) -> bool { + unsafe { + RB_TYPE_P(self, RUBY_T_STRUCT) && + FL_TEST_RAW(self, VALUE(RSTRUCT_EMBED_LEN_MASK)) != VALUE(0) + } + } + + pub fn as_fixnum(self) -> i64 { + assert!(self.fixnum_p()); + (self.0 as i64) >> 1 + } + + pub fn as_isize(self) -> isize { + let VALUE(is) = self; + is as isize + } + + pub fn as_i32(self) -> i32 { + self.as_i64().try_into().unwrap() + } + + pub fn as_u32(self) -> u32 { + let VALUE(i) = self; + i.try_into().unwrap() + } + + pub fn as_i64(self) -> i64 { + let VALUE(i) = self; + i as i64 + } + + pub fn as_u64(self) -> u64 { + let VALUE(i) = self; + i.try_into().unwrap() + } + + pub fn as_usize(self) -> usize { + let VALUE(us) = self; + us + } + + pub fn as_ptr<T>(self) -> *const T { + let VALUE(us) = self; + us as *const T + } + + pub fn as_mut_ptr<T>(self) -> *mut T { + let VALUE(us) = self; + us as *mut T + } + + /// For working with opaque pointers and encoding null check. + /// Similar to [std::ptr::NonNull], but for `*const T`. `NonNull<T>` + /// is for `*mut T` while our C functions are setup to use `*const T`. + /// Casting from `NonNull<T>` to `*const T` is too noisy. + pub fn as_optional_ptr<T>(self) -> Option<*const T> { + let ptr: *const T = self.as_ptr(); + + if ptr.is_null() { + None + } else { + Some(ptr) + } + } + + /// Assert that `self` is an iseq in debug builds + pub fn as_iseq(self) -> IseqPtr { + let ptr: IseqPtr = self.as_ptr(); + + #[cfg(debug_assertions)] + if !ptr.is_null() { + unsafe { rb_assert_iseq_handle(self) } + } + + ptr + } + + pub fn cme_p(self) -> bool { + if self == VALUE(0) { return false; } + unsafe { rb_IMEMO_TYPE_P(self, imemo_ment) == 1 } + } + + /// Assert that `self` is a method entry in debug builds + pub fn as_cme(self) -> *const rb_callable_method_entry_t { + let ptr: *const rb_callable_method_entry_t = self.as_ptr(); + + #[cfg(debug_assertions)] + if !ptr.is_null() { + unsafe { rb_assert_cme_handle(self) } + } + + ptr + } + + pub const fn fixnum_from_usize(item: usize) -> Self { + assert!(item <= (RUBY_FIXNUM_MAX as usize)); // An unsigned will always be greater than RUBY_FIXNUM_MIN + let k: usize = item.wrapping_add(item.wrapping_add(1)); + VALUE(k) + } + + pub const fn fixnum_from_isize(item: isize) -> Self { + assert!(item >= RUBY_FIXNUM_MIN); + assert!(item <= RUBY_FIXNUM_MAX); + let k: isize = item.wrapping_add(item.wrapping_add(1)); + VALUE(k as usize) + } + + /// Call the write barrier after separately writing val to self. + pub fn write_barrier(self, val: VALUE) { + // rb_gc_writebarrier() asserts it is not called with a special constant + if !val.special_const_p() { + unsafe { rb_gc_writebarrier(self, val) }; + } + } +} + +pub type IseqParameters = rb_iseq_constant_body_rb_iseq_parameters; + +/// Extension trait to enable method calls on [`IseqPtr`] +pub trait IseqAccess { + unsafe fn params<'a>(self) -> &'a IseqParameters; +} + +impl IseqAccess for IseqPtr { + /// Get a description of the ISEQ's signature. Analogous to `ISEQ_BODY(iseq)->param` in C. + unsafe fn params<'a>(self) -> &'a IseqParameters { + use crate::cast::IntoUsize; + unsafe { &*((*self).body.byte_add(ISEQ_BODY_OFFSET_PARAM.to_usize()) as *const IseqParameters) } + } +} + +impl From<IseqPtr> for VALUE { + /// For `.into()` convenience + fn from(iseq: IseqPtr) -> Self { + VALUE(iseq as usize) + } +} + +impl From<*const rb_callable_method_entry_t> for VALUE { + /// For `.into()` convenience + fn from(cme: *const rb_callable_method_entry_t) -> Self { + VALUE(cme as usize) + } +} + +impl From<&str> for VALUE { + fn from(value: &str) -> Self { + rust_str_to_ruby(value) + } +} + +impl From<String> for VALUE { + fn from(value: String) -> Self { + rust_str_to_ruby(&value) + } +} + +impl From<VALUE> for u64 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + uimm as u64 + } +} + +impl From<VALUE> for i64 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + assert!(uimm <= (i64::MAX as usize)); + uimm as i64 + } +} + +impl From<VALUE> for i32 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + assert!(uimm <= (i32::MAX as usize)); + uimm.try_into().unwrap() + } +} + +impl From<VALUE> for u16 { + fn from(value: VALUE) -> Self { + let VALUE(uimm) = value; + uimm.try_into().unwrap() + } +} + +impl ID { + // Get a debug representation of the contents of the ID. Since `str` is UTF-8 + // and IDs have encodings that are not, this is a lossy representation. + pub fn contents_lossy(&self) -> std::borrow::Cow<'_, str> { + use std::borrow::Cow; + if self.0 == 0 { + Cow::Borrowed("ID(0)") + } else { + // Get the contents as a byte slice. IDs can have internal NUL bytes so rb_id2name, + // which returns a C string is more lossy than this approach. + let contents = unsafe { rb_id2str(*self) }; + if contents == Qfalse { + Cow::Borrowed("ID(0)") + } else { + let slice = unsafe { contents.as_rstring_byte_slice() } + .expect("rb_id2str() returned truthy non-string"); + String::from_utf8_lossy(slice) + } + } + } +} + +/// Produce a Ruby string from a Rust string slice +pub fn rust_str_to_ruby(str: &str) -> VALUE { + unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) } +} + +/// Produce a Ruby ID from a Rust string slice +pub fn rust_str_to_id(str: &str) -> ID { + let c_str = CString::new(str).unwrap(); + let c_ptr: *const c_char = c_str.as_ptr(); + unsafe { rb_intern(c_ptr) } +} + +/// Produce a Ruby symbol from a Rust string slice +pub fn rust_str_to_sym(str: &str) -> VALUE { + let id = rust_str_to_id(str); + unsafe { rb_id2sym(id) } +} + +/// Produce an owned Rust String from a C char pointer +pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { + assert!(!c_char_ptr.is_null()); + + let c_str: &CStr = unsafe { CStr::from_ptr(c_char_ptr) }; + + match c_str.to_str() { + Ok(rust_str) => Some(rust_str.to_string()), + Err(_) => None + } +} + +pub fn iseq_name(iseq: IseqPtr) -> String { + if iseq.is_null() { + return "<NULL>".to_string(); + } + let iseq_label = unsafe { rb_iseq_label(iseq) }; + if iseq_label == Qnil { + "None".to_string() + } else { + ruby_str_to_rust_string(iseq_label) + } +} + +// Location is the file defining the method, colon, method name. +// Filenames are sometimes internal strings supplied to eval, +// so be careful with them. +pub fn iseq_get_location(iseq: IseqPtr, pos: u32) -> String { + let iseq_path = unsafe { rb_iseq_path(iseq) }; + let iseq_lineno = unsafe { rb_iseq_line_no(iseq, pos as usize) }; + + let mut s = iseq_name(iseq); + s.push('@'); + if iseq_path == Qnil { + s.push_str("None"); + } else { + s.push_str(&ruby_str_to_rust_string(iseq_path)); + } + s.push(':'); + s.push_str(&iseq_lineno.to_string()); + s +} + + +// Convert a CRuby UTF-8-encoded RSTRING into a Rust string. +// This should work fine on ASCII strings and anything else +// that is considered legal UTF-8, including embedded nulls. +fn ruby_str_to_rust_string(v: VALUE) -> String { + let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; + let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); + let str_slice: &[u8] = unsafe { std::slice::from_raw_parts(str_ptr, str_len) }; + String::from_utf8(str_slice.to_vec()).unwrap_or_default() +} + +pub fn ruby_sym_to_rust_string(v: VALUE) -> String { + let ruby_str = unsafe { rb_sym2str(v) }; + ruby_str_to_rust_string(ruby_str) +} + +pub fn ruby_call_method_id(cd: *const rb_call_data) -> ID { + let call_info = unsafe { rb_get_call_data_ci(cd) }; + unsafe { rb_vm_ci_mid(call_info) } +} + +pub fn ruby_call_method_name(cd: *const rb_call_data) -> String { + let mid = ruby_call_method_id(cd); + mid.contents_lossy().to_string() +} + +/// A location in Rust code for integrating with debugging facilities defined in C. +/// Use the [src_loc!] macro to crate an instance. +pub struct SourceLocation { + pub file: &'static CStr, + pub line: c_int, +} + +impl Debug for SourceLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}:{}", self.file.to_string_lossy(), self.line)) + } +} + +/// Make a [SourceLocation] at the current spot. +macro_rules! src_loc { + () => { + { + // Nul-terminated string with static lifetime, make a CStr out of it safely. + let file: &'static str = concat!(file!(), '\0'); + $crate::cruby::SourceLocation { + file: unsafe { std::ffi::CStr::from_ptr(file.as_ptr().cast()) }, + line: line!().try_into().unwrap(), + } + } + }; +} + +pub(crate) use src_loc; + +/// Run GC write barrier. Required after making a new edge in the object reference +/// graph from `old` to `young`. +macro_rules! obj_written { + ($old: expr, $young: expr) => { + let (old, young): (VALUE, VALUE) = ($old, $young); + let src_loc = $crate::cruby::src_loc!(); + unsafe { rb_yjit_obj_written(old, young, src_loc.file.as_ptr(), src_loc.line) }; + }; +} +pub(crate) use obj_written; + +/// Acquire the VM lock, make sure all other Ruby threads are asleep then run +/// some code while holding the lock. Returns whatever `func` returns. +/// Use with [src_loc!]. +/// +/// Required for code patching in the presence of ractors. +pub fn with_vm_lock<F, R>(loc: SourceLocation, func: F) -> R +where + F: FnOnce() -> R + UnwindSafe, +{ + let file = loc.file.as_ptr(); + let line = loc.line; + let mut recursive_lock_level: c_uint = 0; + + unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + // Ensure GC is off while we have the VM lock because: + // 1. We create many transient Rust collections that hold VALUEs during compilation. + // It's extremely tricky to properly marked and reference update these, not to + // mention the overhead and ergonomics issues. + // 2. If we yield to the GC while compiling, it re-enters our mark and update functions. + // This breaks `&mut` exclusivity since mark functions derive fresh `&mut` from statics + // while there is a stack frame below it that has an overlapping `&mut`. That's UB. + let gc_disabled_pre_call = unsafe { rb_gc_disable() }.test(); + + let ret = match catch_unwind(func) { + Ok(result) => result, + Err(_) => { + // Theoretically we can recover from some of these panics, + // but it's too late if the unwind reaches here. + + let _ = catch_unwind(|| { + // IO functions can panic too. + eprintln!( + "ZJIT panicked while holding VM lock acquired at {}:{}. Aborting...", + loc.file.to_string_lossy(), + line, + ); + }); + std::process::abort(); + } + }; + + unsafe { + if !gc_disabled_pre_call { + rb_gc_enable(); + } + rb_jit_vm_unlock(&mut recursive_lock_level, file, line); + }; + + ret +} + +/// At the moment, we abort in all cases we panic. +/// To aid with getting diagnostics in the wild without requiring people to set +/// RUST_BACKTRACE=1, register a panic hook that crash using rb_bug() for release builds. +/// rb_bug() might not be as good at printing a call trace as Rust's stdlib, but +/// it dumps some other info that might be relevant. +/// +/// In case we want to start doing fancier exception handling with panic=unwind, +/// we can revisit this later. For now, this helps to get us good bug reports. +pub fn rb_bug_panic_hook() { + use std::env; + use std::panic; + use std::io::{stderr, Write}; + + // Probably the default hook. We do this very early during process boot. + let previous_hook = panic::take_hook(); + + panic::set_hook(Box::new(move |panic_info| { + // Not using `eprintln` to avoid double panic. + let _ = stderr().write_all(b"ruby: ZJIT has panicked. More info to follow...\n"); + + // Always show a Rust backtrace for release builds. + // You should set RUST_BACKTRACE=1 for dev builds. + let release_build = cfg!(not(debug_assertions)); + if release_build { + unsafe { env::set_var("RUST_BACKTRACE", "1"); } + } + previous_hook(panic_info); + + // Dump information about the interpreter for release builds. + // You may also use ZJIT_RB_BUG=1 to trigger this on dev builds. + if release_build || env::var("ZJIT_RB_BUG").is_ok() { + // Abort with rb_bug(). It has a length limit on the message. + let panic_message = &format!("{panic_info}")[..]; + let len = std::cmp::min(0x100, panic_message.len()) as c_int; + unsafe { rb_bug(b"ZJIT: %*s\0".as_ref().as_ptr() as *const c_char, len, panic_message.as_ptr()); } + } else { + eprintln!("note: run with `ZJIT_RB_BUG=1` environment variable to display a Ruby backtrace"); + } + })); +} + +// Non-idiomatic capitalization for consistency with CRuby code +#[allow(non_upper_case_globals)] +pub const Qfalse: VALUE = VALUE(RUBY_Qfalse as usize); +#[allow(non_upper_case_globals)] +pub const Qnil: VALUE = VALUE(RUBY_Qnil as usize); +#[allow(non_upper_case_globals)] +pub const Qtrue: VALUE = VALUE(RUBY_Qtrue as usize); +#[allow(non_upper_case_globals)] +pub const Qundef: VALUE = VALUE(RUBY_Qundef as usize); + +#[allow(unused)] +mod manual_defs { + use super::*; + + pub const SIZEOF_VALUE: usize = 8; + pub const BITS_PER_BYTE: usize = 8; + pub const SIZEOF_VALUE_I32: i32 = SIZEOF_VALUE as i32; + pub const VALUE_BITS: u8 = BITS_PER_BYTE as u8 * SIZEOF_VALUE as u8; + + pub const RUBY_LONG_MIN: isize = std::os::raw::c_long::MIN as isize; + pub const RUBY_LONG_MAX: isize = std::os::raw::c_long::MAX as isize; + + pub const RUBY_FIXNUM_MIN: isize = RUBY_LONG_MIN / 2; + pub const RUBY_FIXNUM_MAX: isize = RUBY_LONG_MAX / 2; + + // From vm_callinfo.h - uses calculation that seems to confuse bindgen + pub const VM_CALL_ARGS_SIMPLE: u32 = 1 << VM_CALL_ARGS_SIMPLE_bit; + pub const VM_CALL_ARGS_SPLAT: u32 = 1 << VM_CALL_ARGS_SPLAT_bit; + pub const VM_CALL_ARGS_SPLAT_MUT: u32 = 1 << VM_CALL_ARGS_SPLAT_MUT_bit; + pub const VM_CALL_ARGS_BLOCKARG: u32 = 1 << VM_CALL_ARGS_BLOCKARG_bit; + pub const VM_CALL_FORWARDING: u32 = 1 << VM_CALL_FORWARDING_bit; + pub const VM_CALL_FCALL: u32 = 1 << VM_CALL_FCALL_bit; + pub const VM_CALL_KWARG: u32 = 1 << VM_CALL_KWARG_bit; + pub const VM_CALL_KW_SPLAT: u32 = 1 << VM_CALL_KW_SPLAT_bit; + pub const VM_CALL_KW_SPLAT_MUT: u32 = 1 << VM_CALL_KW_SPLAT_MUT_bit; + pub const VM_CALL_TAILCALL: u32 = 1 << VM_CALL_TAILCALL_bit; + pub const VM_CALL_SUPER : u32 = 1 << VM_CALL_SUPER_bit; + pub const VM_CALL_ZSUPER : u32 = 1 << VM_CALL_ZSUPER_bit; + pub const VM_CALL_OPT_SEND : u32 = 1 << VM_CALL_OPT_SEND_bit; + + // From internal/struct.h - in anonymous enum, so we can't easily import it + pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; + + // From iseq.h - via a different constant, which seems to confuse bindgen + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + + // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to + // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants. + pub const RUBY_OFFSET_RBASIC_FLAGS: i32 = 0; // struct RBasic, field "flags" + pub const RUBY_OFFSET_RBASIC_KLASS: i32 = 8; // struct RBasic, field "klass" + pub const RUBY_OFFSET_RARRAY_AS_HEAP_LEN: i32 = 16; // struct RArray, subfield "as.heap.len" + pub const RUBY_OFFSET_RARRAY_AS_HEAP_PTR: i32 = 32; // struct RArray, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RARRAY_AS_ARY: i32 = 16; // struct RArray, subfield "as.ary" + + pub const RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR: i32 = 24; // struct RStruct, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RSTRUCT_AS_ARY: i32 = 16; // struct RStruct, subfield "as.ary" + + pub const RUBY_OFFSET_RSTRING_AS_HEAP_PTR: i32 = 24; // struct RString, subfield "as.heap.ptr" + pub const RUBY_OFFSET_RSTRING_AS_ARY: i32 = 24; // struct RString, subfield "as.embed.ary" + + // Constants from rb_control_frame_t vm_core.h + pub const RUBY_OFFSET_CFP_PC: i32 = 0; + pub const RUBY_OFFSET_CFP_SP: i32 = 8; + pub const RUBY_OFFSET_CFP_ISEQ: i32 = 16; + pub const RUBY_OFFSET_CFP_SELF: i32 = 24; + pub const RUBY_OFFSET_CFP_EP: i32 = 32; + pub const RUBY_OFFSET_CFP_BLOCK_CODE: i32 = 40; + pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; + pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; + + // Constants from rb_thread_t in vm_core.h + pub const RUBY_OFFSET_THREAD_SELF: i32 = 16; + + // Constants from iseq_inline_constant_cache (IC) and iseq_inline_constant_cache_entry (ICE) in vm_core.h + pub const RUBY_OFFSET_IC_ENTRY: i32 = 0; + pub const RUBY_OFFSET_ICE_VALUE: i32 = 8; +} +pub use manual_defs::*; + +#[cfg(test)] +pub mod test_utils { + use std::{ptr::null, sync::Once}; + + use crate::{options::{rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold, DEFAULT_CALL_THRESHOLD}, state::{rb_zjit_entry, ZJITState}}; + + use super::*; + + static RUBY_VM_INIT: Once = Once::new(); + + /// Boot and initialize the Ruby VM for Rust testing + fn boot_rubyvm() { + // Boot the VM + unsafe { + // TODO(alan): this init_stack call is incorrect. It sets the stack bottom, but + // when we return from this function will be be deeper in the stack. + // The callback for with_rubyvm() should run on a frame higher than this frame + // so the GC scans all the VALUEs on the stack. + // Consequently with_rubyvm() can only be used once per process, i.e. you can't + // boot and then run a few callbacks, because that risks putting VALUE outside + // the marked stack memory range. + // + // Need to also address the ergnomic issues addressed by + // <https://github.com/Shopify/zjit/pull/37>, though + let mut var: VALUE = Qnil; + ruby_init_stack(&mut var as *mut VALUE as *mut _); + rb_zjit_prepare_options(); // enable `#with_jit` on builtins + ruby_init(); + + // The default rb_zjit_profile_threshold is too high, so lower it for HIR tests. + if rb_zjit_call_threshold == DEFAULT_CALL_THRESHOLD { + set_call_threshold(2); + } + + // Pass command line options so the VM loads core library methods defined in + // ruby such as from `kernel.rb`. + // We drive ZJIT manually in tests, so disable heuristic compilation triggers. + // (Also, pass this in case we offer a -DFORCE_ENABLE_ZJIT option which turns + // ZJIT on by default.) + let cmdline = [c"--disable-all".as_ptr().cast_mut(), c"-e0".as_ptr().cast_mut()]; + let options_ret = ruby_options(2, cmdline.as_ptr().cast_mut()); + assert_ne!(0, ruby_executable_node(options_ret, std::ptr::null_mut()), "command-line parsing failed"); + + crate::cruby::ids::init(); // for ID! usages in tests + } + + // Set up globals for convenience + let zjit_entry = ZJITState::init(); + + // Enable zjit_* instructions + unsafe { rb_zjit_entry = zjit_entry; } + } + + /// Make sure the Ruby VM is set up and run a given callback with rb_protect() + pub fn with_rubyvm<T>(mut func: impl FnMut() -> T) -> T { + RUBY_VM_INIT.call_once(boot_rubyvm); + + // Set up a callback wrapper to store a return value + let mut result: Option<T> = None; + let mut data: &mut dyn FnMut() = &mut || { + // Store the result externally + result.replace(func()); + }; + + // Invoke callback through rb_protect() so exceptions don't crash the process. + // "Fun" double pointer dance to get a thin function pointer to pass through C + unsafe extern "C" fn callback_wrapper(data: VALUE) -> VALUE { + // SAFETY: shorter lifetime than the data local in the caller frame + let callback: &mut &mut dyn FnMut() = unsafe { std::mem::transmute(data) }; + callback(); + Qnil + } + + let mut state: c_int = 0; + unsafe { super::rb_protect(Some(callback_wrapper), VALUE((&mut data) as *mut _ as usize), &mut state) }; + if state != 0 { + unsafe { rb_zjit_print_exception(); } + assert_eq!(0, state, "Exceptional unwind in callback. Ruby exception?"); + } + + result.expect("Callback did not set result") + } + + /// Compile an ISeq via `RubyVM::InstructionSequence.compile`. + pub fn compile_to_iseq(program: &str) -> *const rb_iseq_t { + with_rubyvm(|| { + let wrapped_iseq = compile_to_wrapped_iseq(program); + unsafe { rb_iseqw_to_iseq(wrapped_iseq) } + }) + } + + pub fn define_class(name: &str, superclass: VALUE) -> VALUE { + let name = CString::new(name).unwrap(); + unsafe { rb_define_class(name.as_ptr(), superclass) } + } + + /// Evaluate a given Ruby program + pub fn eval(program: &str) -> VALUE { + with_rubyvm(|| { + let wrapped_iseq = compile_to_wrapped_iseq(&unindent(program, false)); + unsafe { rb_funcallv(wrapped_iseq, ID!(eval), 0, null()) } + }) + } + + /// Get the #inspect of a given Ruby program in Rust string + pub fn inspect(program: &str) -> String { + let inspect = format!("({program}).inspect"); + ruby_str_to_rust_string(eval(&inspect)) + } + + /// Get IseqPtr for a specified method + pub fn get_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified instance method + pub fn get_instance_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.instance_method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified Proc object + pub fn get_proc_iseq(obj: &str) -> *const rb_iseq_t { + let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of({obj})")); + unsafe { rb_iseqw_to_iseq(wrapped_iseq) } + } + + /// Remove the minimum indent from every line, skipping the first and last lines if `trim_lines`. + pub fn unindent(string: &str, trim_lines: bool) -> String { + // Break up a string into multiple lines + let mut lines: Vec<String> = string.split_inclusive("\n").map(|s| s.to_string()).collect(); + if trim_lines { // raw string literals come with extra lines + lines.remove(0); + lines.remove(lines.len() - 1); + } + + // Count the minimum number of spaces + let spaces = lines.iter().filter_map(|line| { + for (i, ch) in line.as_bytes().iter().enumerate() { + if *ch != b' ' { + return Some(i); + } + } + None + }).min().unwrap_or(0); + + // Join lines, removing spaces + let mut unindented: Vec<u8> = vec![]; + for line in lines.iter() { + if line.len() > spaces { + unindented.extend_from_slice(&line.as_bytes()[spaces..]); + } else { + unindented.extend_from_slice(line.as_bytes()); + } + } + String::from_utf8(unindented).unwrap() + } + + /// Compile a program into a RubyVM::InstructionSequence object + fn compile_to_wrapped_iseq(program: &str) -> VALUE { + let bytes = program.as_bytes().as_ptr() as *const c_char; + unsafe { + let program_str = rb_utf8_str_new(bytes, program.len().try_into().unwrap()); + rb_funcallv(rb_cISeq, ID!(compile), 1, &program_str) + } + } + + #[test] + fn boot_vm() { + // Test that we loaded kernel.rb and have Kernel#class + eval("1.class"); + } + + #[test] + #[should_panic] + fn ruby_exception_causes_panic() { + eval("raise"); + } + + #[test] + fn value_from_fixnum_in_range() { + assert_eq!(VALUE::fixnum_from_usize(2), VALUE(5)); + assert_eq!(VALUE::fixnum_from_usize(0), VALUE(1)); + assert_eq!(VALUE::fixnum_from_isize(-1), VALUE(0xffffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(-2), VALUE(0xfffffffffffffffd)); + assert_eq!(VALUE::fixnum_from_usize(RUBY_FIXNUM_MAX as usize), VALUE(0x7fffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX), VALUE(0x7fffffffffffffff)); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN), VALUE(0x8000000000000001)); + } + + #[test] + fn value_as_fixnum() { + assert_eq!(VALUE::fixnum_from_usize(2).as_fixnum(), 2); + assert_eq!(VALUE::fixnum_from_usize(0).as_fixnum(), 0); + assert_eq!(VALUE::fixnum_from_isize(-1).as_fixnum(), -1); + assert_eq!(VALUE::fixnum_from_isize(-2).as_fixnum(), -2); + assert_eq!(VALUE::fixnum_from_usize(RUBY_FIXNUM_MAX as usize).as_fixnum(), RUBY_FIXNUM_MAX.try_into().unwrap()); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX).as_fixnum(), RUBY_FIXNUM_MAX.try_into().unwrap()); + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN).as_fixnum(), RUBY_FIXNUM_MIN.try_into().unwrap()); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_big_usize() { + assert_eq!(VALUE::fixnum_from_usize((RUBY_FIXNUM_MAX+1) as usize), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_big_isize() { + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MAX+1), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_small_usize() { + assert_eq!(VALUE::fixnum_from_usize((RUBY_FIXNUM_MIN-1) as usize), VALUE(1)); + } + + #[test] + #[should_panic] + fn value_from_fixnum_too_small_isize() { + assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN-1), VALUE(1)); + } +} +#[cfg(test)] +pub use test_utils::*; + +/// Get class name from a class pointer. +pub fn get_class_name(class: VALUE) -> String { + // type checks for rb_class2name() + if unsafe { RB_TYPE_P(class, RUBY_T_MODULE) || RB_TYPE_P(class, RUBY_T_CLASS) } { + Some(class) + } else { + None + }.and_then(|class| unsafe { + cstr_to_rust_string(rb_class2name(class)) + }).unwrap_or_else(|| "Unknown".to_string()) +} + +pub fn class_has_leaf_allocator(class: VALUE) -> bool { + // empty_hash_alloc + if class == unsafe { rb_cHash } { return true; } + // empty_ary_alloc + if class == unsafe { rb_cArray } { return true; } + // empty_str_alloc + if class == unsafe { rb_cString } { return true; } + // rb_reg_s_alloc + if class == unsafe { rb_cRegexp } { return true; } + // rb_class_allocate_instance + unsafe { rb_zjit_class_has_default_allocator(class) } +} + +/// Interned ID values for Ruby symbols and method names. +/// See [type@crate::cruby::ID] and usages outside of ZJIT. +pub(crate) mod ids { + use std::sync::atomic::AtomicU64; + /// Globals to cache IDs on boot. Atomic to use with relaxed ordering + /// so reads can happen without `unsafe`. Synchronization done through + /// the VM lock. + macro_rules! def_ids { + ($(name: $name:ident $(content: $content:literal)?)*) => { + $( + pub static $name: AtomicU64 = AtomicU64::new(0); + )* + + pub(crate) fn init() { + $( + let content = stringify!($name); + _ = content; + $(let content = &$content;)? + let ptr: *const u8 = content.as_ptr(); + + // Lookup and cache each ID + $name.store( + unsafe { $crate::cruby::rb_intern2(ptr.cast(), content.len() as _) }.0, + std::sync::atomic::Ordering::Relaxed + ); + )* + + } + } + } + + def_ids! { + name: NULL content: b"" + name: respond_to_missing content: b"respond_to_missing?" + name: eq content: b"==" + name: string_eq content: b"String#==" + name: include_p content: b"include?" + name: to_ary + name: to_s + name: compile + name: eval + name: plus content: b"+" + name: minus content: b"-" + name: mult content: b"*" + name: div content: b"/" + name: modulo content: b"%" + name: neq content: b"!=" + name: lt content: b"<" + name: le content: b"<=" + name: gt content: b">" + name: ge content: b">=" + name: and content: b"&" + name: or content: b"|" + name: xor content: b"^" + name: freeze + name: minusat content: b"-@" + name: aref content: b"[]" + name: len + name: _as_heap + name: thread_ptr + name: self_ content: b"self" + name: rb_ivar_get_at_no_ractor_check + name: _shape_id + } + + /// Get an CRuby `ID` to an interned string, e.g. a particular method name. + macro_rules! ID { + ($id_name:ident) => {{ + let id = $crate::cruby::ids::$id_name.load(std::sync::atomic::Ordering::Relaxed); + debug_assert_ne!(0, id, "ids module should be initialized"); + ID(id) + }} + } + pub(crate) use ID; +} +pub(crate) use ids::ID; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs new file mode 100644 index 0000000000..15533180da --- /dev/null +++ b/zjit/src/cruby_bindings.inc.rs @@ -0,0 +1,2203 @@ +/* automatically generated by rust-bindgen 0.71.1 */ + +#[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit<Storage> { + storage: Storage, +} +impl<Storage> __BindgenBitfieldUnit<Storage> { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl<Storage> __BindgenBitfieldUnit<Storage> +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + fn extract_bit(byte: u8, index: usize) -> bool { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + Self::extract_bit(byte, index) + } + #[inline] + pub unsafe fn raw_get_bit(this: *const Self, index: usize) -> bool { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = *(core::ptr::addr_of!((*this).storage) as *const u8).offset(byte_index as isize); + Self::extract_bit(byte, index) + } + #[inline] + fn change_bit(byte: u8, index: usize, val: bool) -> u8 { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + byte | mask + } else { + byte & !mask + } + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub unsafe fn raw_set_bit(this: *mut Self, index: usize, val: bool) { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = + (core::ptr::addr_of_mut!((*this).storage) as *mut u8).offset(byte_index as isize); + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub unsafe fn raw_get(this: *const Self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if Self::raw_get_bit(this, i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } + #[inline] + pub unsafe fn raw_set(this: *mut Self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + Self::raw_set_bit(this, index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]); +impl<T> __IncompleteArrayField<T> { + #[inline] + pub const fn new() -> Self { + __IncompleteArrayField(::std::marker::PhantomData, []) + } + #[inline] + pub fn as_ptr(&self) -> *const T { + self as *const _ as *const T + } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::std::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} +impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__IncompleteArrayField") + } +} +#[repr(C)] +pub struct __BindgenUnionField<T>(::std::marker::PhantomData<T>); +impl<T> __BindgenUnionField<T> { + #[inline] + pub const fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_ref(&self) -> &T { + ::std::mem::transmute(self) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl<T> ::std::default::Default for __BindgenUnionField<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl<T> ::std::clone::Clone for __BindgenUnionField<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} +impl<T> ::std::marker::Copy for __BindgenUnionField<T> {} +impl<T> ::std::fmt::Debug for __BindgenUnionField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} +impl<T> ::std::hash::Hash for __BindgenUnionField<T> { + fn hash<H: ::std::hash::Hasher>(&self, _state: &mut H) {} +} +impl<T> ::std::cmp::PartialEq for __BindgenUnionField<T> { + fn eq(&self, _other: &__BindgenUnionField<T>) -> bool { + true + } +} +impl<T> ::std::cmp::Eq for __BindgenUnionField<T> {} +pub const ONIG_OPTION_IGNORECASE: u32 = 1; +pub const ONIG_OPTION_EXTEND: u32 = 2; +pub const ONIG_OPTION_MULTILINE: u32 = 4; +pub const ARG_ENCODING_FIXED: u32 = 16; +pub const ARG_ENCODING_NONE: u32 = 32; +pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1; +pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2; +pub const STRING_REDEFINED_OP_FLAG: u32 = 4; +pub const ARRAY_REDEFINED_OP_FLAG: u32 = 8; +pub const HASH_REDEFINED_OP_FLAG: u32 = 16; +pub const SYMBOL_REDEFINED_OP_FLAG: u32 = 64; +pub const TIME_REDEFINED_OP_FLAG: u32 = 128; +pub const REGEXP_REDEFINED_OP_FLAG: u32 = 256; +pub const NIL_REDEFINED_OP_FLAG: u32 = 512; +pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024; +pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048; +pub const PROC_REDEFINED_OP_FLAG: u32 = 4096; +pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31; +pub const VM_ENV_DATA_SIZE: u32 = 3; +pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2; +pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1; +pub const VM_ENV_DATA_INDEX_FLAGS: u32 = 0; +pub const VM_BLOCK_HANDLER_NONE: u32 = 0; +pub const SHAPE_ID_NUM_BITS: u32 = 32; +pub type rb_alloc_func_t = ::std::option::Option<unsafe extern "C" fn(klass: VALUE) -> VALUE>; +pub const RUBY_Qfalse: ruby_special_consts = 0; +pub const RUBY_Qnil: ruby_special_consts = 4; +pub const RUBY_Qtrue: ruby_special_consts = 20; +pub const RUBY_Qundef: ruby_special_consts = 36; +pub const RUBY_IMMEDIATE_MASK: ruby_special_consts = 7; +pub const RUBY_FIXNUM_FLAG: ruby_special_consts = 1; +pub const RUBY_FLONUM_MASK: ruby_special_consts = 3; +pub const RUBY_FLONUM_FLAG: ruby_special_consts = 2; +pub const RUBY_SYMBOL_FLAG: ruby_special_consts = 12; +pub const RUBY_SPECIAL_SHIFT: ruby_special_consts = 8; +pub type ruby_special_consts = u32; +#[repr(C)] +pub struct RBasic { + pub flags: VALUE, + pub klass: VALUE, +} +pub const RUBY_T_NONE: ruby_value_type = 0; +pub const RUBY_T_OBJECT: ruby_value_type = 1; +pub const RUBY_T_CLASS: ruby_value_type = 2; +pub const RUBY_T_MODULE: ruby_value_type = 3; +pub const RUBY_T_FLOAT: ruby_value_type = 4; +pub const RUBY_T_STRING: ruby_value_type = 5; +pub const RUBY_T_REGEXP: ruby_value_type = 6; +pub const RUBY_T_ARRAY: ruby_value_type = 7; +pub const RUBY_T_HASH: ruby_value_type = 8; +pub const RUBY_T_STRUCT: ruby_value_type = 9; +pub const RUBY_T_BIGNUM: ruby_value_type = 10; +pub const RUBY_T_FILE: ruby_value_type = 11; +pub const RUBY_T_DATA: ruby_value_type = 12; +pub const RUBY_T_MATCH: ruby_value_type = 13; +pub const RUBY_T_COMPLEX: ruby_value_type = 14; +pub const RUBY_T_RATIONAL: ruby_value_type = 15; +pub const RUBY_T_NIL: ruby_value_type = 17; +pub const RUBY_T_TRUE: ruby_value_type = 18; +pub const RUBY_T_FALSE: ruby_value_type = 19; +pub const RUBY_T_SYMBOL: ruby_value_type = 20; +pub const RUBY_T_FIXNUM: ruby_value_type = 21; +pub const RUBY_T_UNDEF: ruby_value_type = 22; +pub const RUBY_T_IMEMO: ruby_value_type = 26; +pub const RUBY_T_NODE: ruby_value_type = 27; +pub const RUBY_T_ICLASS: ruby_value_type = 28; +pub const RUBY_T_ZOMBIE: ruby_value_type = 29; +pub const RUBY_T_MOVED: ruby_value_type = 30; +pub const RUBY_T_MASK: ruby_value_type = 31; +pub type ruby_value_type = u32; +pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12; +pub type ruby_fl_ushift = u32; +pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; +pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; +pub const RUBY_FL_USERPRIV0: ruby_fl_type = 64; +pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; +pub const RUBY_FL_EXIVAR: ruby_fl_type = 0; +pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; +pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512; +pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024; +pub const RUBY_FL_FREEZE: ruby_fl_type = 2048; +pub const RUBY_FL_USER0: ruby_fl_type = 4096; +pub const RUBY_FL_USER1: ruby_fl_type = 8192; +pub const RUBY_FL_USER2: ruby_fl_type = 16384; +pub const RUBY_FL_USER3: ruby_fl_type = 32768; +pub const RUBY_FL_USER4: ruby_fl_type = 65536; +pub const RUBY_FL_USER5: ruby_fl_type = 131072; +pub const RUBY_FL_USER6: ruby_fl_type = 262144; +pub const RUBY_FL_USER7: ruby_fl_type = 524288; +pub const RUBY_FL_USER8: ruby_fl_type = 1048576; +pub const RUBY_FL_USER9: ruby_fl_type = 2097152; +pub const RUBY_FL_USER10: ruby_fl_type = 4194304; +pub const RUBY_FL_USER11: ruby_fl_type = 8388608; +pub const RUBY_FL_USER12: ruby_fl_type = 16777216; +pub const RUBY_FL_USER13: ruby_fl_type = 33554432; +pub const RUBY_FL_USER14: ruby_fl_type = 67108864; +pub const RUBY_FL_USER15: ruby_fl_type = 134217728; +pub const RUBY_FL_USER16: ruby_fl_type = 268435456; +pub const RUBY_FL_USER17: ruby_fl_type = 536870912; +pub const RUBY_FL_USER18: ruby_fl_type = 1073741824; +pub const RUBY_FL_USER19: ruby_fl_type = -2147483648; +pub const RUBY_ELTS_SHARED: ruby_fl_type = 4096; +pub const RUBY_FL_SINGLETON: ruby_fl_type = 8192; +pub type ruby_fl_type = i32; +pub const RSTRING_NOEMBED: ruby_rstring_flags = 8192; +pub const RSTRING_FSTR: ruby_rstring_flags = 536870912; +pub type ruby_rstring_flags = u32; +pub type st_data_t = ::std::os::raw::c_ulong; +pub type st_index_t = st_data_t; +pub const ST_CONTINUE: st_retval = 0; +pub const ST_STOP: st_retval = 1; +pub const ST_DELETE: st_retval = 2; +pub const ST_CHECK: st_retval = 3; +pub const ST_REPLACE: st_retval = 4; +pub type st_retval = u32; +pub type st_foreach_callback_func = ::std::option::Option< + unsafe extern "C" fn( + arg1: st_data_t, + arg2: st_data_t, + arg3: st_data_t, + ) -> ::std::os::raw::c_int, +>; +pub const RARRAY_EMBED_FLAG: ruby_rarray_flags = 8192; +pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536; +pub type ruby_rarray_flags = u32; +pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; +pub type ruby_rarray_consts = u32; +pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192; +pub type ruby_rmodule_flags = u32; +pub const ROBJECT_HEAP: ruby_robject_flags = 65536; +pub type ruby_robject_flags = u32; +pub type rb_event_flag_t = u32; +pub type rb_block_call_func = ::std::option::Option< + unsafe extern "C" fn( + yielded_arg: VALUE, + callback_arg: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + blockarg: VALUE, + ) -> VALUE, +>; +pub type rb_block_call_func_t = rb_block_call_func; +pub const RUBY_ENCODING_INLINE_MAX: ruby_encoding_consts = 127; +pub const RUBY_ENCODING_SHIFT: ruby_encoding_consts = 22; +pub const RUBY_ENCODING_MASK: ruby_encoding_consts = 532676608; +pub const RUBY_ENCODING_MAXNAMELEN: ruby_encoding_consts = 42; +pub type ruby_encoding_consts = u32; +pub const RUBY_ENCINDEX_ASCII_8BIT: ruby_preserved_encindex = 0; +pub const RUBY_ENCINDEX_UTF_8: ruby_preserved_encindex = 1; +pub const RUBY_ENCINDEX_US_ASCII: ruby_preserved_encindex = 2; +pub const RUBY_ENCINDEX_UTF_16BE: ruby_preserved_encindex = 3; +pub const RUBY_ENCINDEX_UTF_16LE: ruby_preserved_encindex = 4; +pub const RUBY_ENCINDEX_UTF_32BE: ruby_preserved_encindex = 5; +pub const RUBY_ENCINDEX_UTF_32LE: ruby_preserved_encindex = 6; +pub const RUBY_ENCINDEX_UTF_16: ruby_preserved_encindex = 7; +pub const RUBY_ENCINDEX_UTF_32: ruby_preserved_encindex = 8; +pub const RUBY_ENCINDEX_UTF8_MAC: ruby_preserved_encindex = 9; +pub const RUBY_ENCINDEX_EUC_JP: ruby_preserved_encindex = 10; +pub const RUBY_ENCINDEX_Windows_31J: ruby_preserved_encindex = 11; +pub const RUBY_ENCINDEX_BUILTIN_MAX: ruby_preserved_encindex = 12; +pub type ruby_preserved_encindex = u32; +pub const BOP_PLUS: ruby_basic_operators = 0; +pub const BOP_MINUS: ruby_basic_operators = 1; +pub const BOP_MULT: ruby_basic_operators = 2; +pub const BOP_DIV: ruby_basic_operators = 3; +pub const BOP_MOD: ruby_basic_operators = 4; +pub const BOP_EQ: ruby_basic_operators = 5; +pub const BOP_EQQ: ruby_basic_operators = 6; +pub const BOP_LT: ruby_basic_operators = 7; +pub const BOP_LE: ruby_basic_operators = 8; +pub const BOP_LTLT: ruby_basic_operators = 9; +pub const BOP_AREF: ruby_basic_operators = 10; +pub const BOP_ASET: ruby_basic_operators = 11; +pub const BOP_LENGTH: ruby_basic_operators = 12; +pub const BOP_SIZE: ruby_basic_operators = 13; +pub const BOP_EMPTY_P: ruby_basic_operators = 14; +pub const BOP_NIL_P: ruby_basic_operators = 15; +pub const BOP_SUCC: ruby_basic_operators = 16; +pub const BOP_GT: ruby_basic_operators = 17; +pub const BOP_GE: ruby_basic_operators = 18; +pub const BOP_GTGT: ruby_basic_operators = 19; +pub const BOP_NOT: ruby_basic_operators = 20; +pub const BOP_NEQ: ruby_basic_operators = 21; +pub const BOP_MATCH: ruby_basic_operators = 22; +pub const BOP_FREEZE: ruby_basic_operators = 23; +pub const BOP_UMINUS: ruby_basic_operators = 24; +pub const BOP_MAX: ruby_basic_operators = 25; +pub const BOP_MIN: ruby_basic_operators = 26; +pub const BOP_HASH: ruby_basic_operators = 27; +pub const BOP_CALL: ruby_basic_operators = 28; +pub const BOP_AND: ruby_basic_operators = 29; +pub const BOP_OR: ruby_basic_operators = 30; +pub const BOP_CMP: ruby_basic_operators = 31; +pub const BOP_DEFAULT: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 33; +pub const BOP_INCLUDE_P: ruby_basic_operators = 34; +pub const BOP_LAST_: ruby_basic_operators = 35; +pub type ruby_basic_operators = u32; +pub type rb_serial_t = ::std::os::raw::c_ulonglong; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_id_table { + _unused: [u8; 0], +} +pub const imemo_env: imemo_type = 0; +pub const imemo_cref: imemo_type = 1; +pub const imemo_svar: imemo_type = 2; +pub const imemo_throw_data: imemo_type = 3; +pub const imemo_ifunc: imemo_type = 4; +pub const imemo_memo: imemo_type = 5; +pub const imemo_ment: imemo_type = 6; +pub const imemo_iseq: imemo_type = 7; +pub const imemo_tmpbuf: imemo_type = 8; +pub const imemo_callinfo: imemo_type = 10; +pub const imemo_callcache: imemo_type = 11; +pub const imemo_constcache: imemo_type = 12; +pub const imemo_fields: imemo_type = 13; +pub type imemo_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vm_ifunc_argc { + pub min: ::std::os::raw::c_int, + pub max: ::std::os::raw::c_int, +} +#[repr(C)] +pub struct vm_ifunc { + pub flags: VALUE, + pub svar_lep: *mut VALUE, + pub func: rb_block_call_func_t, + pub data: *const ::std::os::raw::c_void, + pub argc: vm_ifunc_argc, +} +pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; +pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; +pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2; +pub const METHOD_VISI_PROTECTED: rb_method_visibility_t = 3; +pub const METHOD_VISI_MASK: rb_method_visibility_t = 3; +pub type rb_method_visibility_t = u32; +#[repr(C)] +pub struct rb_method_entry_struct { + pub flags: VALUE, + pub defined_class: VALUE, + pub def: *mut rb_method_definition_struct, + pub called_id: ID, + pub owner: VALUE, +} +pub type rb_method_entry_t = rb_method_entry_struct; +#[repr(C)] +pub struct rb_callable_method_entry_struct { + pub flags: VALUE, + pub defined_class: VALUE, + pub def: *mut rb_method_definition_struct, + pub called_id: ID, + pub owner: VALUE, +} +pub type rb_callable_method_entry_t = rb_callable_method_entry_struct; +pub const VM_METHOD_TYPE_ISEQ: rb_method_type_t = 0; +pub const VM_METHOD_TYPE_CFUNC: rb_method_type_t = 1; +pub const VM_METHOD_TYPE_ATTRSET: rb_method_type_t = 2; +pub const VM_METHOD_TYPE_IVAR: rb_method_type_t = 3; +pub const VM_METHOD_TYPE_BMETHOD: rb_method_type_t = 4; +pub const VM_METHOD_TYPE_ZSUPER: rb_method_type_t = 5; +pub const VM_METHOD_TYPE_ALIAS: rb_method_type_t = 6; +pub const VM_METHOD_TYPE_UNDEF: rb_method_type_t = 7; +pub const VM_METHOD_TYPE_NOTIMPLEMENTED: rb_method_type_t = 8; +pub const VM_METHOD_TYPE_OPTIMIZED: rb_method_type_t = 9; +pub const VM_METHOD_TYPE_MISSING: rb_method_type_t = 10; +pub const VM_METHOD_TYPE_REFINED: rb_method_type_t = 11; +pub type rb_method_type_t = u32; +pub type rb_iseq_t = rb_iseq_struct; +pub type rb_cfunc_t = ::std::option::Option<unsafe extern "C" fn() -> VALUE>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_method_cfunc_struct { + pub func: rb_cfunc_t, + pub invoker: ::std::option::Option< + unsafe extern "C" fn( + recv: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + func: ::std::option::Option<unsafe extern "C" fn() -> VALUE>, + ) -> VALUE, + >, + pub argc: ::std::os::raw::c_int, +} +pub const OPTIMIZED_METHOD_TYPE_SEND: method_optimized_type = 0; +pub const OPTIMIZED_METHOD_TYPE_CALL: method_optimized_type = 1; +pub const OPTIMIZED_METHOD_TYPE_BLOCK_CALL: method_optimized_type = 2; +pub const OPTIMIZED_METHOD_TYPE_STRUCT_AREF: method_optimized_type = 3; +pub const OPTIMIZED_METHOD_TYPE_STRUCT_ASET: method_optimized_type = 4; +pub const OPTIMIZED_METHOD_TYPE__MAX: method_optimized_type = 5; +pub type method_optimized_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_position_struct { + pub lineno: ::std::os::raw::c_int, + pub column: ::std::os::raw::c_int, +} +pub type rb_code_position_t = rb_code_position_struct; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_location_struct { + pub beg_pos: rb_code_position_t, + pub end_pos: rb_code_position_t, +} +pub type rb_code_location_t = rb_code_location_struct; +pub type rb_num_t = ::std::os::raw::c_ulong; +pub type rb_snum_t = ::std::os::raw::c_long; +pub const RUBY_TAG_NONE: ruby_tag_type = 0; +pub const RUBY_TAG_RETURN: ruby_tag_type = 1; +pub const RUBY_TAG_BREAK: ruby_tag_type = 2; +pub const RUBY_TAG_NEXT: ruby_tag_type = 3; +pub const RUBY_TAG_RETRY: ruby_tag_type = 4; +pub const RUBY_TAG_REDO: ruby_tag_type = 5; +pub const RUBY_TAG_RAISE: ruby_tag_type = 6; +pub const RUBY_TAG_THROW: ruby_tag_type = 7; +pub const RUBY_TAG_FATAL: ruby_tag_type = 8; +pub const RUBY_TAG_MASK: ruby_tag_type = 15; +pub type ruby_tag_type = u32; +pub const VM_THROW_NO_ESCAPE_FLAG: ruby_vm_throw_flags = 32768; +pub const VM_THROW_STATE_MASK: ruby_vm_throw_flags = 255; +pub type ruby_vm_throw_flags = u32; +#[repr(C)] +pub struct iseq_inline_constant_cache_entry { + pub flags: VALUE, + pub value: VALUE, + pub ic_cref: *const rb_cref_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_inline_constant_cache { + pub entry: *mut iseq_inline_constant_cache_entry, + pub segments: *const ID, +} +#[repr(C)] +pub struct iseq_inline_iv_cache_entry { + pub value: u64, + pub iv_set_name: ID, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_inline_cvar_cache_entry { + pub entry: *mut rb_cvar_class_tbl_entry, +} +#[repr(C)] +#[repr(align(8))] +#[derive(Copy, Clone)] +pub struct iseq_inline_storage_entry { + pub _bindgen_opaque_blob: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_location_struct { + pub pathobj: VALUE, + pub base_label: VALUE, + pub label: VALUE, + pub first_lineno: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub code_location: rb_code_location_t, +} +pub type rb_iseq_location_t = rb_iseq_location_struct; +pub type iseq_bits_t = usize; +pub const ISEQ_TYPE_TOP: rb_iseq_type = 0; +pub const ISEQ_TYPE_METHOD: rb_iseq_type = 1; +pub const ISEQ_TYPE_BLOCK: rb_iseq_type = 2; +pub const ISEQ_TYPE_CLASS: rb_iseq_type = 3; +pub const ISEQ_TYPE_RESCUE: rb_iseq_type = 4; +pub const ISEQ_TYPE_ENSURE: rb_iseq_type = 5; +pub const ISEQ_TYPE_EVAL: rb_iseq_type = 6; +pub const ISEQ_TYPE_MAIN: rb_iseq_type = 7; +pub const ISEQ_TYPE_PLAIN: rb_iseq_type = 8; +pub type rb_iseq_type = u32; +pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1; +pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2; +pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4; +pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8; +pub type rb_builtin_attr = u32; +pub type rb_jit_func_t = ::std::option::Option< + unsafe extern "C" fn( + arg1: *mut rb_execution_context_struct, + arg2: *mut rb_control_frame_struct, + ) -> VALUE, +>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters { + pub flags: rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1, + pub size: ::std::os::raw::c_uint, + pub lead_num: ::std::os::raw::c_int, + pub opt_num: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub post_start: ::std::os::raw::c_int, + pub post_num: ::std::os::raw::c_int, + pub block_start: ::std::os::raw::c_int, + pub opt_table: *const VALUE, + pub keyword: *const rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword, +} +#[repr(C)] +#[repr(align(4))] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 2usize]>, + pub __bindgen_padding_0: u16, +} +impl rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + #[inline] + pub fn has_lead(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_lead(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_lead_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_lead_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_opt(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_opt(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_opt_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_opt_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_post(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_post(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_post_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 3usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_post_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 3usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kw(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kw(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kw_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 4usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kw_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 4usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 5usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 5usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 6usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 6usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ambiguous_param0(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u32) } + } + #[inline] + pub fn set_ambiguous_param0(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ambiguous_param0_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 7usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ambiguous_param0_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 7usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_kwarg(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_kwarg(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_kwarg_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 8usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_kwarg_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 8usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ruby2_keywords(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u32) } + } + #[inline] + pub fn set_ruby2_keywords(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ruby2_keywords_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 9usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ruby2_keywords_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 9usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 10usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 10usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 11usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 11usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn use_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u32) } + } + #[inline] + pub fn set_use_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn use_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 12usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_use_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 12usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn forwardable(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u32) } + } + #[inline] + pub fn set_forwardable(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn forwardable_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 13usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_forwardable_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 13usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + has_lead: ::std::os::raw::c_uint, + has_opt: ::std::os::raw::c_uint, + has_rest: ::std::os::raw::c_uint, + has_post: ::std::os::raw::c_uint, + has_kw: ::std::os::raw::c_uint, + has_kwrest: ::std::os::raw::c_uint, + has_block: ::std::os::raw::c_uint, + ambiguous_param0: ::std::os::raw::c_uint, + accepts_no_kwarg: ::std::os::raw::c_uint, + ruby2_keywords: ::std::os::raw::c_uint, + anon_rest: ::std::os::raw::c_uint, + anon_kwrest: ::std::os::raw::c_uint, + use_block: ::std::os::raw::c_uint, + forwardable: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 2usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 2usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let has_lead: u32 = unsafe { ::std::mem::transmute(has_lead) }; + has_lead as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let has_opt: u32 = unsafe { ::std::mem::transmute(has_opt) }; + has_opt as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let has_rest: u32 = unsafe { ::std::mem::transmute(has_rest) }; + has_rest as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let has_post: u32 = unsafe { ::std::mem::transmute(has_post) }; + has_post as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let has_kw: u32 = unsafe { ::std::mem::transmute(has_kw) }; + has_kw as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let has_kwrest: u32 = unsafe { ::std::mem::transmute(has_kwrest) }; + has_kwrest as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let has_block: u32 = unsafe { ::std::mem::transmute(has_block) }; + has_block as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let ambiguous_param0: u32 = unsafe { ::std::mem::transmute(ambiguous_param0) }; + ambiguous_param0 as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let accepts_no_kwarg: u32 = unsafe { ::std::mem::transmute(accepts_no_kwarg) }; + accepts_no_kwarg as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let ruby2_keywords: u32 = unsafe { ::std::mem::transmute(ruby2_keywords) }; + ruby2_keywords as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let anon_rest: u32 = unsafe { ::std::mem::transmute(anon_rest) }; + anon_rest as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let anon_kwrest: u32 = unsafe { ::std::mem::transmute(anon_kwrest) }; + anon_kwrest as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let use_block: u32 = unsafe { ::std::mem::transmute(use_block) }; + use_block as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let forwardable: u32 = unsafe { ::std::mem::transmute(forwardable) }; + forwardable as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword { + pub num: ::std::os::raw::c_int, + pub required_num: ::std::os::raw::c_int, + pub bits_start: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub table: *const ID, + pub default_values: *mut VALUE, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_iseq_insn_info { + pub body: *const iseq_insn_info_entry, + pub positions: *mut ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub succ_index_table: *mut succ_index_table, +} +pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; +pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; +pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; +pub type rb_iseq_constant_body_lvar_state = u32; +#[repr(C)] +pub struct rb_iseq_constant_body__bindgen_ty_1 { + pub flip_count: rb_snum_t, + pub script_lines: VALUE, + pub coverage: VALUE, + pub pc2branchindex: VALUE, + pub original_iseq: *mut VALUE, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union rb_iseq_constant_body__bindgen_ty_2 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +pub struct rb_iseq_struct { + pub flags: VALUE, + pub wrapper: VALUE, + pub body: *mut rb_iseq_constant_body, + pub aux: rb_iseq_struct__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1 { + pub compile_data: __BindgenUnionField<*mut iseq_compile_data>, + pub loader: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_1>, + pub exec: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_2>, + pub bindgen_union_field: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_1 { + pub obj: VALUE, + pub index: ::std::os::raw::c_int, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_2 { + pub local_hooks_cnt: ::std::os::raw::c_uint, + pub global_trace_events: rb_event_flag_t, +} +#[repr(C)] +pub struct rb_captured_block { + pub self_: VALUE, + pub ep: *const VALUE, + pub code: rb_captured_block__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_captured_block__bindgen_ty_1 { + pub iseq: __BindgenUnionField<*const rb_iseq_t>, + pub ifunc: __BindgenUnionField<*const vm_ifunc>, + pub val: __BindgenUnionField<VALUE>, + pub bindgen_union_field: u64, +} +pub const block_type_iseq: rb_block_type = 0; +pub const block_type_ifunc: rb_block_type = 1; +pub const block_type_symbol: rb_block_type = 2; +pub const block_type_proc: rb_block_type = 3; +pub type rb_block_type = u32; +#[repr(C)] +pub struct rb_block { + pub as_: rb_block__bindgen_ty_1, + pub type_: rb_block_type, +} +#[repr(C)] +pub struct rb_block__bindgen_ty_1 { + pub captured: __BindgenUnionField<rb_captured_block>, + pub symbol: __BindgenUnionField<VALUE>, + pub proc_: __BindgenUnionField<VALUE>, + pub bindgen_union_field: [u64; 3usize], +} +pub type rb_control_frame_t = rb_control_frame_struct; +#[repr(C)] +pub struct rb_proc_t { + pub block: rb_block, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +impl rb_proc_t { + #[inline] + pub fn is_from_method(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_from_method(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_from_method_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_from_method_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_lambda(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_lambda(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_lambda_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_lambda_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_isolated(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_isolated(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_isolated_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_isolated_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + is_from_method: ::std::os::raw::c_uint, + is_lambda: ::std::os::raw::c_uint, + is_isolated: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let is_from_method: u32 = unsafe { ::std::mem::transmute(is_from_method) }; + is_from_method as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let is_lambda: u32 = unsafe { ::std::mem::transmute(is_lambda) }; + is_lambda as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let is_isolated: u32 = unsafe { ::std::mem::transmute(is_isolated) }; + is_isolated as u64 + }); + __bindgen_bitfield_unit + } +} +pub const VM_CHECKMATCH_TYPE_WHEN: vm_check_match_type = 1; +pub const VM_CHECKMATCH_TYPE_CASE: vm_check_match_type = 2; +pub const VM_CHECKMATCH_TYPE_RESCUE: vm_check_match_type = 3; +pub type vm_check_match_type = u32; +pub const VM_OPT_NEWARRAY_SEND_MAX: vm_opt_newarray_send_type = 1; +pub const VM_OPT_NEWARRAY_SEND_MIN: vm_opt_newarray_send_type = 2; +pub const VM_OPT_NEWARRAY_SEND_HASH: vm_opt_newarray_send_type = 3; +pub const VM_OPT_NEWARRAY_SEND_PACK: vm_opt_newarray_send_type = 4; +pub const VM_OPT_NEWARRAY_SEND_PACK_BUFFER: vm_opt_newarray_send_type = 5; +pub const VM_OPT_NEWARRAY_SEND_INCLUDE_P: vm_opt_newarray_send_type = 6; +pub type vm_opt_newarray_send_type = u32; +pub const VM_SPECIAL_OBJECT_VMCORE: vm_special_object_type = 1; +pub const VM_SPECIAL_OBJECT_CBASE: vm_special_object_type = 2; +pub const VM_SPECIAL_OBJECT_CONST_BASE: vm_special_object_type = 3; +pub type vm_special_object_type = u32; +pub type IC = *mut iseq_inline_constant_cache; +pub type IVC = *mut iseq_inline_iv_cache_entry; +pub type ICVARC = *mut iseq_inline_cvar_cache_entry; +pub const VM_FRAME_MAGIC_METHOD: vm_frame_env_flags = 286326785; +pub const VM_FRAME_MAGIC_BLOCK: vm_frame_env_flags = 572653569; +pub const VM_FRAME_MAGIC_CLASS: vm_frame_env_flags = 858980353; +pub const VM_FRAME_MAGIC_TOP: vm_frame_env_flags = 1145307137; +pub const VM_FRAME_MAGIC_CFUNC: vm_frame_env_flags = 1431633921; +pub const VM_FRAME_MAGIC_IFUNC: vm_frame_env_flags = 1717960705; +pub const VM_FRAME_MAGIC_EVAL: vm_frame_env_flags = 2004287489; +pub const VM_FRAME_MAGIC_RESCUE: vm_frame_env_flags = 2022178817; +pub const VM_FRAME_MAGIC_DUMMY: vm_frame_env_flags = 2040070145; +pub const VM_FRAME_MAGIC_MASK: vm_frame_env_flags = 2147418113; +pub const VM_FRAME_FLAG_FINISH: vm_frame_env_flags = 32; +pub const VM_FRAME_FLAG_BMETHOD: vm_frame_env_flags = 64; +pub const VM_FRAME_FLAG_CFRAME: vm_frame_env_flags = 128; +pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256; +pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512; +pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024; +pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048; +pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096; +pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2; +pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4; +pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8; +pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; +pub type vm_frame_env_flags = u32; +pub type attr_index_t = u16; +pub type shape_id_t = u32; +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 29360128; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 33554432; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 67108864; +pub const SHAPE_ID_FL_TOO_COMPLEX: shape_id_fl_type = 134217728; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 100663296; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 264241152; +pub type shape_id_fl_type = u32; +#[repr(C)] +pub struct rb_cvar_class_tbl_entry { + pub index: u32, + pub global_cvar_state: rb_serial_t, + pub cref: *const rb_cref_t, + pub class_value: VALUE, +} +pub const VM_CALL_ARGS_SPLAT_bit: vm_call_flag_bits = 0; +pub const VM_CALL_ARGS_BLOCKARG_bit: vm_call_flag_bits = 1; +pub const VM_CALL_FCALL_bit: vm_call_flag_bits = 2; +pub const VM_CALL_VCALL_bit: vm_call_flag_bits = 3; +pub const VM_CALL_ARGS_SIMPLE_bit: vm_call_flag_bits = 4; +pub const VM_CALL_KWARG_bit: vm_call_flag_bits = 5; +pub const VM_CALL_KW_SPLAT_bit: vm_call_flag_bits = 6; +pub const VM_CALL_TAILCALL_bit: vm_call_flag_bits = 7; +pub const VM_CALL_SUPER_bit: vm_call_flag_bits = 8; +pub const VM_CALL_ZSUPER_bit: vm_call_flag_bits = 9; +pub const VM_CALL_OPT_SEND_bit: vm_call_flag_bits = 10; +pub const VM_CALL_KW_SPLAT_MUT_bit: vm_call_flag_bits = 11; +pub const VM_CALL_ARGS_SPLAT_MUT_bit: vm_call_flag_bits = 12; +pub const VM_CALL_FORWARDING_bit: vm_call_flag_bits = 13; +pub const VM_CALL__END: vm_call_flag_bits = 14; +pub type vm_call_flag_bits = u32; +#[repr(C)] +pub struct rb_callinfo_kwarg { + pub keyword_len: ::std::os::raw::c_int, + pub references: ::std::os::raw::c_int, + pub keywords: __IncompleteArrayField<VALUE>, +} +#[repr(C)] +pub struct rb_callinfo { + pub flags: VALUE, + pub kwarg: *const rb_callinfo_kwarg, + pub mid: VALUE, + pub flag: VALUE, + pub argc: VALUE, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_call_data { + pub ci: *const rb_callinfo, + pub cc: *const rb_callcache, +} +pub const RSTRING_CHILLED: ruby_rstring_private_flags = 49152; +pub type ruby_rstring_private_flags = u32; +pub const RHASH_PASS_AS_KEYWORDS: ruby_rhash_flags = 8192; +pub const RHASH_PROC_DEFAULT: ruby_rhash_flags = 16384; +pub const RHASH_ST_TABLE_FLAG: ruby_rhash_flags = 32768; +pub const RHASH_AR_TABLE_SIZE_MASK: ruby_rhash_flags = 983040; +pub const RHASH_AR_TABLE_SIZE_SHIFT: ruby_rhash_flags = 16; +pub const RHASH_AR_TABLE_BOUND_MASK: ruby_rhash_flags = 15728640; +pub const RHASH_AR_TABLE_BOUND_SHIFT: ruby_rhash_flags = 20; +pub const RHASH_LEV_SHIFT: ruby_rhash_flags = 25; +pub const RHASH_LEV_MAX: ruby_rhash_flags = 127; +pub type ruby_rhash_flags = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_builtin_function { + pub func_ptr: *const ::std::os::raw::c_void, + pub argc: ::std::os::raw::c_int, + pub index: ::std::os::raw::c_int, + pub name: *const ::std::os::raw::c_char, +} +pub const YARVINSN_nop: ruby_vminsn_type = 0; +pub const YARVINSN_getlocal: ruby_vminsn_type = 1; +pub const YARVINSN_setlocal: ruby_vminsn_type = 2; +pub const YARVINSN_getblockparam: ruby_vminsn_type = 3; +pub const YARVINSN_setblockparam: ruby_vminsn_type = 4; +pub const YARVINSN_getblockparamproxy: ruby_vminsn_type = 5; +pub const YARVINSN_getspecial: ruby_vminsn_type = 6; +pub const YARVINSN_setspecial: ruby_vminsn_type = 7; +pub const YARVINSN_getinstancevariable: ruby_vminsn_type = 8; +pub const YARVINSN_setinstancevariable: ruby_vminsn_type = 9; +pub const YARVINSN_getclassvariable: ruby_vminsn_type = 10; +pub const YARVINSN_setclassvariable: ruby_vminsn_type = 11; +pub const YARVINSN_opt_getconstant_path: ruby_vminsn_type = 12; +pub const YARVINSN_getconstant: ruby_vminsn_type = 13; +pub const YARVINSN_setconstant: ruby_vminsn_type = 14; +pub const YARVINSN_getglobal: ruby_vminsn_type = 15; +pub const YARVINSN_setglobal: ruby_vminsn_type = 16; +pub const YARVINSN_putnil: ruby_vminsn_type = 17; +pub const YARVINSN_putself: ruby_vminsn_type = 18; +pub const YARVINSN_putobject: ruby_vminsn_type = 19; +pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20; +pub const YARVINSN_putstring: ruby_vminsn_type = 21; +pub const YARVINSN_putchilledstring: ruby_vminsn_type = 22; +pub const YARVINSN_concatstrings: ruby_vminsn_type = 23; +pub const YARVINSN_anytostring: ruby_vminsn_type = 24; +pub const YARVINSN_toregexp: ruby_vminsn_type = 25; +pub const YARVINSN_intern: ruby_vminsn_type = 26; +pub const YARVINSN_newarray: ruby_vminsn_type = 27; +pub const YARVINSN_pushtoarraykwsplat: ruby_vminsn_type = 28; +pub const YARVINSN_duparray: ruby_vminsn_type = 29; +pub const YARVINSN_duphash: ruby_vminsn_type = 30; +pub const YARVINSN_expandarray: ruby_vminsn_type = 31; +pub const YARVINSN_concatarray: ruby_vminsn_type = 32; +pub const YARVINSN_concattoarray: ruby_vminsn_type = 33; +pub const YARVINSN_pushtoarray: ruby_vminsn_type = 34; +pub const YARVINSN_splatarray: ruby_vminsn_type = 35; +pub const YARVINSN_splatkw: ruby_vminsn_type = 36; +pub const YARVINSN_newhash: ruby_vminsn_type = 37; +pub const YARVINSN_newrange: ruby_vminsn_type = 38; +pub const YARVINSN_pop: ruby_vminsn_type = 39; +pub const YARVINSN_dup: ruby_vminsn_type = 40; +pub const YARVINSN_dupn: ruby_vminsn_type = 41; +pub const YARVINSN_swap: ruby_vminsn_type = 42; +pub const YARVINSN_opt_reverse: ruby_vminsn_type = 43; +pub const YARVINSN_topn: ruby_vminsn_type = 44; +pub const YARVINSN_setn: ruby_vminsn_type = 45; +pub const YARVINSN_adjuststack: ruby_vminsn_type = 46; +pub const YARVINSN_defined: ruby_vminsn_type = 47; +pub const YARVINSN_definedivar: ruby_vminsn_type = 48; +pub const YARVINSN_checkmatch: ruby_vminsn_type = 49; +pub const YARVINSN_checkkeyword: ruby_vminsn_type = 50; +pub const YARVINSN_checktype: ruby_vminsn_type = 51; +pub const YARVINSN_defineclass: ruby_vminsn_type = 52; +pub const YARVINSN_definemethod: ruby_vminsn_type = 53; +pub const YARVINSN_definesmethod: ruby_vminsn_type = 54; +pub const YARVINSN_send: ruby_vminsn_type = 55; +pub const YARVINSN_sendforward: ruby_vminsn_type = 56; +pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 57; +pub const YARVINSN_opt_new: ruby_vminsn_type = 58; +pub const YARVINSN_objtostring: ruby_vminsn_type = 59; +pub const YARVINSN_opt_ary_freeze: ruby_vminsn_type = 60; +pub const YARVINSN_opt_hash_freeze: ruby_vminsn_type = 61; +pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 62; +pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 63; +pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 64; +pub const YARVINSN_opt_duparray_send: ruby_vminsn_type = 65; +pub const YARVINSN_opt_newarray_send: ruby_vminsn_type = 66; +pub const YARVINSN_invokesuper: ruby_vminsn_type = 67; +pub const YARVINSN_invokesuperforward: ruby_vminsn_type = 68; +pub const YARVINSN_invokeblock: ruby_vminsn_type = 69; +pub const YARVINSN_leave: ruby_vminsn_type = 70; +pub const YARVINSN_throw: ruby_vminsn_type = 71; +pub const YARVINSN_jump: ruby_vminsn_type = 72; +pub const YARVINSN_branchif: ruby_vminsn_type = 73; +pub const YARVINSN_branchunless: ruby_vminsn_type = 74; +pub const YARVINSN_branchnil: ruby_vminsn_type = 75; +pub const YARVINSN_once: ruby_vminsn_type = 76; +pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 77; +pub const YARVINSN_opt_plus: ruby_vminsn_type = 78; +pub const YARVINSN_opt_minus: ruby_vminsn_type = 79; +pub const YARVINSN_opt_mult: ruby_vminsn_type = 80; +pub const YARVINSN_opt_div: ruby_vminsn_type = 81; +pub const YARVINSN_opt_mod: ruby_vminsn_type = 82; +pub const YARVINSN_opt_eq: ruby_vminsn_type = 83; +pub const YARVINSN_opt_neq: ruby_vminsn_type = 84; +pub const YARVINSN_opt_lt: ruby_vminsn_type = 85; +pub const YARVINSN_opt_le: ruby_vminsn_type = 86; +pub const YARVINSN_opt_gt: ruby_vminsn_type = 87; +pub const YARVINSN_opt_ge: ruby_vminsn_type = 88; +pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 89; +pub const YARVINSN_opt_and: ruby_vminsn_type = 90; +pub const YARVINSN_opt_or: ruby_vminsn_type = 91; +pub const YARVINSN_opt_aref: ruby_vminsn_type = 92; +pub const YARVINSN_opt_aset: ruby_vminsn_type = 93; +pub const YARVINSN_opt_length: ruby_vminsn_type = 94; +pub const YARVINSN_opt_size: ruby_vminsn_type = 95; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 96; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 97; +pub const YARVINSN_opt_not: ruby_vminsn_type = 98; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 99; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 100; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 101; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 102; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 103; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 104; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 105; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 106; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 107; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 108; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 109; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 110; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 111; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 112; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 113; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 114; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 115; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 116; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 117; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 118; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 119; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 120; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 121; +pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 122; +pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 123; +pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 124; +pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 125; +pub const YARVINSN_trace_putnil: ruby_vminsn_type = 126; +pub const YARVINSN_trace_putself: ruby_vminsn_type = 127; +pub const YARVINSN_trace_putobject: ruby_vminsn_type = 128; +pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 129; +pub const YARVINSN_trace_putstring: ruby_vminsn_type = 130; +pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 131; +pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 132; +pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 133; +pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 134; +pub const YARVINSN_trace_intern: ruby_vminsn_type = 135; +pub const YARVINSN_trace_newarray: ruby_vminsn_type = 136; +pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 137; +pub const YARVINSN_trace_duparray: ruby_vminsn_type = 138; +pub const YARVINSN_trace_duphash: ruby_vminsn_type = 139; +pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 140; +pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 141; +pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 142; +pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 143; +pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 144; +pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 145; +pub const YARVINSN_trace_newhash: ruby_vminsn_type = 146; +pub const YARVINSN_trace_newrange: ruby_vminsn_type = 147; +pub const YARVINSN_trace_pop: ruby_vminsn_type = 148; +pub const YARVINSN_trace_dup: ruby_vminsn_type = 149; +pub const YARVINSN_trace_dupn: ruby_vminsn_type = 150; +pub const YARVINSN_trace_swap: ruby_vminsn_type = 151; +pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 152; +pub const YARVINSN_trace_topn: ruby_vminsn_type = 153; +pub const YARVINSN_trace_setn: ruby_vminsn_type = 154; +pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 155; +pub const YARVINSN_trace_defined: ruby_vminsn_type = 156; +pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 157; +pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 158; +pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 159; +pub const YARVINSN_trace_checktype: ruby_vminsn_type = 160; +pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 161; +pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 162; +pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 163; +pub const YARVINSN_trace_send: ruby_vminsn_type = 164; +pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 165; +pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 166; +pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 167; +pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 168; +pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 171; +pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 175; +pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 176; +pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 177; +pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 178; +pub const YARVINSN_trace_leave: ruby_vminsn_type = 179; +pub const YARVINSN_trace_throw: ruby_vminsn_type = 180; +pub const YARVINSN_trace_jump: ruby_vminsn_type = 181; +pub const YARVINSN_trace_branchif: ruby_vminsn_type = 182; +pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 183; +pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 184; +pub const YARVINSN_trace_once: ruby_vminsn_type = 185; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 186; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 187; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 188; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 189; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 190; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 191; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 192; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 193; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 194; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 195; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 196; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 197; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 198; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 199; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 200; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 201; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 202; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 203; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 204; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 205; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 206; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 207; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 208; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 209; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 210; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 211; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 212; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 213; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 214; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 215; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 216; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 217; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 218; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 219; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 220; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 221; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 223; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 224; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 248; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 249; +pub type ruby_vminsn_type = u32; +pub type rb_iseq_callback = ::std::option::Option< + unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), +>; +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data { + pub _bindgen_opaque_blob: [u64; 24usize], +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union iseq_compile_data__bindgen_ty_1 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_2 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_3 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_insn_info_entry { + pub line_no: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub events: rb_event_flag_t, +} +pub const CATCH_TYPE_RESCUE: rb_catch_type = 3; +pub const CATCH_TYPE_ENSURE: rb_catch_type = 5; +pub const CATCH_TYPE_RETRY: rb_catch_type = 7; +pub const CATCH_TYPE_BREAK: rb_catch_type = 9; +pub const CATCH_TYPE_REDO: rb_catch_type = 11; +pub const CATCH_TYPE_NEXT: rb_catch_type = 13; +pub type rb_catch_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_catch_table_entry { + pub type_: rb_catch_type, + pub iseq: *mut rb_iseq_t, + pub start: ::std::os::raw::c_uint, + pub end: ::std::os::raw::c_uint, + pub cont: ::std::os::raw::c_uint, + pub sp: ::std::os::raw::c_uint, +} +#[repr(C, packed)] +pub struct iseq_catch_table { + pub size: ::std::os::raw::c_uint, + pub entries: __IncompleteArrayField<iseq_catch_table_entry>, +} +#[repr(C)] +#[derive(Debug)] +pub struct iseq_compile_data_storage { + pub next: *mut iseq_compile_data_storage, + pub pos: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub buff: __IncompleteArrayField<::std::os::raw::c_char>, +} +pub const DEFINED_NOT_DEFINED: defined_type = 0; +pub const DEFINED_NIL: defined_type = 1; +pub const DEFINED_IVAR: defined_type = 2; +pub const DEFINED_LVAR: defined_type = 3; +pub const DEFINED_GVAR: defined_type = 4; +pub const DEFINED_CVAR: defined_type = 5; +pub const DEFINED_CONST: defined_type = 6; +pub const DEFINED_METHOD: defined_type = 7; +pub const DEFINED_YIELD: defined_type = 8; +pub const DEFINED_ZSUPER: defined_type = 9; +pub const DEFINED_SELF: defined_type = 10; +pub const DEFINED_TRUE: defined_type = 11; +pub const DEFINED_FALSE: defined_type = 12; +pub const DEFINED_ASGN: defined_type = 13; +pub const DEFINED_EXPR: defined_type = 14; +pub const DEFINED_REF: defined_type = 15; +pub const DEFINED_FUNC: defined_type = 16; +pub const DEFINED_CONST_FROM: defined_type = 17; +pub type defined_type = u32; +pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; +pub type zjit_struct_offsets = u32; +pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; +pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36; +pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48; +pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64; +pub type jit_bindgen_constants = u32; +pub const rb_invalid_shape_id: shape_id_t = 4294967295; +pub type rb_iseq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct succ_index_table { + pub _address: u8, +} +unsafe extern "C" { + pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void); + pub fn rb_class_attached_object(klass: VALUE) -> VALUE; + pub fn rb_singleton_class(obj: VALUE) -> VALUE; + pub fn rb_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_method_basic_definition_p(klass: VALUE, mid: ID) -> ::std::os::raw::c_int; + pub fn rb_bug(fmt: *const ::std::os::raw::c_char, ...) -> !; + pub fn rb_float_new(d: f64) -> VALUE; + pub fn rb_gc_mark(obj: VALUE); + pub fn rb_gc_mark_movable(obj: VALUE); + pub fn rb_gc_location(obj: VALUE) -> VALUE; + pub fn rb_gc_enable() -> VALUE; + pub fn rb_gc_disable() -> VALUE; + pub fn rb_gc_writebarrier(old: VALUE, young: VALUE); + pub fn rb_class_get_superclass(klass: VALUE) -> VALUE; + pub fn rb_funcallv( + recv: VALUE, + mid: ID, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + ) -> VALUE; + pub static mut rb_mKernel: VALUE; + pub static mut rb_cBasicObject: VALUE; + pub static mut rb_cObject: VALUE; + pub static mut rb_cArray: VALUE; + pub static mut rb_cClass: VALUE; + pub static mut rb_cFalseClass: VALUE; + pub static mut rb_cFloat: VALUE; + pub static mut rb_cHash: VALUE; + pub static mut rb_cIO: VALUE; + pub static mut rb_cInteger: VALUE; + pub static mut rb_cModule: VALUE; + pub static mut rb_cNilClass: VALUE; + pub static mut rb_cNumeric: VALUE; + pub static mut rb_cRange: VALUE; + pub static mut rb_cRegexp: VALUE; + pub static mut rb_cSet: VALUE; + pub static mut rb_cString: VALUE; + pub static mut rb_cSymbol: VALUE; + pub static mut rb_cThread: VALUE; + pub static mut rb_cTrueClass: VALUE; + pub fn ruby_init(); + pub fn ruby_options( + argc: ::std::os::raw::c_int, + argv: *mut *mut ::std::os::raw::c_char, + ) -> *mut ::std::os::raw::c_void; + pub fn ruby_executable_node( + n: *mut ::std::os::raw::c_void, + status: *mut ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; + pub fn ruby_init_stack(addr: *mut ::std::os::raw::c_void); + pub fn rb_define_class(name: *const ::std::os::raw::c_char, super_: VALUE) -> VALUE; + pub fn rb_obj_class(obj: VALUE) -> VALUE; + pub fn rb_ary_new_capa(capa: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_store(ary: VALUE, key: ::std::os::raw::c_long, val: VALUE); + pub fn rb_ary_dup(ary: VALUE) -> VALUE; + pub fn rb_ary_resurrect(ary: VALUE) -> VALUE; + pub fn rb_ary_cat(ary: VALUE, train: *const VALUE, len: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_push(ary: VALUE, elem: VALUE) -> VALUE; + pub fn rb_ary_pop(ary: VALUE) -> VALUE; + pub fn rb_ary_entry(ary: VALUE, off: ::std::os::raw::c_long) -> VALUE; + pub fn rb_ary_clear(ary: VALUE) -> VALUE; + pub fn rb_ary_concat(lhs: VALUE, rhs: VALUE) -> VALUE; + pub fn rb_hash_new() -> VALUE; + pub fn rb_hash_aref(hash: VALUE, key: VALUE) -> VALUE; + pub fn rb_hash_aset(hash: VALUE, key: VALUE, val: VALUE) -> VALUE; + pub fn rb_hash_bulk_insert(argc: ::std::os::raw::c_long, argv: *const VALUE, hash: VALUE); + pub fn rb_obj_is_proc(recv: VALUE) -> VALUE; + pub fn rb_protect( + func: ::std::option::Option<unsafe extern "C" fn(args: VALUE) -> VALUE>, + args: VALUE, + state: *mut ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_sym2id(obj: VALUE) -> ID; + pub fn rb_id2sym(id: ID) -> VALUE; + pub fn rb_intern(name: *const ::std::os::raw::c_char) -> ID; + pub fn rb_intern2(name: *const ::std::os::raw::c_char, len: ::std::os::raw::c_long) -> ID; + pub fn rb_id2str(id: ID) -> VALUE; + pub fn rb_sym2str(symbol: VALUE) -> VALUE; + pub fn rb_class2name(klass: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_class_new_instance_pass_kw( + argc: ::std::os::raw::c_int, + argv: *const VALUE, + klass: VALUE, + ) -> VALUE; + pub fn rb_obj_is_kind_of(obj: VALUE, klass: VALUE) -> VALUE; + pub fn rb_obj_alloc(klass: VALUE) -> VALUE; + pub fn rb_obj_frozen_p(obj: VALUE) -> VALUE; + pub fn rb_class_real(klass: VALUE) -> VALUE; + pub fn rb_class_inherited_p(scion: VALUE, ascendant: VALUE) -> VALUE; + pub fn rb_backref_get() -> VALUE; + pub fn rb_range_new(beg: VALUE, end: VALUE, excl: ::std::os::raw::c_int) -> VALUE; + pub fn rb_reg_nth_match(n: ::std::os::raw::c_int, md: VALUE) -> VALUE; + pub fn rb_reg_last_match(md: VALUE) -> VALUE; + pub fn rb_reg_match_pre(md: VALUE) -> VALUE; + pub fn rb_reg_match_post(md: VALUE) -> VALUE; + pub fn rb_reg_match_last(md: VALUE) -> VALUE; + pub fn rb_utf8_str_new( + ptr: *const ::std::os::raw::c_char, + len: ::std::os::raw::c_long, + ) -> VALUE; + pub fn rb_str_buf_append(dst: VALUE, src: VALUE) -> VALUE; + pub fn rb_str_dup(str_: VALUE) -> VALUE; + pub fn rb_str_intern(str_: VALUE) -> VALUE; + pub fn rb_mod_name(mod_: VALUE) -> VALUE; + pub fn rb_ivar_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_ivar_set(obj: VALUE, name: ID, val: VALUE) -> VALUE; + pub fn rb_ivar_defined(obj: VALUE, name: ID) -> VALUE; + pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; + pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; + pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE; + pub fn rb_ary_tmp_new_from_values( + arg1: VALUE, + arg2: ::std::os::raw::c_long, + arg3: *const VALUE, + ) -> VALUE; + pub fn rb_ec_ary_new_from_values( + ec: *mut rb_execution_context_struct, + n: ::std::os::raw::c_long, + elts: *const VALUE, + ) -> VALUE; + pub fn rb_vm_top_self() -> VALUE; + pub static mut rb_vm_insn_count: u64; + pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t; + pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t; + pub fn rb_callable_method_entry_or_negative( + klass: VALUE, + id: ID, + ) -> *const rb_callable_method_entry_t; + pub static mut rb_cISeq: VALUE; + pub static mut rb_mRubyVMFrozenCore: VALUE; + pub static mut rb_block_param_proxy: VALUE; + pub fn rb_vm_ep_local_ep(ep: *const VALUE) -> *const VALUE; + pub fn rb_iseq_path(iseq: *const rb_iseq_t) -> VALUE; + pub fn rb_vm_env_write(ep: *const VALUE, index: ::std::os::raw::c_int, v: VALUE); + pub fn rb_vm_bh_to_procval(ec: *const rb_execution_context_t, block_handler: VALUE) -> VALUE; + pub fn rb_vm_frame_method_entry( + cfp: *const rb_control_frame_t, + ) -> *const rb_callable_method_entry_t; + pub fn rb_obj_info(obj: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_raw_obj_info( + buff: *mut ::std::os::raw::c_char, + buff_size: usize, + obj: VALUE, + ) -> *const ::std::os::raw::c_char; + pub fn rb_ec_stack_check(ec: *mut rb_execution_context_struct) -> ::std::os::raw::c_int; + pub fn rb_gc_writebarrier_remember(obj: VALUE); + pub fn rb_shape_id_offset() -> i32; + pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t; + pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool; + pub fn rb_shape_transition_add_ivar_no_warnings( + klass: VALUE, + original_shape_id: shape_id_t, + id: ID, + ) -> shape_id_t; + pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE; + pub fn rb_gvar_get(arg1: ID) -> VALUE; + pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; + pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32); + pub fn rb_vm_barrier(); + pub fn rb_str_byte_substr(str_: VALUE, beg: VALUE, len: VALUE) -> VALUE; + pub fn rb_str_substr_two_fixnums( + str_: VALUE, + beg: VALUE, + len: VALUE, + empty: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_obj_as_string_result(str_: VALUE, obj: VALUE) -> VALUE; + pub fn rb_str_concat_literals(num: usize, strary: *const VALUE) -> VALUE; + pub fn rb_ec_str_resurrect( + ec: *mut rb_execution_context_struct, + str_: VALUE, + chilled: bool, + ) -> VALUE; + pub fn rb_to_hash_type(obj: VALUE) -> VALUE; + pub fn rb_hash_stlike_foreach( + hash: VALUE, + func: st_foreach_callback_func, + arg: st_data_t, + ) -> ::std::os::raw::c_int; + pub fn rb_hash_new_with_size(size: st_index_t) -> VALUE; + pub fn rb_hash_resurrect(hash: VALUE) -> VALUE; + pub fn rb_hash_stlike_lookup( + hash: VALUE, + key: st_data_t, + pval: *mut st_data_t, + ) -> ::std::os::raw::c_int; + pub fn rb_insn_len(insn: VALUE) -> ::std::os::raw::c_int; + pub fn rb_vm_insn_decode(encoded: VALUE) -> ::std::os::raw::c_int; + pub fn rb_float_plus(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_minus(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_mul(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_float_div(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_fix_aref(fix: VALUE, idx: VALUE) -> VALUE; + pub fn rb_vm_insn_addr2opcode(addr: *const ::std::os::raw::c_void) -> ::std::os::raw::c_int; + pub fn rb_iseq_line_no(iseq: *const rb_iseq_t, pos: usize) -> ::std::os::raw::c_uint; + pub fn rb_iseqw_to_iseq(iseqw: VALUE) -> *const rb_iseq_t; + pub fn rb_iseq_label(iseq: *const rb_iseq_t) -> VALUE; + pub fn rb_iseq_defined_string(type_: defined_type) -> VALUE; + pub fn rb_profile_frames( + start: ::std::os::raw::c_int, + limit: ::std::os::raw::c_int, + buff: *mut VALUE, + lines: *mut ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; + pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_zjit_exit_locations_dict( + zjit_raw_samples: *mut VALUE, + zjit_line_samples: *mut ::std::os::raw::c_int, + samples_len: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); + pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; + pub fn rb_zjit_iseq_insn_set( + iseq: *const rb_iseq_t, + insn_idx: ::std::os::raw::c_uint, + bare_insn: ruby_vminsn_type, + ); + pub fn rb_iseq_get_zjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; + pub fn rb_iseq_set_zjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); + pub fn rb_zjit_print_exception(); + pub fn rb_zjit_singleton_class_p(klass: VALUE) -> bool; + pub fn rb_zjit_defined_ivar(obj: VALUE, id: ID, pushval: VALUE) -> VALUE; + pub fn rb_zjit_method_tracing_currently_enabled() -> bool; + pub fn rb_zjit_insn_leaf(insn: ::std::os::raw::c_int, opes: *const VALUE) -> bool; + pub fn rb_zjit_local_id(iseq: *const rb_iseq_t, idx: ::std::os::raw::c_uint) -> ID; + pub fn rb_zjit_cme_is_cfunc( + me: *const rb_callable_method_entry_t, + func: *const ::std::os::raw::c_void, + ) -> bool; + pub fn rb_zjit_vm_search_method( + cd_owner: VALUE, + cd: *mut rb_call_data, + recv: VALUE, + ) -> *const rb_callable_method_entry_struct; + pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; + pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; + pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; + pub fn rb_zjit_writebarrier_check_immediate(recv: VALUE, val: VALUE); + pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; + pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; + pub fn rb_RSTRING_LEN(str_: VALUE) -> ::std::os::raw::c_ulong; + pub fn rb_RSTRING_PTR(str_: VALUE) -> *mut ::std::os::raw::c_char; + pub fn rb_insn_name(insn: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_vm_ci_argc(ci: *const rb_callinfo) -> ::std::os::raw::c_uint; + pub fn rb_vm_ci_mid(ci: *const rb_callinfo) -> ID; + pub fn rb_vm_ci_flag(ci: *const rb_callinfo) -> ::std::os::raw::c_uint; + pub fn rb_vm_ci_kwarg(ci: *const rb_callinfo) -> *const rb_callinfo_kwarg; + pub fn rb_get_cikw_keyword_len(cikw: *const rb_callinfo_kwarg) -> ::std::os::raw::c_int; + pub fn rb_get_cikw_keywords_idx( + cikw: *const rb_callinfo_kwarg, + idx: ::std::os::raw::c_int, + ) -> VALUE; + pub fn rb_METHOD_ENTRY_VISI(me: *const rb_callable_method_entry_t) -> rb_method_visibility_t; + pub fn rb_get_cme_def_type(cme: *const rb_callable_method_entry_t) -> rb_method_type_t; + pub fn rb_get_cme_def_body_attr_id(cme: *const rb_callable_method_entry_t) -> ID; + pub fn rb_get_cme_def_body_optimized_type( + cme: *const rb_callable_method_entry_t, + ) -> method_optimized_type; + pub fn rb_get_cme_def_body_optimized_index( + cme: *const rb_callable_method_entry_t, + ) -> ::std::os::raw::c_uint; + pub fn rb_get_cme_def_body_cfunc( + cme: *const rb_callable_method_entry_t, + ) -> *mut rb_method_cfunc_t; + pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize; + pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID; + pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; + pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; + pub fn rb_optimized_call( + recv: *mut VALUE, + ec: *mut rb_execution_context_t, + argc: ::std::os::raw::c_int, + argv: *mut VALUE, + kw_splat: ::std::os::raw::c_int, + block_handler: VALUE, + ) -> VALUE; + pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int; + pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; + pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_local_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_parent_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_local_table_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_iseq_encoded(iseq: *const rb_iseq_t) -> *mut VALUE; + pub fn rb_get_iseq_body_stack_max(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_type(iseq: *const rb_iseq_t) -> rb_iseq_type; + pub fn rb_get_iseq_flags_has_lead(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_opt(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_kw(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_post(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_kwrest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_anon_kwrest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_rest(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_ruby2_keywords(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_has_block(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_ambiguous_param0(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_accepts_no_kwarg(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_flags_forwardable(iseq: *const rb_iseq_t) -> bool; + pub fn rb_get_iseq_body_param_keyword( + iseq: *const rb_iseq_t, + ) -> *const rb_iseq_param_keyword_struct; + pub fn rb_get_iseq_body_param_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_get_iseq_body_param_lead_num(iseq: *const rb_iseq_t) -> ::std::os::raw::c_int; + pub fn rb_get_iseq_body_param_opt_num(iseq: *const rb_iseq_t) -> ::std::os::raw::c_int; + pub fn rb_get_iseq_body_param_opt_table(iseq: *const rb_iseq_t) -> *const VALUE; + pub fn rb_get_ec_cfp(ec: *const rb_execution_context_t) -> *mut rb_control_frame_struct; + pub fn rb_get_cfp_iseq(cfp: *mut rb_control_frame_struct) -> *const rb_iseq_t; + pub fn rb_get_cfp_pc(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_sp(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_self(cfp: *mut rb_control_frame_struct) -> VALUE; + pub fn rb_get_cfp_ep(cfp: *mut rb_control_frame_struct) -> *mut VALUE; + pub fn rb_get_cfp_ep_level(cfp: *mut rb_control_frame_struct, lv: u32) -> *const VALUE; + pub fn rb_yarv_class_of(obj: VALUE) -> VALUE; + pub fn rb_FL_TEST(obj: VALUE, flags: VALUE) -> VALUE; + pub fn rb_FL_TEST_RAW(obj: VALUE, flags: VALUE) -> VALUE; + pub fn rb_RB_TYPE_P(obj: VALUE, t: ruby_value_type) -> bool; + pub fn rb_get_call_data_ci(cd: *const rb_call_data) -> *const rb_callinfo; + pub fn rb_BASIC_OP_UNREDEFINED_P(bop: ruby_basic_operators, klass: u32) -> bool; + pub fn rb_RCLASS_ORIGIN(c: VALUE) -> VALUE; + pub fn rb_assert_iseq_handle(handle: VALUE); + pub fn rb_assert_holding_vm_lock(); + pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int; + pub fn rb_assert_cme_handle(handle: VALUE); + pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; + pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); + pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); + pub fn rb_jit_shape_too_complex_p(shape_id: shape_id_t) -> bool; + pub fn rb_jit_multi_ractor_p() -> bool; + pub fn rb_jit_vm_lock_then_barrier( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_jit_vm_unlock( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); + pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; + pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE); + pub fn rb_jit_shape_capacity(shape_id: shape_id_t) -> attr_index_t; +} diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs new file mode 100644 index 0000000000..8121b0065f --- /dev/null +++ b/zjit/src/cruby_methods.rs @@ -0,0 +1,891 @@ +/*! This module contains assertions we make about runtime properties of core library methods. + * Some properties that influence codegen: + * - Whether the method has been redefined since boot + * - Whether the C method can yield to the GC + * - Whether the C method makes any method calls + * + * For Ruby methods, many of these properties can be inferred through analyzing the + * bytecode, but for C methods we resort to annotation and validation in debug builds. + */ + +use crate::cruby::*; +use std::collections::HashMap; +use std::ffi::c_void; +use crate::hir_type::{types, Type}; +use crate::hir; + +pub struct Annotations { + cfuncs: HashMap<*mut c_void, FnProperties>, + builtin_funcs: HashMap<*mut c_void, FnProperties>, +} + +/// Runtime behaviors of C functions that implement a Ruby method +#[derive(Clone, Copy)] +pub struct FnProperties { + /// Whether it's possible for the function to yield to the GC + pub no_gc: bool, + /// Whether it's possible for the function to make a ruby call + pub leaf: bool, + /// What Type the C function returns + pub return_type: Type, + /// Whether it's legal to remove the call if the result is unused + pub elidable: bool, + pub inline: fn(&mut hir::Function, hir::BlockId, hir::InsnId, &[hir::InsnId], hir::InsnId) -> Option<hir::InsnId>, +} + +/// A safe default for un-annotated Ruby methods: we can't optimize them or their returned values. +impl Default for FnProperties { + fn default() -> Self { + Self { + no_gc: false, + leaf: false, + return_type: types::BasicObject, + elidable: false, + inline: no_inline, + } + } +} + +impl Annotations { + /// Query about properties of a C method + pub fn get_cfunc_properties(&self, method: *const rb_callable_method_entry_t) -> Option<FnProperties> { + let fn_ptr = unsafe { + if VM_METHOD_TYPE_CFUNC != get_cme_def_type(method) { + return None; + } + get_mct_func(get_cme_def_body_cfunc(method.cast())) + }; + self.cfuncs.get(&fn_ptr).copied() + } + + /// Query about properties of a builtin function by its pointer + pub fn get_builtin_properties(&self, bf: *const rb_builtin_function) -> Option<FnProperties> { + let func_ptr = unsafe { (*bf).func_ptr as *mut c_void }; + self.builtin_funcs.get(&func_ptr).copied() + } +} + +fn annotate_c_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { + // Lookup function pointer of the C method + let fn_ptr = unsafe { + // TODO(alan): (side quest) make rust methods and clean up glue code for rb_method_cfunc_t and + // rb_method_definition_t. + let method_id = rb_intern2(method_name.as_ptr().cast(), method_name.len() as _); + let method = rb_method_entry_at(class, method_id); + assert!(!method.is_null()); + // ME-to-CME cast is fine due to identical layout + debug_assert_eq!(VM_METHOD_TYPE_CFUNC, get_cme_def_type(method.cast())); + get_mct_func(get_cme_def_body_cfunc(method.cast())) + }; + + props_map.insert(fn_ptr, props); +} + +/// Look up a method and find its builtin function pointer by parsing its ISEQ +/// We currently only support methods with exactly one invokebuiltin instruction +fn annotate_builtin_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { + unsafe { + let method_id = rb_intern2(method_name.as_ptr().cast(), method_name.len().try_into().unwrap()); + let method = rb_method_entry_at(class, method_id); + if method.is_null() { + panic!("Method {}#{} not found", std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), method_name); + } + + // Cast ME to CME - they have identical layout + let cme = method.cast::<rb_callable_method_entry_t>(); + let def_type = get_cme_def_type(cme); + + if def_type != VM_METHOD_TYPE_ISEQ { + panic!("Method {}#{} is not an ISEQ method (type: {})", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name, def_type); + } + + // Get the ISEQ from the method definition + let iseq = get_def_iseq_ptr((*cme).def); + if iseq.is_null() { + panic!("Failed to get ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + + // Get the size of the ISEQ in instruction units + let encoded_size = rb_iseq_encoded_size(iseq); + + // Scan through the ISEQ to find invokebuiltin instructions + let mut insn_idx: u32 = 0; + let mut func_ptr = std::ptr::null_mut::<c_void>(); + + while insn_idx < encoded_size { + // Get the PC for this instruction index + let pc = rb_iseq_pc_at_idx(iseq, insn_idx); + + // Get the opcode using the proper decoder + let opcode = rb_iseq_opcode_at_pc(iseq, pc); + + if opcode == YARVINSN_invokebuiltin as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate_leave as i32 { + // The first operand is the builtin function pointer + let bf_value = *pc.add(1); + let bf_ptr: *const rb_builtin_function = bf_value.as_ptr(); + + if func_ptr.is_null() { + func_ptr = (*bf_ptr).func_ptr as *mut c_void; + } else { + panic!("Multiple invokebuiltin instructions found in ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + } + + // Move to the next instruction using the proper length + insn_idx = insn_idx.saturating_add(rb_insn_len(VALUE(opcode as usize)).try_into().unwrap()); + } + + // Only insert the properties if its iseq has exactly one invokebuiltin instruction + props_map.insert(func_ptr, props); + } +} + +/// Gather annotations. Run this right after boot since the annotations +/// are about the stock versions of methods. +pub fn init() -> Annotations { + let cfuncs = &mut HashMap::new(); + let builtin_funcs = &mut HashMap::new(); + + macro_rules! annotate { + ($module:ident, $method_name:literal, $inline:ident) => { + let mut props = FnProperties::default(); + props.inline = $inline; + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + } + } + + macro_rules! annotate_builtin { + ($module:ident, $method_name:literal, $return_type:expr) => { + annotate_builtin!($module, $method_name, $return_type, no_gc, leaf, elidable) + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + } + } + + annotate!(rb_mKernel, "itself", inline_kernel_itself); + annotate!(rb_mKernel, "block_given?", inline_kernel_block_given_p); + annotate!(rb_mKernel, "===", inline_eqq); + annotate!(rb_mKernel, "is_a?", inline_kernel_is_a_p); + annotate!(rb_cString, "bytesize", inline_string_bytesize); + annotate!(rb_cString, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "length", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "getbyte", inline_string_getbyte); + annotate!(rb_cString, "setbyte", inline_string_setbyte); + annotate!(rb_cString, "empty?", inline_string_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cString, "<<", inline_string_append); + annotate!(rb_cString, "==", inline_string_eq); + // Not elidable; has a side effect of setting the encoding if ENC_CODERANGE_UNKNOWN. + // TOOD(max): Turn this into a load/compare. Will need to side-exit or do the full call if + // ENC_CODERANGE_UNKNOWN. + annotate!(rb_cString, "ascii_only?", types::BoolExact, no_gc, leaf); + annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable); + annotate!(rb_cModule, "===", inline_module_eqq, types::BoolExact, no_gc, leaf); + annotate!(rb_cArray, "length", inline_array_length, types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cArray, "empty?", inline_array_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cArray, "reverse", types::ArrayExact, leaf, elidable); + annotate!(rb_cArray, "join", types::StringExact); + annotate!(rb_cArray, "[]", inline_array_aref); + annotate!(rb_cArray, "[]=", inline_array_aset); + annotate!(rb_cArray, "<<", inline_array_push); + annotate!(rb_cArray, "push", inline_array_push); + annotate!(rb_cArray, "pop", inline_array_pop); + annotate!(rb_cHash, "[]", inline_hash_aref); + annotate!(rb_cHash, "[]=", inline_hash_aset); + annotate!(rb_cHash, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cNilClass, "nil?", inline_nilclass_nil_p); + annotate!(rb_mKernel, "nil?", inline_kernel_nil_p); + annotate!(rb_mKernel, "respond_to?", inline_kernel_respond_to_p); + annotate!(rb_cBasicObject, "==", inline_basic_object_eq, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!", inline_basic_object_not, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!=", inline_basic_object_neq, types::BoolExact); + annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); + annotate!(rb_cInteger, "succ", inline_integer_succ); + annotate!(rb_cInteger, "^", inline_integer_xor); + annotate!(rb_cInteger, "==", inline_integer_eq); + annotate!(rb_cInteger, "+", inline_integer_plus); + annotate!(rb_cInteger, "-", inline_integer_minus); + annotate!(rb_cInteger, "*", inline_integer_mult); + annotate!(rb_cInteger, "/", inline_integer_div); + annotate!(rb_cInteger, "%", inline_integer_mod); + annotate!(rb_cInteger, "&", inline_integer_and); + annotate!(rb_cInteger, "|", inline_integer_or); + annotate!(rb_cInteger, ">", inline_integer_gt); + annotate!(rb_cInteger, ">=", inline_integer_ge); + annotate!(rb_cInteger, "<", inline_integer_lt); + annotate!(rb_cInteger, "<=", inline_integer_le); + annotate!(rb_cInteger, "<<", inline_integer_lshift); + annotate!(rb_cInteger, ">>", inline_integer_rshift); + annotate!(rb_cInteger, "[]", inline_integer_aref); + annotate!(rb_cInteger, "to_s", types::StringExact); + annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact); + let thread_singleton = unsafe { rb_singleton_class(rb_cThread) }; + annotate!(thread_singleton, "current", inline_thread_current, types::BasicObject, no_gc, leaf); + + annotate_builtin!(rb_mKernel, "Float", types::Float); + annotate_builtin!(rb_mKernel, "Integer", types::Integer); + // TODO(max): Annotate rb_mKernel#class as returning types::Class. Right now there is a subtle + // type system bug that causes an issue if we make it return types::Class. + annotate_builtin!(rb_mKernel, "class", inline_kernel_class, types::HeapObject, leaf); + annotate_builtin!(rb_mKernel, "frozen?", types::BoolExact); + annotate_builtin!(rb_cSymbol, "name", types::StringExact); + annotate_builtin!(rb_cSymbol, "to_s", types::StringExact); + + Annotations { + cfuncs: std::mem::take(cfuncs), + builtin_funcs: std::mem::take(builtin_funcs), + } +} + +fn no_inline(_fun: &mut hir::Function, _block: hir::BlockId, _recv: hir::InsnId, _args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + None +} + +fn inline_string_to_s(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::StringExact, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + return Some(recv); + } + None +} + +fn inline_thread_current(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let ec = fun.push_insn(block, hir::Insn::LoadEC); + let thread_ptr = fun.push_insn(block, hir::Insn::LoadField { + recv: ec, + id: ID!(thread_ptr), + offset: RUBY_OFFSET_EC_THREAD_PTR as i32, + return_type: types::CPtr, + }); + let thread_self = fun.push_insn(block, hir::Insn::LoadField { + recv: thread_ptr, + id: ID!(self_), + offset: RUBY_OFFSET_THREAD_SELF as i32, + // TODO(max): Add Thread type. But Thread.current is not guaranteed to be an exact Thread. + // You can make subclasses... + return_type: types::BasicObject, + }); + Some(thread_self) +} + +fn inline_kernel_itself(_fun: &mut hir::Function, _block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() { + // No need to coerce the receiver; that is done by the SendWithoutBlock rewriting. + return Some(recv); + } + None +} + +fn inline_kernel_block_given_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(fun.iseq()) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + let lep = fun.push_insn(block, hir::Insn::GetLEP); + Some(fun.push_insn(block, hir::Insn::IsBlockGiven { lep })) + } else { + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) + } +} + +fn inline_array_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index] = args { + if fun.likely_a(index, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, state }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + return Some(result); + } + } + None +} + +fn inline_array_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index, val] = args { + if fun.likely_a(recv, types::ArrayExact, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::ArrayExact, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let recv = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + let recv = fun.push_insn(block, hir::Insn::GuardNotShared { recv, state }); + + // Bounds check: unbox Fixnum index and guard 0 <= idx < length. + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, state }); + + let _ = fun.push_insn(block, hir::Insn::ArrayAset { array: recv, index, val }); + fun.push_insn(block, hir::Insn::WriteBarrier { recv, val }); + return Some(val); + } + } + None +} + +fn inline_array_push(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Inline only the case of `<<` or `push` when called with a single argument. + if let &[val] = args { + let _ = fun.push_insn(block, hir::Insn::ArrayPush { array: recv, val, state }); + return Some(recv); + } + None +} + +fn inline_array_pop(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Only inline the case of no arguments. + let &[] = args else { return None; }; + // We know that all Array are HeapObject, so no need to insert a GuardType(HeapObject). + let arr = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + Some(fun.push_insn(block, hir::Insn::ArrayPop { array: arr, state })) +} + +fn inline_hash_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let result = fun.push_insn(block, hir::Insn::HashAref { hash: recv, key, state }); + Some(result) + } else { + None + } +} + +fn inline_hash_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key, val] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let _ = fun.push_insn(block, hir::Insn::HashAset { hash: recv, key, val, state }); + // Hash#[]= returns the value, not the hash + Some(val) + } else { + None + } +} + +fn inline_string_bytesize(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + + let result = fun.push_insn(block, hir::Insn::BoxFixnum { + val: len, + state, + }); + + return Some(result); + } + None +} + +fn inline_string_getbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) { + // String#getbyte with a Fixnum is leaf and nogc; otherwise it may run arbitrary Ruby code + // when converting the index to a C integer. + let index = fun.coerce_to(block, index, types::Fixnum, state); + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + // TODO(max): Find a way to mark these guards as not needed for correctness... as in, once + // the data dependency is gone (say, the StringGetbyte is elided), they can also be elided. + // + // This is unlike most other guards. + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, state }); + let result = fun.push_insn(block, hir::Insn::StringGetbyte { string: recv, index: unboxed_index }); + return Some(result); + } + None +} + +fn inline_string_setbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index, value] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) && fun.likely_a(value, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let value = fun.coerce_to(block, value, types::Fixnum, state); + + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, state }); + // We know that all String are HeapObject, so no need to insert a GuardType(HeapObject). + let recv = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + let _ = fun.push_insn(block, hir::Insn::StringSetbyteFixnum { string: recv, index, value }); + // String#setbyte returns the fixnum provided as its `value` argument back to the caller. + Some(value) + } else { + None + } +} + +fn inline_string_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let is_zero = fun.push_insn(block, hir::Insn::IsBitEqual { left: len, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: is_zero }); + Some(result) +} + +fn inline_string_append(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Inline only StringExact << String, which matches original type check from + // `vm_opt_ltlt`, which checks `RB_TYPE_P(obj, T_STRING)`. + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::String, state); + let _ = fun.push_insn(block, hir::Insn::StringAppend { recv, other, state }); + return Some(recv); + } + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::Fixnum, state); + let _ = fun.push_insn(block, hir::Insn::StringAppendCodepoint { recv, other, state }); + return Some(recv); + } + None +} + +fn inline_string_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.likely_a(recv, types::String, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let other = fun.coerce_to(block, other, types::String, state); + let return_type = types::BoolExact; + let elidable = true; + // TODO(max): Make StringEqual its own opcode so that we can later constant-fold StringEqual(a, a) => true + let result = fun.push_insn(block, hir::Insn::CCall { + cfunc: rb_yarv_str_eql_internal as *const u8, + recv, + args: vec![other], + name: ID!(string_eq), + return_type, + elidable, + }); + return Some(result); + } + None +} + +fn inline_module_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(recv, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: other, class: recv }); + return Some(result); + } + None +} + +fn inline_array_length(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let result = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + return Some(result); + } + None +} + +fn inline_array_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let result_c = fun.push_insn(block, hir::Insn::IsBitEqual { left: length_cint, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: result_c }); + return Some(result); + } + None +} + +fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + if fun.likely_a(recv, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left, right, state }); + return Some(result); + } + None +} + +fn inline_integer_xor(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[right] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumXor { left, right }); + return Some(result); + } + None +} + +fn try_inline_fixnum_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, left: hir::InsnId, right: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + return None; + } + if fun.likely_a(left, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + if bop == BOP_NEQ { + // For opt_neq, the interpreter checks that both neq and eq are unchanged. + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); + } + // Rely on the MethodRedefined PatchPoint for other bops. + let left = fun.coerce_to(block, left, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + return Some(fun.push_insn(block, f(left, right))); + } + None +} + +fn inline_integer_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumEq { left, right }, BOP_EQ, recv, other, state) +} + +fn inline_integer_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, other, state) +} + +fn inline_integer_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, other, state) +} + +fn inline_integer_mult(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMult { left, right, state }, BOP_MULT, recv, other, state) +} + +fn inline_integer_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, other, state) +} + +fn inline_integer_mod(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMod { left, right, state }, BOP_MOD, recv, other, state) +} + +fn inline_integer_and(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAnd { left, right, }, BOP_AND, recv, other, state) +} + +fn inline_integer_or(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumOr { left, right, }, BOP_OR, recv, other, state) +} + +fn inline_integer_gt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGt { left, right }, BOP_GT, recv, other, state) +} + +fn inline_integer_ge(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGe { left, right }, BOP_GE, recv, other, state) +} + +fn inline_integer_lt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLt { left, right }, BOP_LT, recv, other, state) +} + +fn inline_integer_le(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLe { left, right }, BOP_LE, recv, other, state) +} + +fn inline_integer_lshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLShift { left, right, state }, BOP_LTLT, recv, other, state) +} + +fn inline_integer_rshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + // TODO(max): If other_value > 63, rewrite to constant zero. + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumRShift { left, right }, BOP_GTGT, recv, other, state) +} + +fn inline_integer_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(index, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::Fixnum, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumAref { recv, index }); + return Some(result); + } + None +} + +fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_not(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.type_of(recv).is_known_truthy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) }); + return Some(result); + } + if fun.type_of(recv).is_known_falsy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + return Some(result); + } + None +} + +fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let result = try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumNeq { left, right }, BOP_NEQ, recv, other, state); + if result.is_some() { + return result; + } + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_initialize(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qnil) }); + Some(result) +} + +fn inline_nilclass_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) })) +} + +fn inline_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_kernel_is_a_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(other, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: recv, class: other }); + return Some(result); + } + None +} + +fn inline_kernel_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) +} + +fn inline_kernel_respond_to_p( + fun: &mut hir::Function, + block: hir::BlockId, + recv: hir::InsnId, + args: &[hir::InsnId], + state: hir::InsnId, +) -> Option<hir::InsnId> { + // Parse arguments: respond_to?(method_name, allow_priv = false) + let (method_name, allow_priv) = match *args { + [method_name] => (method_name, false), + [method_name, arg] => match fun.type_of(arg) { + t if t.is_known_truthy() => (method_name, true), + t if t.is_known_falsy() => (method_name, false), + // Unknown type; bail out + _ => return None, + }, + // Unknown args; bail out + _ => return None, + }; + + // Method name must be a static symbol + let method_name = fun.type_of(method_name).ruby_object()?; + if !method_name.static_sym_p() { + return None; + } + + // The receiver must have a known class to call `respond_to?` on + // TODO: This is technically overly strict. This would also work if all of the + // observed objects at this point agree on `respond_to?` and we can add many patchpoints. + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // Get the method ID and its corresponding callable method entry + let mid = unsafe { rb_sym2id(method_name) }; + let target_cme = unsafe { rb_callable_method_entry_or_negative(recv_class, mid) }; + assert!( + !target_cme.is_null(), + "Should never be null, as in that case we will be returned a \"negative CME\"" + ); + + let cme_def_type = unsafe { get_cme_def_type(target_cme) }; + + // Cannot inline a refined method, since their refinement depends on lexical scope + if cme_def_type == VM_METHOD_TYPE_REFINED { + return None; + } + + let visibility = match cme_def_type { + VM_METHOD_TYPE_UNDEF => METHOD_VISI_UNDEF, + _ => unsafe { METHOD_ENTRY_VISI(target_cme) }, + }; + + let result = match (visibility, allow_priv) { + // Method undefined; check `respond_to_missing?` + (METHOD_VISI_UNDEF, _) => { + let respond_to_missing = ID!(respond_to_missing); + if unsafe { rb_method_basic_definition_p(recv_class, respond_to_missing) } == 0 { + return None; // Custom definition of respond_to_missing?, so cannot inline + } + let respond_to_missing_cme = + unsafe { rb_callable_method_entry(recv_class, respond_to_missing) }; + // Protect against redefinition of `respond_to_missing?` + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::NoTracePoint, + state, + }, + ); + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: respond_to_missing, + cme: respond_to_missing_cme, + }, + state, + }, + ); + Qfalse + } + // Private method with allow priv=false, so `respond_to?` returns false + (METHOD_VISI_PRIVATE, false) => Qfalse, + // Public method or allow_priv=true: check if implemented + (METHOD_VISI_PUBLIC, _) | (_, true) => { + if cme_def_type == VM_METHOD_TYPE_NOTIMPLEMENTED { + // C method with rb_f_notimplement(). `respond_to?` returns false + // without consulting `respond_to_missing?`. See also: rb_add_method_cfunc() + Qfalse + } else { + Qtrue + } + } + (_, _) => return None, // not public and include_all not known, can't compile + }; + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + return None; + } + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::NoTracePoint, state }); + fun.push_insn(block, hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: mid, + cme: target_cme + }, state + }); + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(result) })) +} + +fn inline_kernel_class(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + let real_class = unsafe { rb_class_real(recv_class) }; + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(real_class) })) +} diff --git a/zjit/src/disasm.rs b/zjit/src/disasm.rs new file mode 100644 index 0000000000..d4f6591594 --- /dev/null +++ b/zjit/src/disasm.rs @@ -0,0 +1,52 @@ +use crate::asm::CodeBlock; + +pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> String { + use std::fmt::Write; + + let mut out = String::from(""); + + // Initialize capstone + use capstone::prelude::*; + + #[cfg(target_arch = "x86_64")] + let mut cs = Capstone::new() + .x86() + .mode(arch::x86::ArchMode::Mode64) + .syntax(arch::x86::ArchSyntax::Intel) + .build() + .unwrap(); + #[cfg(target_arch = "aarch64")] + let mut cs = Capstone::new() + .arm64() + .mode(arch::arm64::ArchMode::Arm) + .detail(true) + .build() + .unwrap(); + + cs.set_skipdata(true).unwrap(); + + // Disassemble the instructions + let code_size = end_addr - start_addr; + let code_slice = unsafe { std::slice::from_raw_parts(start_addr as _, code_size) }; + // Stabilize output for cargo test + #[cfg(test)] + let start_addr = 0; + let insns = cs.disasm_all(code_slice, start_addr as u64).unwrap(); + + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + + // For each instruction in this block + for insn in insns.as_ref() { + // Comments for this block + if let Some(comment_list) = cb.comments_at(insn.address() as usize) { + for comment in comment_list { + writeln!(&mut out, " {bold_begin}# {comment}{bold_end}").unwrap(); + } + } + writeln!(&mut out, " {}", format!("{insn}").trim()).unwrap(); + } + + out +} diff --git a/zjit/src/distribution.rs b/zjit/src/distribution.rs new file mode 100644 index 0000000000..2c6ffb3ae6 --- /dev/null +++ b/zjit/src/distribution.rs @@ -0,0 +1,276 @@ +//! Type frequency distribution tracker. + +/// This implementation was inspired by the type feedback module from Google's S6, which was +/// written in C++ for use with Python. This is a new implementation in Rust created for use with +/// Ruby instead of Python. +#[derive(Debug, Clone)] +pub struct Distribution<T: Copy + PartialEq + Default, const N: usize> { + /// buckets and counts have the same length + /// `buckets[0]` is always the most common item + buckets: [T; N], + counts: [usize; N], + /// if there is no more room, increment the fallback + other: usize, + // TODO(max): Add count disparity, which can help determine when to reset the distribution +} + +impl<T: Copy + PartialEq + Default, const N: usize> Distribution<T, N> { + pub fn new() -> Self { + Self { buckets: [Default::default(); N], counts: [0; N], other: 0 } + } + + pub fn observe(&mut self, item: T) { + for (bucket, count) in self.buckets.iter_mut().zip(self.counts.iter_mut()) { + if *bucket == item || *count == 0 { + *bucket = item; + *count += 1; + // Keep the most frequent item at the front + self.bubble_up(); + return; + } + } + self.other += 1; + } + + /// Keep the highest counted bucket at index 0 + fn bubble_up(&mut self) { + if N == 0 { return; } + let max_index = self.counts.into_iter().enumerate().max_by_key(|(_, val)| *val).unwrap().0; + if max_index != 0 { + self.counts.swap(0, max_index); + self.buckets.swap(0, max_index); + } + } + + pub fn each_item(&self) -> impl Iterator<Item = T> + '_ { + self.buckets.iter().zip(self.counts.iter()) + .filter_map(|(&bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } + + pub fn each_item_mut(&mut self) -> impl Iterator<Item = &mut T> + '_ { + self.buckets.iter_mut().zip(self.counts.iter()) + .filter_map(|(bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } +} + +#[derive(PartialEq, Debug, Clone, Copy)] +enum DistributionKind { + /// No types seen + Empty, + /// One type seen + Monomorphic, + /// Between 2 and (fixed) N types seen + Polymorphic, + /// Polymorphic, but with a significant skew towards one type + SkewedPolymorphic, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic, +} + +#[derive(Debug)] +pub struct DistributionSummary<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> { + kind: DistributionKind, + buckets: [T; N], + // TODO(max): Determine if we need some notion of stability +} + +const SKEW_THRESHOLD: f64 = 0.75; + +impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> DistributionSummary<T, N> { + pub fn new(dist: &Distribution<T, N>) -> Self { + #[cfg(debug_assertions)] + { + let first_count = dist.counts[0]; + for &count in &dist.counts[1..] { + assert!(first_count >= count, "First count should be the largest"); + } + } + let num_seen = dist.counts.iter().sum::<usize>() + dist.other; + let kind = if dist.other == 0 { + // Seen <= N types total + if dist.counts[0] == 0 { + DistributionKind::Empty + } else if dist.counts[1] == 0 { + DistributionKind::Monomorphic + } else if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedPolymorphic + } else { + DistributionKind::Polymorphic + } + } else { + // Seen > N types total; considered megamorphic + if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedMegamorphic + } else { + DistributionKind::Megamorphic + } + }; + Self { kind, buckets: dist.buckets } + } + + pub fn is_monomorphic(&self) -> bool { + self.kind == DistributionKind::Monomorphic + } + + pub fn is_polymorphic(&self) -> bool { + self.kind == DistributionKind::Polymorphic + } + + pub fn is_skewed_polymorphic(&self) -> bool { + self.kind == DistributionKind::SkewedPolymorphic + } + + pub fn is_megamorphic(&self) -> bool { + self.kind == DistributionKind::Megamorphic + } + + pub fn is_skewed_megamorphic(&self) -> bool { + self.kind == DistributionKind::SkewedMegamorphic + } + + pub fn bucket(&self, idx: usize) -> T { + assert!(idx < N, "index {idx} out of bounds for buckets[{N}]"); + self.buckets[idx] + } +} + +#[cfg(test)] +mod distribution_tests { + use super::*; + + #[test] + fn start_empty() { + let dist = Distribution::<usize, 4>::new(); + assert_eq!(dist.other, 0); + assert!(dist.counts.iter().all(|&b| b == 0)); + } + + #[test] + fn observe_adds_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 1); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_increments_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_two() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + assert_eq!(dist.buckets[0], 11); + assert_eq!(dist.counts[0], 3); + assert_eq!(dist.buckets[1], 10); + assert_eq!(dist.counts[1], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_with_max_increments_other() { + let mut dist = Distribution::<usize, 0>::new(); + dist.observe(10); + assert!(dist.buckets.is_empty()); + assert!(dist.counts.is_empty()); + assert_eq!(dist.other, 1); + } + + #[test] + fn empty_distribution_returns_empty_summary() { + let dist = Distribution::<usize, 4>::new(); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Empty); + } + + #[test] + fn monomorphic_distribution_returns_monomorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Monomorphic); + assert_eq!(summary.buckets[0], 10); + } + + #[test] + fn polymorphic_distribution_returns_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Polymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn skewed_polymorphic_distribution_returns_skewed_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedPolymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn megamorphic_distribution_returns_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(12); + dist.observe(13); + dist.observe(14); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Megamorphic); + assert_eq!(summary.buckets[0], 11); + } + + #[test] + fn skewed_megamorphic_distribution_returns_skewed_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(13); + dist.observe(14); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedMegamorphic); + assert_eq!(summary.buckets[0], 12); + } +} diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs new file mode 100644 index 0000000000..40230ccc8d --- /dev/null +++ b/zjit/src/gc.rs @@ -0,0 +1,211 @@ +//! This module is responsible for marking/moving objects on GC. + +use std::ptr::null; +use std::{ffi::c_void, ops::Range}; +use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; +use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; +use crate::stats::Counter::gc_time_ns; +use crate::state::gc_mark_raw_samples; + +/// GC callback for marking GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_mark(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to mark + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &*(payload as *const IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_mark(payload)); +} + +/// GC callback for updating GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to update + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &mut *(payload as *mut IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_update_references(payload)); +} + +/// GC callback for finalizing an ISEQ +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_free(iseq: IseqPtr) { + if !ZJITState::has_instance() { + return; + } + + // TODO(Shopify/ruby#682): Free `IseqPayload` + let payload = get_or_create_iseq_payload(iseq); + for version in payload.versions.iter_mut() { + unsafe { version.as_mut() }.iseq = null(); + } + + let invariants = ZJITState::get_invariants(); + invariants.forget_iseq(iseq); +} + +/// GC callback for finalizing a CME +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_free(cme: *const rb_callable_method_entry_struct) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_cme(cme); +} + +/// GC callback for finalizing a class +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_klass_free(klass: VALUE) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_klass(klass); +} + +/// GC callback for updating object references after all object moves +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_update_references() { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.update_references(); +} + +fn iseq_mark(payload: &IseqPayload) { + // Mark objects retained by profiling instructions + payload.profile.each_object(|object| { + unsafe { rb_gc_mark_movable(object); } + }); + + // Mark objects baked in JIT code + let cb = ZJITState::get_code_block(); + for version in payload.versions.iter() { + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + unsafe { + let object = value_ptr.read_unaligned(); + rb_gc_mark_movable(object); + } + } + } +} + +/// This is a mirror of [iseq_mark]. +fn iseq_update_references(payload: &mut IseqPayload) { + // Move objects retained by profiling instructions + payload.profile.each_object_mut(|old_object| { + let new_object = unsafe { rb_gc_location(*old_object) }; + if *old_object != new_object { + *old_object = new_object; + } + }); + + for &version in payload.versions.iter() { + iseq_version_update_references(version); + } +} + +fn iseq_version_update_references(mut version: IseqVersionRef) { + // Move ISEQ in the payload + unsafe { version.as_mut() }.iseq = unsafe { rb_gc_location(version.as_ref().iseq.into()) }.as_iseq(); + + // Move ISEQ references in incoming IseqCalls + for iseq_call in unsafe { version.as_mut() }.incoming.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move ISEQ references in outgoing IseqCalls + for iseq_call in unsafe { version.as_mut() }.outgoing.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move objects baked in JIT code + let cb = ZJITState::get_code_block(); + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + let object = unsafe { value_ptr.read_unaligned() }; + let new_addr = unsafe { rb_gc_location(object) }; + + // Only write when the VALUE moves, to be copy-on-write friendly. + if new_addr != object { + for (byte_idx, &byte) in new_addr.as_u64().to_le_bytes().iter().enumerate() { + let byte_code_ptr = offset.add_bytes(byte_idx); + cb.write_mem(byte_code_ptr, byte).expect("patching existing code should be within bounds"); + } + } + } + cb.mark_all_executable(); +} + +/// Append a set of gc_offsets to the iseq's payload +pub fn append_gc_offsets(iseq: IseqPtr, mut version: IseqVersionRef, offsets: &Vec<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.extend(offsets); + + // Call writebarrier on each newly added value + let cb = ZJITState::get_code_block(); + for &offset in offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + let value_ptr = value_ptr as *const VALUE; + unsafe { + let object = value_ptr.read_unaligned(); + VALUE::from(iseq).write_barrier(object); + } + } +} + +/// Remove GC offsets that overlap with a given removed_range. +/// We do this when invalidation rewrites some code with a jump instruction +/// and GC offsets are corrupted by the rewrite, assuming no on-stack code +/// will step into the instruction with the GC offsets after invalidation. +pub fn remove_gc_offsets(mut version: IseqVersionRef, removed_range: &Range<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.retain(|&gc_offset| { + let offset_range = gc_offset..(gc_offset.add_bytes(SIZEOF_VALUE)); + !ranges_overlap(&offset_range, removed_range) + }); +} + +/// Return true if given `Range<CodePtr>` ranges overlap with each other +fn ranges_overlap<T>(left: &Range<T>, right: &Range<T>) -> bool where T: PartialOrd { + left.start < right.end && right.start < left.end +} + +/// Callback for marking GC objects inside [crate::invariants::Invariants]. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_mark() { + gc_mark_raw_samples(); +} diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs new file mode 100644 index 0000000000..4326d37b34 --- /dev/null +++ b/zjit/src/hir.rs @@ -0,0 +1,7761 @@ +//! High-level intermediary representation (IR) in static single-assignment (SSA) form. + +// We use the YARV bytecode constants which have a CRuby-style name +#![allow(non_upper_case_globals)] + +#![allow(clippy::if_same_then_else)] +#![allow(clippy::match_like_matches_macro)] +use crate::{ + backend::lir::C_ARG_OPNDS, + cast::IntoUsize, codegen::local_idx_to_ep_offset, cruby::*, invariants::has_singleton_class_of, payload::{get_or_create_iseq_payload, IseqPayload}, options::{debug, get_option, DumpHIR}, state::ZJITState, json::Json +}; +use std::{ + cell::RefCell, collections::{BTreeSet, HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, c_int, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter +}; +use crate::hir_type::{Type, types}; +use crate::hir_effect::{Effect, abstract_heaps, effects}; +use crate::bitset::BitSet; +use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::stats::Counter; +use SendFallbackReason::*; + +mod tests; +mod opt_tests; + +/// An index of an [`Insn`] in a [`Function`]. This is a popular +/// type since this effectively acts as a pointer to an [`Insn`]. +/// See also: [`Function::find`]. +#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Debug)] +pub struct InsnId(pub usize); + +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 + } +} + +impl std::fmt::Display for InsnId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "v{}", self.0) + } +} + +/// The index of a [`Block`], which effectively acts like a pointer. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "bb{}", self.0) + } +} + +type InsnSet = BitSet<InsnId>; +type BlockSet = BitSet<BlockId>; + +fn write_vec<T: std::fmt::Display>(f: &mut std::fmt::Formatter, objs: &Vec<T>) -> std::fmt::Result { + write!(f, "[")?; + let mut prefix = ""; + for obj in objs { + write!(f, "{prefix}{obj}")?; + prefix = ", "; + } + write!(f, "]") +} + +impl std::fmt::Display for VALUE { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl VALUE { + pub fn print(self, ptr_map: &PtrPrintMap) -> VALUEPrinter<'_> { + VALUEPrinter { inner: self, ptr_map } + } +} + +/// Print adaptor for [`VALUE`]. See [`PtrPrintMap`]. +pub struct VALUEPrinter<'a> { + inner: VALUE, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for VALUEPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + val if val.fixnum_p() => write!(f, "{}", val.as_fixnum()), + Qnil => write!(f, "nil"), + Qtrue => write!(f, "true"), + Qfalse => write!(f, "false"), + val => write!(f, "VALUE({:p})", self.ptr_map.map_ptr(val.as_ptr::<VALUE>())), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<InsnId>, +} + +impl std::fmt::Display for BranchEdge { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}(", self.target)?; + let mut prefix = ""; + for arg in &self.args { + write!(f, "{prefix}{arg}")?; + prefix = ", "; + } + write!(f, ")") + } +} + +/// Invalidation reasons +#[derive(Debug, Clone, Copy)] +pub enum Invariant { + /// Basic operation is redefined + BOPRedefined { + /// {klass}_REDEFINED_OP_FLAG + klass: RedefinitionFlag, + /// BOP_{bop} + bop: ruby_basic_operators, + }, + MethodRedefined { + /// The class object whose method we want to assume unchanged + klass: VALUE, + /// The method ID of the method we want to assume unchanged + method: ID, + /// The callable method entry that we want to track + cme: *const rb_callable_method_entry_t, + }, + /// A list of constant expression path segments that must have not been written to for the + /// following code to be valid. + StableConstantNames { + idlist: *const ID, + }, + /// TracePoint is not enabled. If TracePoint is enabled, this is invalidated. + NoTracePoint, + /// cfp->ep is not escaped to the heap on the ISEQ + NoEPEscape(IseqPtr), + /// There is one ractor running. If a non-root ractor gets spawned, this is invalidated. + SingleRactorMode, + /// Objects of this class have no singleton class. + /// When a singleton class is created for an object of this class, this is invalidated. + NoSingletonClass { + klass: VALUE, + }, +} + +impl Invariant { + pub fn print(self, ptr_map: &PtrPrintMap) -> InvariantPrinter<'_> { + InvariantPrinter { inner: self, ptr_map } + } +} + +impl Display for Invariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialObjectType { + VMCore = 1, + CBase = 2, + ConstBase = 3, +} + +impl From<u32> for SpecialObjectType { + fn from(value: u32) -> Self { + match value { + VM_SPECIAL_OBJECT_VMCORE => SpecialObjectType::VMCore, + VM_SPECIAL_OBJECT_CBASE => SpecialObjectType::CBase, + VM_SPECIAL_OBJECT_CONST_BASE => SpecialObjectType::ConstBase, + _ => panic!("Invalid special object type: {value}"), + } + } +} + +impl From<SpecialObjectType> for u64 { + fn from(special_type: SpecialObjectType) -> Self { + special_type as u64 + } +} + +impl std::fmt::Display for SpecialObjectType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SpecialObjectType::VMCore => write!(f, "VMCore"), + SpecialObjectType::CBase => write!(f, "CBase"), + SpecialObjectType::ConstBase => write!(f, "ConstBase"), + } + } +} + +/// Print adaptor for [`Invariant`]. See [`PtrPrintMap`]. +pub struct InvariantPrinter<'a> { + inner: Invariant, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for InvariantPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + Invariant::BOPRedefined { klass, bop } => { + write!(f, "BOPRedefined(")?; + match klass { + INTEGER_REDEFINED_OP_FLAG => write!(f, "INTEGER_REDEFINED_OP_FLAG")?, + STRING_REDEFINED_OP_FLAG => write!(f, "STRING_REDEFINED_OP_FLAG")?, + ARRAY_REDEFINED_OP_FLAG => write!(f, "ARRAY_REDEFINED_OP_FLAG")?, + HASH_REDEFINED_OP_FLAG => write!(f, "HASH_REDEFINED_OP_FLAG")?, + _ => write!(f, "{klass}")?, + } + write!(f, ", ")?; + match bop { + BOP_PLUS => write!(f, "BOP_PLUS")?, + BOP_MINUS => write!(f, "BOP_MINUS")?, + BOP_MULT => write!(f, "BOP_MULT")?, + BOP_DIV => write!(f, "BOP_DIV")?, + BOP_MOD => write!(f, "BOP_MOD")?, + BOP_EQ => write!(f, "BOP_EQ")?, + BOP_EQQ => write!(f, "BOP_EQQ")?, + BOP_LT => write!(f, "BOP_LT")?, + BOP_LE => write!(f, "BOP_LE")?, + BOP_LTLT => write!(f, "BOP_LTLT")?, + BOP_AREF => write!(f, "BOP_AREF")?, + BOP_ASET => write!(f, "BOP_ASET")?, + BOP_LENGTH => write!(f, "BOP_LENGTH")?, + BOP_SIZE => write!(f, "BOP_SIZE")?, + BOP_EMPTY_P => write!(f, "BOP_EMPTY_P")?, + BOP_NIL_P => write!(f, "BOP_NIL_P")?, + BOP_SUCC => write!(f, "BOP_SUCC")?, + BOP_GT => write!(f, "BOP_GT")?, + BOP_GE => write!(f, "BOP_GE")?, + BOP_NOT => write!(f, "BOP_NOT")?, + BOP_NEQ => write!(f, "BOP_NEQ")?, + BOP_MATCH => write!(f, "BOP_MATCH")?, + BOP_FREEZE => write!(f, "BOP_FREEZE")?, + BOP_UMINUS => write!(f, "BOP_UMINUS")?, + BOP_MAX => write!(f, "BOP_MAX")?, + BOP_MIN => write!(f, "BOP_MIN")?, + BOP_HASH => write!(f, "BOP_HASH")?, + BOP_CALL => write!(f, "BOP_CALL")?, + BOP_AND => write!(f, "BOP_AND")?, + BOP_OR => write!(f, "BOP_OR")?, + BOP_CMP => write!(f, "BOP_CMP")?, + BOP_DEFAULT => write!(f, "BOP_DEFAULT")?, + BOP_PACK => write!(f, "BOP_PACK")?, + BOP_INCLUDE_P => write!(f, "BOP_INCLUDE_P")?, + _ => write!(f, "{bop}")?, + } + write!(f, ")") + } + Invariant::MethodRedefined { klass, method, cme } => { + let class_name = get_class_name(klass); + write!(f, "MethodRedefined({class_name}@{:p}, {}@{:p}, cme:{:p})", + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>()), + method.contents_lossy(), + self.ptr_map.map_id(method.0), + self.ptr_map.map_ptr(cme) + ) + } + Invariant::StableConstantNames { idlist } => { + write!(f, "StableConstantNames({:p}, ", self.ptr_map.map_ptr(idlist))?; + let mut idx = 0; + let mut sep = ""; + loop { + let id = unsafe { *idlist.wrapping_add(idx) }; + if id.0 == 0 { + break; + } + write!(f, "{sep}{}", id.contents_lossy())?; + sep = "::"; + idx += 1; + } + write!(f, ")") + } + Invariant::NoTracePoint => write!(f, "NoTracePoint"), + Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)), + Invariant::SingleRactorMode => write!(f, "SingleRactorMode"), + Invariant::NoSingletonClass { klass } => { + let class_name = get_class_name(klass); + write!(f, "NoSingletonClass({}@{:p})", + class_name, + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>())) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Copy)] +pub enum Const { + Value(VALUE), + CBool(bool), + CInt8(i8), + CInt16(i16), + CInt32(i32), + CInt64(i64), + CUInt8(u8), + CUInt16(u16), + CUInt32(u32), + CShape(ShapeId), + CUInt64(u64), + CPtr(*const u8), + CDouble(f64), +} + +impl std::fmt::Display for Const { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl Const { + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { + ConstPrinter { inner: self, ptr_map } + } +} + +#[derive(Clone, Copy)] +pub enum RangeType { + Inclusive = 0, // include the end value + Exclusive = 1, // exclude the end value +} + +impl std::fmt::Display for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", match self { + RangeType::Inclusive => "NewRangeInclusive", + RangeType::Exclusive => "NewRangeExclusive", + }) + } +} + +impl std::fmt::Debug for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{self}") + } +} + +impl From<u32> for RangeType { + fn from(flag: u32) -> Self { + match flag { + 0 => RangeType::Inclusive, + 1 => RangeType::Exclusive, + _ => panic!("Invalid range flag: {flag}"), + } + } +} + +/// Special regex backref symbol types +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialBackrefSymbol { + LastMatch, // $& + PreMatch, // $` + PostMatch, // $' + LastGroup, // $+ +} + +impl TryFrom<u8> for SpecialBackrefSymbol { + type Error = String; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value as char { + '&' => Ok(SpecialBackrefSymbol::LastMatch), + '`' => Ok(SpecialBackrefSymbol::PreMatch), + '\'' => Ok(SpecialBackrefSymbol::PostMatch), + '+' => Ok(SpecialBackrefSymbol::LastGroup), + c => Err(format!("invalid backref symbol: '{c}'")), + } + } +} + +/// Print adaptor for [`Const`]. See [`PtrPrintMap`]. +pub struct ConstPrinter<'a> { + inner: &'a Const, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for ConstPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.inner { + Const::Value(val) => write!(f, "Value({})", val.print(self.ptr_map)), + // TODO: Break out CPtr as a special case. For some reason, + // when we do that now, {:p} prints a completely different + // number than {:?} does and we don't know why. + // We'll have to resolve that first. + Const::CPtr(val) => write!(f, "CPtr({:?})", self.ptr_map.map_ptr(val)), + &Const::CShape(shape_id) => write!(f, "CShape({:p})", self.ptr_map.map_shape(shape_id)), + _ => write!(f, "{:?}", self.inner), + } + } +} + +/// For output stability in tests, we assign each pointer with a stable +/// address the first time we see it. This mapping is off by default; +/// set [`PtrPrintMap::map_ptrs`] to switch it on. +/// +/// Because this is extra state external to any pointer being printed, a +/// printing adapter struct that wraps the pointer along with this map is +/// required to make use of this effectively. The [`std::fmt::Display`] +/// implementation on the adapter struct can then be reused to implement +/// `Display` on the inner type with a default [`PtrPrintMap`], which +/// does not perform any mapping. +pub struct PtrPrintMap { + inner: RefCell<PtrPrintMapInner>, + map_ptrs: bool, +} + +struct PtrPrintMapInner { + map: HashMap<*const c_void, *const c_void>, + next_ptr: *const c_void, +} + +impl PtrPrintMap { + /// Return a mapper that maps the pointer to itself. + pub fn identity() -> Self { + Self { + map_ptrs: false, + inner: RefCell::new(PtrPrintMapInner { + map: HashMap::default(), next_ptr: + ptr::without_provenance(0x1000) // Simulate 4 KiB zero page + }) + } + } +} + +impl PtrPrintMap { + /// Map a pointer for printing + pub fn map_ptr<T>(&self, ptr: *const T) -> *const T { + // When testing, address stability is not a concern so print real address to enable code + // reuse + if !self.map_ptrs { + return ptr; + } + + use std::collections::hash_map::Entry::*; + let ptr = ptr.cast(); + let inner = &mut *self.inner.borrow_mut(); + match inner.map.entry(ptr) { + Occupied(entry) => entry.get().cast(), + Vacant(entry) => { + // Pick a fake address that is suitably aligns for T and remember it in the map + let mapped = inner.next_ptr.wrapping_add(inner.next_ptr.align_offset(align_of::<T>())); + entry.insert(mapped); + + // Bump for the next pointer + inner.next_ptr = mapped.wrapping_add(size_of::<T>()); + mapped.cast() + } + } + } + + /// Map a Ruby ID (index into intern table) for printing + fn map_id(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + /// Map an index into a Ruby object (e.g. for an ivar) for printing + fn map_index(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + fn map_offset(&self, id: i32) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + /// Map shape ID into a pointer for printing + pub fn map_shape(&self, id: ShapeId) -> *const c_void { + self.map_ptr(id.0 as *const c_void) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum SideExitReason { + UnhandledNewarraySend(vm_opt_newarray_send_type), + UnhandledDuparraySend(u64), + UnknownSpecialVariable(u64), + UnhandledHIRInsn(InsnId), + UnhandledYARVInsn(u32), + UnhandledCallType(CallType), + UnhandledBlockArg, + TooManyKeywordParameters, + FixnumAddOverflow, + FixnumSubOverflow, + FixnumMultOverflow, + FixnumLShiftOverflow, + GuardType(Type), + GuardTypeNot(Type), + GuardShape(ShapeId), + ExpandArray, + GuardNotFrozen, + GuardNotShared, + GuardLess, + GuardGreaterEq, + GuardSuperMethodEntry, + PatchPoint(Invariant), + CalleeSideExit, + ObjToStringFallback, + Interrupt, + BlockParamProxyModified, + BlockParamProxyNotIseqOrIfunc, + StackOverflow, + FixnumModByZero, + FixnumDivByZero, + BoxFixnumOverflow, +} + +#[derive(Debug, Clone, Copy)] +pub enum MethodType { + Iseq, + Cfunc, + Attrset, + Ivar, + Bmethod, + Zsuper, + Alias, + Undefined, + NotImplemented, + Optimized, + Missing, + Refined, + Null, +} + +impl From<u32> for MethodType { + fn from(value: u32) -> Self { + match value { + VM_METHOD_TYPE_ISEQ => MethodType::Iseq, + VM_METHOD_TYPE_CFUNC => MethodType::Cfunc, + VM_METHOD_TYPE_ATTRSET => MethodType::Attrset, + VM_METHOD_TYPE_IVAR => MethodType::Ivar, + VM_METHOD_TYPE_BMETHOD => MethodType::Bmethod, + VM_METHOD_TYPE_ZSUPER => MethodType::Zsuper, + VM_METHOD_TYPE_ALIAS => MethodType::Alias, + VM_METHOD_TYPE_UNDEF => MethodType::Undefined, + VM_METHOD_TYPE_NOTIMPLEMENTED => MethodType::NotImplemented, + VM_METHOD_TYPE_OPTIMIZED => MethodType::Optimized, + VM_METHOD_TYPE_MISSING => MethodType::Missing, + VM_METHOD_TYPE_REFINED => MethodType::Refined, + _ => unreachable!("unknown send_without_block def_type: {}", value), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum OptimizedMethodType { + Send, + Call, + BlockCall, + StructAref, + StructAset, +} + +impl From<u32> for OptimizedMethodType { + fn from(value: u32) -> Self { + match value { + OPTIMIZED_METHOD_TYPE_SEND => OptimizedMethodType::Send, + OPTIMIZED_METHOD_TYPE_CALL => OptimizedMethodType::Call, + OPTIMIZED_METHOD_TYPE_BLOCK_CALL => OptimizedMethodType::BlockCall, + OPTIMIZED_METHOD_TYPE_STRUCT_AREF => OptimizedMethodType::StructAref, + OPTIMIZED_METHOD_TYPE_STRUCT_ASET => OptimizedMethodType::StructAset, + _ => unreachable!("unknown send_without_block optimized method type: {}", value), + } + } +} + +impl std::fmt::Display for SideExitReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SideExitReason::UnhandledYARVInsn(opcode) => write!(f, "UnhandledYARVInsn({})", insn_name(*opcode as usize)), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MAX) => write!(f, "UnhandledNewarraySend(MAX)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MIN) => write!(f, "UnhandledNewarraySend(MIN)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_HASH) => write!(f, "UnhandledNewarraySend(HASH)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK) => write!(f, "UnhandledNewarraySend(PACK)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK_BUFFER) => write!(f, "UnhandledNewarraySend(PACK_BUFFER)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_INCLUDE_P) => write!(f, "UnhandledNewarraySend(INCLUDE_P)"), + SideExitReason::UnhandledDuparraySend(method_id) => write!(f, "UnhandledDuparraySend({method_id})"), + SideExitReason::GuardType(guard_type) => write!(f, "GuardType({guard_type})"), + SideExitReason::GuardTypeNot(guard_type) => write!(f, "GuardTypeNot({guard_type})"), + SideExitReason::GuardNotShared => write!(f, "GuardNotShared"), + SideExitReason::PatchPoint(invariant) => write!(f, "PatchPoint({invariant})"), + _ => write!(f, "{self:?}"), + } + } +} + +/// Result of resolving the receiver type for method dispatch optimization. +/// Represents whether we know the receiver's class statically at compile-time, +/// have profiled type information, or know nothing about it. +pub enum ReceiverTypeResolution { + /// No profile information available for the receiver + NoProfile, + /// The receiver has a monomorphic profile (single type observed, guard needed) + Monomorphic { profiled_type: ProfiledType }, + /// The receiver is polymorphic (multiple types, none dominant) + Polymorphic, + /// The receiver has a skewed polymorphic profile (dominant type with some other types, guard needed) + SkewedPolymorphic { profiled_type: ProfiledType }, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic { profiled_type: ProfiledType }, + /// The receiver's class is statically known at JIT compile-time (no guard needed) + StaticallyKnown { class: VALUE }, +} + +/// Reason why a send-ish instruction cannot be optimized from a fallback instruction +#[derive(Debug, Clone, Copy)] +pub enum SendFallbackReason { + SendWithoutBlockPolymorphic, + SendWithoutBlockMegamorphic, + SendWithoutBlockNoProfiles, + SendWithoutBlockCfuncNotVariadic, + SendWithoutBlockCfuncArrayVariadic, + SendWithoutBlockNotOptimizedMethodType(MethodType), + SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType), + SendWithoutBlockNotOptimizedNeedPermission, + SendWithoutBlockBopRedefined, + SendWithoutBlockOperandsNotFixnum, + SendWithoutBlockDirectKeywordMismatch, + SendWithoutBlockDirectKeywordCountMismatch, + SendWithoutBlockDirectMissingKeyword, + SendWithoutBlockDirectTooManyKeywords, + SendPolymorphic, + SendMegamorphic, + SendNoProfiles, + SendCfuncVariadic, + SendCfuncArrayVariadic, + SendNotOptimizedMethodType(MethodType), + SendNotOptimizedNeedPermission, + CCallWithFrameTooManyArgs, + ObjToStringNotString, + TooManyArgsForLir, + /// The Proc object for a BMETHOD is not defined by an ISEQ. (See `enum rb_block_type`.) + BmethodNonIseqProc, + /// Caller supplies too few or too many arguments than what the callee's parameters expects. + ArgcParamMismatch, + /// The call has at least one feature on the caller or callee side that the optimizer does not + /// support. + ComplexArgPass, + /// Caller has keyword arguments but callee doesn't expect them; need to convert to hash. + UnexpectedKeywordArgs, + /// A singleton class has been seen for the receiver class, so we skip the optimization + /// to avoid an invalidation loop. + SingletonClassSeen, + /// The super call is passed a block that the optimizer does not support. + SuperCallWithBlock, + /// The profiled super class cannot be found. + SuperClassNotFound, + /// The `super` call uses a complex argument pattern that the optimizer does not support. + SuperComplexArgsPass, + /// The cached target of a `super` call could not be found. + SuperTargetNotFound, + /// Attempted to specialize a `super` call that doesn't have profile data. + SuperNoProfiles, + /// Cannot optimize the `super` call due to the target method. + SuperNotOptimizedMethodType(MethodType), + /// The `super` call is polymorpic. + SuperPolymorphic, + /// The `super` target call uses a complex argument pattern that the optimizer does not support. + SuperTargetComplexArgsPass, + /// Initial fallback reason for every instruction, which should be mutated to + /// a more actionable reason when an attempt to specialize the instruction fails. + Uncategorized(ruby_vminsn_type), +} + +impl Display for SendFallbackReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SendWithoutBlockPolymorphic => write!(f, "SendWithoutBlock: polymorphic call site"), + SendWithoutBlockMegamorphic => write!(f, "SendWithoutBlock: megamorphic call site"), + SendWithoutBlockNoProfiles => write!(f, "SendWithoutBlock: no profile data available"), + SendWithoutBlockCfuncNotVariadic => write!(f, "SendWithoutBlock: C function is not variadic"), + SendWithoutBlockCfuncArrayVariadic => write!(f, "SendWithoutBlock: C function expects array variadic"), + SendWithoutBlockNotOptimizedMethodType(method_type) => write!(f, "SendWithoutBlock: unsupported method type {:?}", method_type), + SendWithoutBlockNotOptimizedMethodTypeOptimized(opt_type) => write!(f, "SendWithoutBlock: unsupported optimized method type {:?}", opt_type), + SendWithoutBlockNotOptimizedNeedPermission => write!(f, "SendWithoutBlock: method private or protected and no FCALL"), + SendNotOptimizedNeedPermission => write!(f, "Send: method private or protected and no FCALL"), + SendWithoutBlockBopRedefined => write!(f, "SendWithoutBlock: basic operation was redefined"), + SendWithoutBlockOperandsNotFixnum => write!(f, "SendWithoutBlock: operands are not fixnums"), + SendWithoutBlockDirectKeywordMismatch => write!(f, "SendWithoutBlockDirect: keyword mismatch"), + SendWithoutBlockDirectKeywordCountMismatch => write!(f, "SendWithoutBlockDirect: keyword count mismatch"), + SendWithoutBlockDirectMissingKeyword => write!(f, "SendWithoutBlockDirect: missing keyword"), + SendWithoutBlockDirectTooManyKeywords => write!(f, "SendWithoutBlockDirect: too many keywords for fixnum bitmask"), + SendPolymorphic => write!(f, "Send: polymorphic call site"), + SendMegamorphic => write!(f, "Send: megamorphic call site"), + SendNoProfiles => write!(f, "Send: no profile data available"), + SendCfuncVariadic => write!(f, "Send: C function is variadic"), + SendCfuncArrayVariadic => write!(f, "Send: C function expects array variadic"), + SendNotOptimizedMethodType(method_type) => write!(f, "Send: unsupported method type {:?}", method_type), + CCallWithFrameTooManyArgs => write!(f, "CCallWithFrame: too many arguments"), + ObjToStringNotString => write!(f, "ObjToString: result is not a string"), + TooManyArgsForLir => write!(f, "Too many arguments for LIR"), + BmethodNonIseqProc => write!(f, "Bmethod: Proc object is not defined by an ISEQ"), + ArgcParamMismatch => write!(f, "Argument count does not match parameter count"), + ComplexArgPass => write!(f, "Complex argument passing"), + UnexpectedKeywordArgs => write!(f, "Unexpected Keyword Args"), + SingletonClassSeen => write!(f, "Singleton class previously created for receiver class"), + SuperCallWithBlock => write!(f, "super: call made with a block"), + SuperClassNotFound => write!(f, "super: profiled class cannot be found"), + SuperComplexArgsPass => write!(f, "super: complex argument passing to `super` call"), + SuperNoProfiles => write!(f, "super: no profile data available"), + SuperNotOptimizedMethodType(method_type) => write!(f, "super: unsupported target method type {:?}", method_type), + SuperPolymorphic => write!(f, "super: polymorphic call site"), + SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), + SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), + } + } +} + +/// An instruction in the SSA IR. The output of an instruction is referred to by the index of +/// the instruction ([`InsnId`]). SSA form enables this, and [`UnionFind`] ([`Function::find`]) +/// helps with editing. +#[derive(Debug, Clone)] +pub enum Insn { + Const { val: Const }, + /// SSA block parameter. Also used for function parameters in the function's entry block. + Param, + + StringCopy { val: InsnId, chilled: bool, state: InsnId }, + StringIntern { val: InsnId, state: InsnId }, + StringConcat { strings: Vec<InsnId>, state: InsnId }, + /// Call rb_str_getbyte with known-Fixnum index + StringGetbyte { string: InsnId, index: InsnId }, + StringSetbyteFixnum { string: InsnId, index: InsnId, value: InsnId }, + StringAppend { recv: InsnId, other: InsnId, state: InsnId }, + StringAppendCodepoint { recv: InsnId, other: InsnId, state: InsnId }, + + /// Combine count stack values into a regexp + ToRegexp { opt: usize, values: Vec<InsnId>, state: InsnId }, + + /// Put special object (VMCORE, CBASE, etc.) based on value_type + PutSpecialObject { value_type: SpecialObjectType }, + + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. + ToArray { val: InsnId, state: InsnId }, + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. If we + /// called `to_a`, duplicate the returned array. + ToNewArray { val: InsnId, state: InsnId }, + NewArray { elements: Vec<InsnId>, state: InsnId }, + /// NewHash contains a vec of (key, value) pairs + NewHash { elements: Vec<InsnId>, state: InsnId }, + NewRange { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + NewRangeFixnum { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + ArrayDup { val: InsnId, state: InsnId }, + ArrayHash { elements: Vec<InsnId>, state: InsnId }, + ArrayMax { elements: Vec<InsnId>, state: InsnId }, + ArrayInclude { elements: Vec<InsnId>, target: InsnId, state: InsnId }, + ArrayPackBuffer { elements: Vec<InsnId>, fmt: InsnId, buffer: InsnId, state: InsnId }, + DupArrayInclude { ary: VALUE, target: InsnId, state: InsnId }, + /// Extend `left` with the elements from `right`. `left` and `right` must both be `Array`. + ArrayExtend { left: InsnId, right: InsnId, state: InsnId }, + /// Push `val` onto `array`, where `array` is already `Array`. + ArrayPush { array: InsnId, val: InsnId, state: InsnId }, + ArrayAref { array: InsnId, index: InsnId }, + ArrayAset { array: InsnId, index: InsnId, val: InsnId }, + ArrayPop { array: InsnId, state: InsnId }, + /// Return the length of the array as a C `long` ([`types::CInt64`]) + ArrayLength { array: InsnId }, + + HashAref { hash: InsnId, key: InsnId, state: InsnId }, + HashAset { hash: InsnId, key: InsnId, val: InsnId, state: InsnId }, + HashDup { val: InsnId, state: InsnId }, + + /// Allocate an instance of the `val` object without calling `#initialize` on it. + /// This can: + /// * raise an exception if `val` is not a class + /// * run arbitrary code if `val` is a class with a custom allocator + ObjectAlloc { val: InsnId, state: InsnId }, + /// Allocate an instance of the `val` class without calling `#initialize` on it. + /// This requires that `class` has the default allocator (for example via `IsMethodCfunc`). + /// This won't raise or run arbitrary code because `class` has the default allocator. + ObjectAllocClass { class: VALUE, state: InsnId }, + + /// Check if the value is truthy and "return" a C boolean. In reality, we will likely fuse this + /// with IfTrue/IfFalse in the backend to generate jcc. + Test { val: InsnId }, + /// Return C `true` if `val` is `Qnil`, else `false`. + IsNil { val: InsnId }, + /// Return C `true` if `val`'s method on cd resolves to the cfunc. + IsMethodCfunc { val: InsnId, cd: *const rb_call_data, cfunc: *const u8, state: InsnId }, + /// Return C `true` if left == right + IsBitEqual { left: InsnId, right: InsnId }, + /// Return C `true` if left != right + IsBitNotEqual { left: InsnId, right: InsnId }, + /// Convert a C `bool` to a Ruby `Qtrue`/`Qfalse`. Same as `RBOOL` macro. + BoxBool { val: InsnId }, + /// Convert a C `long` to a Ruby `Fixnum`. Side exit on overflow. + BoxFixnum { val: InsnId, state: InsnId }, + UnboxFixnum { val: InsnId }, + FixnumAref { recv: InsnId, index: InsnId }, + // TODO(max): In iseq body types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. + Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, state: InsnId }, + GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, + /// Kernel#block_given? but without pushing a frame. Similar to [`Insn::Defined`] with + /// `DEFINED_YIELD` + IsBlockGiven { lep: InsnId }, + /// Test the bit at index of val, a Fixnum. + /// Return Qtrue if the bit is set, else Qfalse. + FixnumBitCheck { val: InsnId, index: u8 }, + /// Return Qtrue if `val` is an instance of `class`, else Qfalse. + /// Equivalent to `class_search_ancestor(CLASS_OF(val), class)`. + IsA { val: InsnId, class: InsnId }, + + /// Get a global variable named `id` + GetGlobal { id: ID, state: InsnId }, + /// Set a global variable named `id` to `val` + SetGlobal { id: ID, val: InsnId, state: InsnId }, + + //NewObject? + /// Get an instance variable `id` from `self_val`, using the inline cache `ic` if present + GetIvar { self_val: InsnId, id: ID, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Set `self_val`'s instance variable `id` to `val`, using the inline cache `ic` if present + SetIvar { self_val: InsnId, id: ID, val: InsnId, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Check whether an instance variable exists on `self_val` + DefinedIvar { self_val: InsnId, id: ID, pushval: VALUE, state: InsnId }, + + /// Load cfp->pc + LoadPC, + /// Load EC + LoadEC, + /// Load cfp->self + LoadSelf, + LoadField { recv: InsnId, id: ID, offset: i32, return_type: Type }, + /// Write `val` at an offset of `recv`. + /// When writing a Ruby object to a Ruby object, one must use GuardNotFrozen (or equivalent) before and WriteBarrier after. + StoreField { recv: InsnId, id: ID, offset: i32, val: InsnId }, + WriteBarrier { recv: InsnId, val: InsnId }, + + /// Get a local variable from a higher scope or the heap. + /// If `use_sp` is true, it uses the SP register to optimize the read. + /// `rest_param` is used by infer_types to infer the ArrayExact type. + GetLocal { level: u32, ep_offset: u32, use_sp: bool, rest_param: bool }, + /// Set a local variable in a higher scope or the heap + SetLocal { level: u32, ep_offset: u32, val: InsnId }, + GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, + GetSpecialNumber { nth: u64, state: InsnId }, + + /// Get a class variable `id` + GetClassVar { id: ID, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Set a class variable `id` to `val` + SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + + /// Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + /// Equivalent of GET_LEP() macro. + GetLEP, + + /// Own a FrameState so that instructions can look up their dominating FrameState when + /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate + /// any code. + Snapshot { state: FrameState }, + + /// Unconditional jump + Jump(BranchEdge), + + /// Conditional branch instructions + IfTrue { val: InsnId, target: BranchEdge }, + IfFalse { val: InsnId, target: BranchEdge }, + + /// Call a C function without pushing a frame + /// `name` is for printing purposes only + CCall { cfunc: *const u8, recv: InsnId, args: Vec<InsnId>, name: ID, return_type: Type, elidable: bool }, + + /// Call a C function that pushes a frame + CCallWithFrame { + cd: *const rb_call_data, // cd for falling back to SendWithoutBlock + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + blockiseq: Option<IseqPtr>, + }, + + /// Call a variadic C function with signature: func(int argc, VALUE *argv, VALUE recv) + /// This handles frame setup, argv creation, and frame teardown all in one + CCallVariadic { + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + blockiseq: Option<IseqPtr>, + }, + + /// Un-optimized fallback implementation (dynamic dispatch) for send-ish instructions + /// Ignoring keyword arguments etc for now + SendWithoutBlock { + recv: InsnId, + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + Send { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + SendForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuper { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeBlock { + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + /// Call Proc#call optimized method type. + InvokeProc { + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + kw_splat: bool, + }, + + /// Optimized ISEQ call + SendWithoutBlockDirect { + recv: InsnId, + cd: *const rb_call_data, + cme: *const rb_callable_method_entry_t, + iseq: IseqPtr, + args: Vec<InsnId>, + kw_bits: u32, + state: InsnId, + }, + + // Invoke a builtin function + InvokeBuiltin { + bf: rb_builtin_function, + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + leaf: bool, + return_type: Option<Type>, // None for unannotated builtins + }, + + /// Set up frame. Remember the address as the JIT entry for the insn_idx in `jit_entry_insns()[jit_entry_idx]`. + EntryPoint { jit_entry_idx: Option<usize> }, + /// Control flow instructions + Return { val: InsnId }, + /// Non-local control flow. See the throw YARV instruction + Throw { throw_state: u32, val: InsnId, state: InsnId }, + + /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, |, ^, << + FixnumAdd { left: InsnId, right: InsnId, state: InsnId }, + FixnumSub { left: InsnId, right: InsnId, state: InsnId }, + FixnumMult { left: InsnId, right: InsnId, state: InsnId }, + FixnumDiv { left: InsnId, right: InsnId, state: InsnId }, + FixnumMod { left: InsnId, right: InsnId, state: InsnId }, + FixnumEq { left: InsnId, right: InsnId }, + FixnumNeq { left: InsnId, right: InsnId }, + FixnumLt { left: InsnId, right: InsnId }, + FixnumLe { left: InsnId, right: InsnId }, + FixnumGt { left: InsnId, right: InsnId }, + FixnumGe { left: InsnId, right: InsnId }, + FixnumAnd { left: InsnId, right: InsnId }, + FixnumOr { left: InsnId, right: InsnId }, + FixnumXor { left: InsnId, right: InsnId }, + FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, + FixnumRShift { left: InsnId, right: InsnId }, + + // Distinct from `SendWithoutBlock` with `mid:to_s` because does not have a patch point for String to_s being redefined + ObjToString { val: InsnId, cd: *const rb_call_data, state: InsnId }, + AnyToString { val: InsnId, str: InsnId, state: InsnId }, + + /// Refine the known type information of with additional type information. + /// Computes the intersection of the existing type and the new type. + RefineType { val: InsnId, new_type: Type }, + + /// Side-exit if val doesn't have the expected type. + GuardType { val: InsnId, guard_type: Type, state: InsnId }, + GuardTypeNot { val: InsnId, guard_type: Type, state: InsnId }, + /// Side-exit if val is not the expected Const. + GuardBitEquals { val: InsnId, expected: Const, reason: SideExitReason, state: InsnId }, + /// Side-exit if val doesn't have the expected shape. + GuardShape { val: InsnId, shape: ShapeId, state: InsnId }, + /// Side-exit if the block param has been modified or the block handler for the frame + /// is neither ISEQ nor ifunc, which makes it incompatible with rb_block_param_proxy. + GuardBlockParamProxy { level: u32, state: InsnId }, + /// Side-exit if val is frozen. Does *not* check if the val is an immediate; assumes that it is + /// a heap object. + GuardNotFrozen { recv: InsnId, state: InsnId }, + /// Side-exit if val is shared. Does *not* check if the val is an immediate; assumes + /// that it is a heap object. + GuardNotShared { recv: InsnId, state: InsnId }, + /// Side-exit if left is not greater than or equal to right (both operands are C long). + GuardGreaterEq { left: InsnId, right: InsnId, state: InsnId }, + /// Side-exit if left is not less than right (both operands are C long). + GuardLess { left: InsnId, right: InsnId, state: InsnId }, + /// Side-exit if the method entry at ep[VM_ENV_DATA_INDEX_ME_CREF] doesn't match the expected CME. + /// Used to ensure super calls are made from the expected method context. + GuardSuperMethodEntry { lep: InsnId, cme: *const rb_callable_method_entry_t, state: InsnId }, + /// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). + GetBlockHandler { lep: InsnId }, + + /// Generate no code (or padding if necessary) and insert a patch point + /// that can be rewritten to a side exit when the Invariant is broken. + PatchPoint { invariant: Invariant, state: InsnId }, + + /// Side-exit into the interpreter. + SideExit { state: InsnId, reason: SideExitReason }, + + /// Increment a counter in ZJIT stats + IncrCounter(Counter), + + /// Increment a counter in ZJIT stats for the given counter pointer + IncrCounterPtr { counter_ptr: *mut u64 }, + + /// Equivalent of RUBY_VM_CHECK_INTS. Automatically inserted by the compiler before jumps and + /// return instructions. + CheckInterrupts { state: InsnId }, +} + +impl Insn { + /// Not every instruction returns a value. Return true if the instruction does and false otherwise. + pub fn has_output(&self) -> bool { + match self { + Insn::Jump(_) + | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::EntryPoint { .. } | Insn::Return { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. } + | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } + | Insn::ArrayAset { .. } => false, + _ => true, + } + } + + /// Return true if the instruction ends a basic block and false otherwise. + pub fn is_terminator(&self) -> bool { + match self { + Insn::Jump(_) | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => true, + _ => false, + } + } + + pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap, iseq: Option<IseqPtr>) -> InsnPrinter<'a> { + InsnPrinter { inner: self.clone(), ptr_map, iseq } + } + + // TODO(Jacob): Model SP. ie, all allocations modify stack size but using the effect for stack modification feels excessive + // TODO(Jacob): Add sideeffect failure bit + fn effects_of(&self) -> Effect { + const allocates: Effect = Effect::read_write(abstract_heaps::PC.union(abstract_heaps::Allocator), abstract_heaps::Allocator); + match &self { + Insn::Const { .. } => effects::Empty, + Insn::Param { .. } => effects::Empty, + Insn::StringCopy { .. } => allocates, + Insn::StringIntern { .. } => effects::Any, + Insn::StringConcat { .. } => effects::Any, + Insn::StringGetbyte { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StringSetbyteFixnum { .. } => effects::Any, + Insn::StringAppend { .. } => effects::Any, + Insn::StringAppendCodepoint { .. } => effects::Any, + Insn::ToRegexp { .. } => effects::Any, + Insn::PutSpecialObject { .. } => effects::Any, + Insn::ToArray { .. } => effects::Any, + Insn::ToNewArray { .. } => effects::Any, + Insn::NewArray { .. } => allocates, + Insn::NewHash { elements, .. } => { + // NewHash's operands may be hashed and compared for equality, which could have + // side-effects. Empty hashes are definitely elidable. + if elements.is_empty() { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::NewRange { .. } => effects::Any, + Insn::NewRangeFixnum { .. } => allocates, + Insn::ArrayDup { .. } => allocates, + Insn::ArrayHash { .. } => effects::Any, + Insn::ArrayMax { .. } => effects::Any, + Insn::ArrayInclude { .. } => effects::Any, + Insn::ArrayPackBuffer { .. } => effects::Any, + Insn::DupArrayInclude { .. } => effects::Any, + Insn::ArrayExtend { .. } => effects::Any, + Insn::ArrayPush { .. } => effects::Any, + Insn::ArrayAref { .. } => effects::Any, + Insn::ArrayAset { .. } => effects::Any, + Insn::ArrayPop { .. } => effects::Any, + Insn::ArrayLength { .. } => Effect::write(abstract_heaps::Empty), + Insn::HashAref { .. } => effects::Any, + Insn::HashAset { .. } => effects::Any, + Insn::HashDup { .. } => allocates, + Insn::ObjectAlloc { .. } => effects::Any, + Insn::ObjectAllocClass { .. } => allocates, + Insn::Test { .. } => effects::Empty, + Insn::IsNil { .. } => effects::Empty, + Insn::IsMethodCfunc { .. } => effects::Any, + Insn::IsBitEqual { .. } => effects::Empty, + Insn::IsBitNotEqual { .. } => effects::Any, + Insn::BoxBool { .. } => effects::Empty, + Insn::BoxFixnum { .. } => effects::Empty, + Insn::UnboxFixnum { .. } => effects::Any, + Insn::FixnumAref { .. } => effects::Empty, + Insn::Defined { .. } => effects::Any, + Insn::GetConstantPath { .. } => effects::Any, + Insn::IsBlockGiven { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::FixnumBitCheck { .. } => effects::Any, + Insn::IsA { .. } => effects::Empty, + Insn::GetGlobal { .. } => effects::Any, + Insn::SetGlobal { .. } => effects::Any, + Insn::GetIvar { .. } => effects::Any, + Insn::SetIvar { .. } => effects::Any, + Insn::DefinedIvar { .. } => effects::Any, + Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), + Insn::LoadEC { .. } => effects::Empty, + Insn::GetLEP { .. } => effects::Empty, + Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), + Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StoreField { .. } => effects::Any, + Insn::WriteBarrier { .. } => effects::Any, + Insn::GetLocal { .. } => Effect::read_write(abstract_heaps::Locals, abstract_heaps::Empty), + Insn::SetLocal { .. } => effects::Any, + Insn::GetSpecialSymbol { .. } => effects::Any, + Insn::GetSpecialNumber { .. } => effects::Any, + Insn::GetClassVar { .. } => effects::Any, + Insn::SetClassVar { .. } => effects::Any, + Insn::Snapshot { .. } => effects::Empty, + Insn::Jump(_) => effects::Any, + Insn::IfTrue { .. } => effects::Any, + Insn::IfFalse { .. } => effects::Any, + Insn::CCall { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallWithFrame { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallVariadic { .. } => effects::Any, + Insn::SendWithoutBlock { .. } => effects::Any, + Insn::Send { .. } => effects::Any, + Insn::SendForward { .. } => effects::Any, + Insn::InvokeSuper { .. } => effects::Any, + Insn::InvokeBlock { .. } => effects::Any, + Insn::SendWithoutBlockDirect { .. } => effects::Any, + Insn::InvokeBuiltin { .. } => effects::Any, + Insn::EntryPoint { .. } => effects::Any, + Insn::Return { .. } => effects::Any, + Insn::Throw { .. } => effects::Any, + Insn::FixnumAdd { .. } => effects::Empty, + Insn::FixnumSub { .. } => effects::Empty, + Insn::FixnumMult { .. } => effects::Empty, + Insn::FixnumDiv { .. } => effects::Any, + Insn::FixnumMod { .. } => effects::Any, + Insn::FixnumEq { .. } => effects::Empty, + Insn::FixnumNeq { .. } => effects::Empty, + Insn::FixnumLt { .. } => effects::Empty, + Insn::FixnumLe { .. } => effects::Empty, + Insn::FixnumGt { .. } => effects::Empty, + Insn::FixnumGe { .. } => effects::Empty, + Insn::FixnumAnd { .. } => effects::Empty, + Insn::FixnumOr { .. } => effects::Empty, + Insn::FixnumXor { .. } => effects::Empty, + Insn::FixnumLShift { .. } => effects::Empty, + Insn::FixnumRShift { .. } => effects::Empty, + Insn::ObjToString { .. } => effects::Any, + Insn::AnyToString { .. } => effects::Any, + Insn::GuardType { .. } => effects::Any, + Insn::GuardTypeNot { .. } => effects::Any, + Insn::GuardBitEquals { .. } => effects::Any, + Insn::GuardShape { .. } => effects::Any, + Insn::GuardBlockParamProxy { .. } => effects::Any, + Insn::GuardNotFrozen { .. } => effects::Any, + Insn::GuardNotShared { .. } => effects::Any, + Insn::GuardGreaterEq { .. } => effects::Any, + Insn::GuardSuperMethodEntry { .. } => effects::Any, + Insn::GetBlockHandler { .. } => effects::Any, + Insn::GuardLess { .. } => effects::Any, + Insn::PatchPoint { .. } => effects::Any, + Insn::SideExit { .. } => effects::Any, + Insn::IncrCounter(_) => effects::Any, + Insn::IncrCounterPtr { .. } => effects::Any, + Insn::CheckInterrupts { .. } => effects::Any, + Insn::InvokeProc { .. } => effects::Any, + Insn::RefineType { .. } => effects::Empty, + } + } + + /// Return true if we can safely omit the instruction. This occurs when one of the following + /// conditions are met. + /// 1. The instruction does not write anything. + /// 2. The instruction only allocates and writes nothing else. + /// Calling the effects of our instruction `insn_effects`, we need: + /// `effects::Empty` to include `insn_effects.write` or `effects::Allocator` to include + /// `insn_effects.write`. + /// We can simplify this to `effects::Empty.union(effects::Allocator).includes(insn_effects.write)`. + /// But the union of `Allocator` and `Empty` is simply `Allocator`, so our entire function + /// collapses to `effects::Allocator.includes(insn_effects.write)`. + /// Note: These are restrictions on the `write` `EffectSet` only. Even instructions with + /// `read: effects::Any` could potentially be omitted. + fn is_elidable(&self) -> bool { + abstract_heaps::Allocator.includes(self.effects_of().write_bits()) + } +} + +/// Print adaptor for [`Insn`]. See [`PtrPrintMap`]. +pub struct InsnPrinter<'a> { + inner: Insn, + ptr_map: &'a PtrPrintMap, + iseq: Option<IseqPtr>, +} + +/// Get the name of a local variable given iseq, level, and ep_offset. +/// Returns +/// - `":name"` if iseq is available and name is a real identifier, +/// - `"<empty>"` for anonymous locals. +/// - `None` if iseq is not available. +/// (When `Insn` is printed in a panic/debug message the `Display::fmt` method is called, which can't access an iseq.) +/// +/// This mimics local_var_name() from iseq.c. +fn get_local_var_name_for_printer(iseq: Option<IseqPtr>, level: u32, ep_offset: u32) -> Option<String> { + let mut current_iseq = iseq?; + for _ in 0..level { + current_iseq = unsafe { rb_get_iseq_body_parent_iseq(current_iseq) }; + } + let local_idx = ep_offset_to_local_idx(current_iseq, ep_offset); + let id: ID = unsafe { rb_zjit_local_id(current_iseq, local_idx.try_into().unwrap()) }; + + if id.0 == 0 || unsafe { rb_id2str(id) } == Qfalse { + return Some(String::from("<empty>")); + } + + Some(format!(":{}", id.contents_lossy())) +} + +static REGEXP_FLAGS: &[(u32, &str)] = &[ + (ONIG_OPTION_MULTILINE, "MULTILINE"), + (ONIG_OPTION_IGNORECASE, "IGNORECASE"), + (ONIG_OPTION_EXTEND, "EXTENDED"), + (ARG_ENCODING_FIXED, "FIXEDENCODING"), + (ARG_ENCODING_NONE, "NOENCODING"), +]; + +impl<'a> std::fmt::Display for InsnPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match &self.inner { + Insn::Const { val } => { write!(f, "Const {}", val.print(self.ptr_map)) } + Insn::Param => { write!(f, "Param") } + Insn::NewArray { elements, .. } => { + write!(f, "NewArray")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayAref { array, index, .. } => { + write!(f, "ArrayAref {array}, {index}") + } + Insn::ArrayAset { array, index, val, ..} => { + write!(f, "ArrayAset {array}, {index}, {val}") + } + Insn::ArrayPop { array, .. } => { + write!(f, "ArrayPop {array}") + } + Insn::ArrayLength { array } => { + write!(f, "ArrayLength {array}") + } + Insn::NewHash { elements, .. } => { + write!(f, "NewHash")?; + let mut prefix = " "; + for chunk in elements.chunks(2) { + if let [key, value] = chunk { + write!(f, "{prefix}{key}: {value}")?; + prefix = ", "; + } + } + Ok(()) + } + Insn::NewRange { low, high, flag, .. } => { + write!(f, "NewRange {low} {flag} {high}") + } + Insn::NewRangeFixnum { low, high, flag, .. } => { + write!(f, "NewRangeFixnum {low} {flag} {high}") + } + Insn::ArrayMax { elements, .. } => { + write!(f, "ArrayMax")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayHash { elements, .. } => { + write!(f, "ArrayHash")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayInclude { elements, target, .. } => { + write!(f, "ArrayInclude")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + write!(f, " | {target}") + } + Insn::ArrayPackBuffer { elements, fmt, buffer, .. } => { + write!(f, "ArrayPackBuffer ")?; + for element in elements { + write!(f, "{element}, ")?; + } + write!(f, "fmt: {fmt}, buf: {buffer}") + } + Insn::DupArrayInclude { ary, target, .. } => { + write!(f, "DupArrayInclude {} | {}", ary.print(self.ptr_map), target) + } + Insn::ArrayDup { val, .. } => { write!(f, "ArrayDup {val}") } + Insn::HashDup { val, .. } => { write!(f, "HashDup {val}") } + Insn::HashAref { hash, key, .. } => { write!(f, "HashAref {hash}, {key}")} + Insn::HashAset { hash, key, val, .. } => { write!(f, "HashAset {hash}, {key}, {val}")} + Insn::ObjectAlloc { val, .. } => { write!(f, "ObjectAlloc {val}") } + &Insn::ObjectAllocClass { class, .. } => { + let class_name = get_class_name(class); + write!(f, "ObjectAllocClass {class_name}:{}", class.print(self.ptr_map)) + } + Insn::StringCopy { val, .. } => { write!(f, "StringCopy {val}") } + Insn::StringConcat { strings, .. } => { + write!(f, "StringConcat")?; + let mut prefix = " "; + for string in strings { + write!(f, "{prefix}{string}")?; + prefix = ", "; + } + + Ok(()) + } + Insn::StringGetbyte { string, index, .. } => { + write!(f, "StringGetbyte {string}, {index}") + } + Insn::StringSetbyteFixnum { string, index, value, .. } => { + write!(f, "StringSetbyteFixnum {string}, {index}, {value}") + } + Insn::StringAppend { recv, other, .. } => { + write!(f, "StringAppend {recv}, {other}") + } + Insn::StringAppendCodepoint { recv, other, .. } => { + write!(f, "StringAppendCodepoint {recv}, {other}") + } + Insn::ToRegexp { values, opt, .. } => { + write!(f, "ToRegexp")?; + let mut prefix = " "; + for value in values { + write!(f, "{prefix}{value}")?; + prefix = ", "; + } + + let opt = *opt as u32; + if opt != 0 { + write!(f, ", ")?; + let mut sep = ""; + for (flag, name) in REGEXP_FLAGS { + if opt & flag != 0 { + write!(f, "{sep}{name}")?; + sep = "|"; + } + } + } + + Ok(()) + } + Insn::Test { val } => { write!(f, "Test {val}") } + Insn::IsNil { val } => { write!(f, "IsNil {val}") } + Insn::IsMethodCfunc { val, cd, .. } => { write!(f, "IsMethodCFunc {val}, :{}", ruby_call_method_name(*cd)) } + Insn::IsBitEqual { left, right } => write!(f, "IsBitEqual {left}, {right}"), + Insn::IsBitNotEqual { left, right } => write!(f, "IsBitNotEqual {left}, {right}"), + Insn::BoxBool { val } => write!(f, "BoxBool {val}"), + Insn::BoxFixnum { val, .. } => write!(f, "BoxFixnum {val}"), + Insn::UnboxFixnum { val } => write!(f, "UnboxFixnum {val}"), + Insn::FixnumAref { recv, index } => write!(f, "FixnumAref {recv}, {index}"), + Insn::Jump(target) => { write!(f, "Jump {target}") } + Insn::IfTrue { val, target } => { write!(f, "IfTrue {val}, {target}") } + Insn::IfFalse { val, target } => { write!(f, "IfFalse {val}, {target}") } + Insn::SendWithoutBlock { recv, cd, args, reason, .. } => { + write!(f, "SendWithoutBlock {recv}, :{}", ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::SendWithoutBlockDirect { recv, cd, iseq, args, .. } => { + write!(f, "SendWithoutBlockDirect {recv}, :{} ({:?})", ruby_call_method_name(*cd), self.ptr_map.map_ptr(iseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + Insn::Send { recv, cd, args, blockiseq, reason, .. } => { + // For tests, we want to check HIR snippets textually. Addresses change + // between runs, making tests fail. Instead, pick an arbitrary hex value to + // use as a "pointer" so we can check the rest of the HIR. + write!(f, "Send {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::SendForward { recv, cd, args, blockiseq, reason, .. } => { + write!(f, "SendForward {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuper { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuper {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeBlock { args, reason, .. } => { + write!(f, "InvokeBlock")?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeProc { recv, args, kw_splat, .. } => { + write!(f, "InvokeProc {recv}")?; + for arg in args { + write!(f, ", {arg}")?; + } + if *kw_splat { + write!(f, ", kw_splat")?; + } + Ok(()) + } + Insn::InvokeBuiltin { bf, args, leaf, .. } => { + let bf_name = unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap(); + write!(f, "InvokeBuiltin{} {}", + if *leaf { " leaf" } else { "" }, + // e.g. Code that use `Primitive.cexpr!`. From BUILTIN_INLINE_PREFIX. + if bf_name.starts_with("_bi") { "<inline_expr>" } else { bf_name })?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + &Insn::EntryPoint { jit_entry_idx: Some(idx) } => write!(f, "EntryPoint JIT({idx})"), + &Insn::EntryPoint { jit_entry_idx: None } => write!(f, "EntryPoint interpreter"), + Insn::Return { val } => { write!(f, "Return {val}") } + Insn::FixnumAdd { left, right, .. } => { write!(f, "FixnumAdd {left}, {right}") }, + Insn::FixnumSub { left, right, .. } => { write!(f, "FixnumSub {left}, {right}") }, + Insn::FixnumMult { left, right, .. } => { write!(f, "FixnumMult {left}, {right}") }, + Insn::FixnumDiv { left, right, .. } => { write!(f, "FixnumDiv {left}, {right}") }, + Insn::FixnumMod { left, right, .. } => { write!(f, "FixnumMod {left}, {right}") }, + Insn::FixnumEq { left, right, .. } => { write!(f, "FixnumEq {left}, {right}") }, + Insn::FixnumNeq { left, right, .. } => { write!(f, "FixnumNeq {left}, {right}") }, + Insn::FixnumLt { left, right, .. } => { write!(f, "FixnumLt {left}, {right}") }, + Insn::FixnumLe { left, right, .. } => { write!(f, "FixnumLe {left}, {right}") }, + Insn::FixnumGt { left, right, .. } => { write!(f, "FixnumGt {left}, {right}") }, + Insn::FixnumGe { left, right, .. } => { write!(f, "FixnumGe {left}, {right}") }, + Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, + Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, + Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, + Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, + Insn::GuardType { val, guard_type, .. } => { write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map)) }, + Insn::RefineType { val, new_type, .. } => { write!(f, "RefineType {val}, {}", new_type.print(self.ptr_map)) }, + Insn::GuardTypeNot { val, guard_type, .. } => { write!(f, "GuardTypeNot {val}, {}", guard_type.print(self.ptr_map)) }, + Insn::GuardBitEquals { val, expected, .. } => { write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map)) }, + &Insn::GuardShape { val, shape, .. } => { write!(f, "GuardShape {val}, {:p}", self.ptr_map.map_shape(shape)) }, + Insn::GuardBlockParamProxy { level, .. } => write!(f, "GuardBlockParamProxy l{level}"), + Insn::GuardNotFrozen { recv, .. } => write!(f, "GuardNotFrozen {recv}"), + Insn::GuardNotShared { recv, .. } => write!(f, "GuardNotShared {recv}"), + Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), + Insn::GuardGreaterEq { left, right, .. } => write!(f, "GuardGreaterEq {left}, {right}"), + Insn::GuardSuperMethodEntry { lep, cme, .. } => write!(f, "GuardSuperMethodEntry {lep}, {:p}", self.ptr_map.map_ptr(cme)), + Insn::GetBlockHandler { lep } => write!(f, "GetBlockHandler {lep}"), + Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, + Insn::GetConstantPath { ic, .. } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, + Insn::IsBlockGiven { lep } => { write!(f, "IsBlockGiven {lep}") }, + Insn::FixnumBitCheck {val, index} => { write!(f, "FixnumBitCheck {val}, {index}") }, + Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => { + write!(f, "CCall {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::CCallWithFrame { cfunc, recv, args, name, blockiseq, .. } => { + write!(f, "CCallWithFrame {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + if let Some(blockiseq) = blockiseq { + write!(f, ", block={:p}", self.ptr_map.map_ptr(blockiseq))?; + } + Ok(()) + }, + Insn::CCallVariadic { cfunc, recv, args, name, .. } => { + write!(f, "CCallVariadic {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::IncrCounterPtr { .. } => write!(f, "IncrCounterPtr"), + Insn::Snapshot { state } => write!(f, "Snapshot {}", state.print(self.ptr_map)), + Insn::Defined { op_type, v, .. } => { + // op_type (enum defined_type) printing logic from iseq.c. + // Not sure why rb_iseq_defined_string() isn't exhaustive. + write!(f, "Defined ")?; + let op_type = *op_type as u32; + if op_type == DEFINED_FUNC { + write!(f, "func")?; + } else if op_type == DEFINED_REF { + write!(f, "ref")?; + } else if op_type == DEFINED_CONST_FROM { + write!(f, "constant-from")?; + } else { + write!(f, "{}", String::from_utf8_lossy(unsafe { rb_iseq_defined_string(op_type).as_rstring_byte_slice().unwrap() }))?; + }; + write!(f, ", {v}") + } + Insn::DefinedIvar { self_val, id, .. } => write!(f, "DefinedIvar {self_val}, :{}", id.contents_lossy()), + Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), + Insn::LoadPC => write!(f, "LoadPC"), + Insn::LoadEC => write!(f, "LoadEC"), + Insn::GetLEP => write!(f, "GetLEP"), + Insn::LoadSelf => write!(f, "LoadSelf"), + &Insn::LoadField { recv, id, offset, return_type: _ } => write!(f, "LoadField {recv}, :{}@{:p}", id.contents_lossy(), self.ptr_map.map_offset(offset)), + &Insn::StoreField { recv, id, offset, val } => write!(f, "StoreField {recv}, :{}@{:p}, {val}", id.contents_lossy(), self.ptr_map.map_offset(offset)), + &Insn::WriteBarrier { recv, val } => write!(f, "WriteBarrier {recv}, {val}"), + Insn::SetIvar { self_val, id, val, .. } => write!(f, "SetIvar {self_val}, :{}, {val}", id.contents_lossy()), + Insn::GetGlobal { id, .. } => write!(f, "GetGlobal :{}", id.contents_lossy()), + Insn::SetGlobal { id, val, .. } => write!(f, "SetGlobal :{}, {val}", id.contents_lossy()), + &Insn::GetLocal { level, ep_offset, use_sp: true, rest_param } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetLocal {name}l{level}, SP@{}{}", ep_offset + 1, if rest_param { ", *" } else { "" }) + }, + &Insn::GetLocal { level, ep_offset, use_sp: false, rest_param } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetLocal {name}l{level}, EP@{ep_offset}{}", if rest_param { ", *" } else { "" }) + }, + &Insn::SetLocal { val, level, ep_offset } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "SetLocal {name}l{level}, EP@{ep_offset}, {val}") + }, + Insn::GetSpecialSymbol { symbol_type, .. } => write!(f, "GetSpecialSymbol {symbol_type:?}"), + Insn::GetSpecialNumber { nth, .. } => write!(f, "GetSpecialNumber {nth}"), + Insn::GetClassVar { id, .. } => write!(f, "GetClassVar :{}", id.contents_lossy()), + Insn::SetClassVar { id, val, .. } => write!(f, "SetClassVar :{}, {val}", id.contents_lossy()), + Insn::ToArray { val, .. } => write!(f, "ToArray {val}"), + Insn::ToNewArray { val, .. } => write!(f, "ToNewArray {val}"), + Insn::ArrayExtend { left, right, .. } => write!(f, "ArrayExtend {left}, {right}"), + Insn::ArrayPush { array, val, .. } => write!(f, "ArrayPush {array}, {val}"), + Insn::ObjToString { val, .. } => { write!(f, "ObjToString {val}") }, + Insn::StringIntern { val, .. } => { write!(f, "StringIntern {val}") }, + Insn::AnyToString { val, str, .. } => { write!(f, "AnyToString {val}, str: {str}") }, + Insn::SideExit { reason, .. } => write!(f, "SideExit {reason}"), + Insn::PutSpecialObject { value_type } => write!(f, "PutSpecialObject {value_type}"), + Insn::Throw { throw_state, val, .. } => { + write!(f, "Throw ")?; + match throw_state & VM_THROW_STATE_MASK { + RUBY_TAG_NONE => write!(f, "TAG_NONE"), + RUBY_TAG_RETURN => write!(f, "TAG_RETURN"), + RUBY_TAG_BREAK => write!(f, "TAG_BREAK"), + RUBY_TAG_NEXT => write!(f, "TAG_NEXT"), + RUBY_TAG_RETRY => write!(f, "TAG_RETRY"), + RUBY_TAG_REDO => write!(f, "TAG_REDO"), + RUBY_TAG_RAISE => write!(f, "TAG_RAISE"), + RUBY_TAG_THROW => write!(f, "TAG_THROW"), + RUBY_TAG_FATAL => write!(f, "TAG_FATAL"), + tag => write!(f, "{tag}") + }?; + if throw_state & VM_THROW_NO_ESCAPE_FLAG != 0 { + write!(f, "|NO_ESCAPE")?; + } + write!(f, ", {val}") + } + Insn::IncrCounter(counter) => write!(f, "IncrCounter {counter:?}"), + Insn::CheckInterrupts { .. } => write!(f, "CheckInterrupts"), + Insn::IsA { val, class } => write!(f, "IsA {val}, {class}"), + } + } +} + +impl std::fmt::Display for Insn { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity(), None).fmt(f) + } +} + +/// An extended basic block in a [`Function`]. +#[derive(Default, Debug)] +pub struct Block { + /// The index of the first YARV instruction for the Block in the ISEQ + pub insn_idx: u32, + params: Vec<InsnId>, + insns: Vec<InsnId>, +} + +impl Block { + /// Return an iterator over params + pub fn params(&self) -> Iter<'_, InsnId> { + self.params.iter() + } + + /// Return an iterator over insns + pub fn insns(&self) -> Iter<'_, InsnId> { + self.insns.iter() + } +} + +/// Pretty printer for [`Function`]. +pub struct FunctionPrinter<'a> { + fun: &'a Function, + display_snapshot_and_tp_patchpoints: bool, + ptr_map: PtrPrintMap, +} + +impl<'a> FunctionPrinter<'a> { + pub fn without_snapshot(fun: &'a Function) -> Self { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + Self { fun, display_snapshot_and_tp_patchpoints: false, ptr_map } + } + + pub fn with_snapshot(fun: &'a Function) -> FunctionPrinter<'a> { + let mut printer = Self::without_snapshot(fun); + printer.display_snapshot_and_tp_patchpoints = true; + printer + } +} + +/// Pretty printer for [`Function`]. +pub struct FunctionGraphvizPrinter<'a> { + fun: &'a Function, + ptr_map: PtrPrintMap, +} + +impl<'a> FunctionGraphvizPrinter<'a> { + pub fn new(fun: &'a Function) -> Self { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + Self { fun, ptr_map } + } +} + +/// Union-Find (Disjoint-Set) is a data structure for managing disjoint sets that has an interface +/// of two operations: +/// +/// * find (what set is this item part of?) +/// * union (join these two sets) +/// +/// Union-Find identifies sets by their *representative*, which is some chosen element of the set. +/// This is implemented by structuring each set as its own graph component with the representative +/// pointing at nothing. For example: +/// +/// * A -> B -> C +/// * D -> E +/// +/// This represents two sets `C` and `E`, with three and two members, respectively. In this +/// example, `find(A)=C`, `find(C)=C`, `find(D)=E`, and so on. +/// +/// To union sets, call `make_equal_to` on any set element. That is, `make_equal_to(A, D)` and +/// `make_equal_to(B, E)` have the same result: the two sets are joined into the same graph +/// component. After this operation, calling `find` on any element will return `E`. +/// +/// This is a useful data structure in compilers because it allows in-place rewriting without +/// linking/unlinking instructions and without replacing all uses. When calling `make_equal_to` on +/// any instruction, all of its uses now implicitly point to the replacement. +/// +/// This does mean that pattern matching and analysis of the instruction graph must be careful to +/// call `find` whenever it is inspecting an instruction (or its operands). If not, this may result +/// in missing optimizations. +#[derive(Debug)] +struct UnionFind<T: Copy + Into<usize>> { + forwarded: Vec<Option<T>>, +} + +impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { + fn new() -> UnionFind<T> { + UnionFind { forwarded: vec![] } + } + + /// Private. Return the internal representation of the forwarding pointer for a given element. + fn at(&self, idx: T) -> Option<T> { + self.forwarded.get(idx.into()).copied().flatten() + } + + /// Private. Set the internal representation of the forwarding pointer for the given element + /// `idx`. Extend the internal vector if necessary. + fn set(&mut self, idx: T, value: T) { + if idx.into() >= self.forwarded.len() { + self.forwarded.resize(idx.into()+1, None); + } + self.forwarded[idx.into()] = Some(value); + } + + /// Find the set representative for `insn`. Perform path compression at the same time to speed + /// up further find operations. For example, before: + /// + /// `A -> B -> C` + /// + /// and after `find(A)`: + /// + /// ``` + /// A -> C + /// B ---^ + /// ``` + pub fn find(&mut self, insn: T) -> T { + let result = self.find_const(insn); + if result != insn { + // Path compression + self.set(insn, result); + } + result + } + + /// Find the set representative for `insn` without doing path compression. + fn find_const(&self, insn: T) -> T { + let mut result = insn; + loop { + match self.at(result) { + None => return result, + Some(insn) => result = insn, + } + } + } + + /// Union the two sets containing `insn` and `target` such that every element in `insn`s set is + /// now part of `target`'s. Neither argument must be the representative in its set. + pub fn make_equal_to(&mut self, insn: T, target: T) { + let found = self.find(insn); + self.set(found, target); + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ValidationError { + BlockHasNoTerminator(BlockId), + // The terminator and its actual position + TerminatorNotAtEnd(BlockId, InsnId, usize), + /// Expected length, actual length + MismatchedBlockArity(BlockId, usize, usize), + JumpTargetNotInRPO(BlockId), + // The offending instruction, and the operand + OperandNotDefined(BlockId, InsnId, InsnId), + /// The offending block and instruction + DuplicateInstruction(BlockId, InsnId), + /// The offending instruction, its operand, expected type string, actual type string + MismatchedOperandType(InsnId, InsnId, String, String), + MiscValidationError(InsnId, String), +} + +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, ci: *const rb_callinfo, send_insn: InsnId, args: &[InsnId]) -> bool { + let mut can_send = true; + let mut count_failure = |counter| { + can_send = false; + function.push_insn(block, Insn::IncrCounter(counter)); + }; + let params = unsafe { iseq.params() }; + + use Counter::*; + if 0 != params.flags.has_rest() { count_failure(complex_arg_pass_param_rest) } + if 0 != params.flags.has_post() { count_failure(complex_arg_pass_param_post) } + if 0 != params.flags.has_block() { count_failure(complex_arg_pass_param_block) } + if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } + + if 0 != params.flags.has_kwrest() { count_failure(complex_arg_pass_param_kwrest) } + + if !can_send { + function.set_dynamic_send_reason(send_insn, ComplexArgPass); + return false; + } + + // Because we exclude e.g. post parameters above, they are also excluded from the sum below. + let lead_num = params.lead_num; + let opt_num = params.opt_num; + let keyword = params.keyword; + let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; + // Minimum args: all required positional + all required keywords + let min_argc = lead_num + kw_req_num; + // Maximum args: all positional (required + optional) + all keywords (required + optional) + let max_argc = lead_num + opt_num + kw_total_num; + + can_send = c_int::try_from(args.len()) + .as_ref() + .map(|argc| (min_argc..=max_argc).contains(argc)) + .unwrap_or(false); + if !can_send { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false + } + + // asm.ccall() doesn't support 6+ args. Compute the final argc after keyword setup: + // final_argc = caller's positional args + callee's total keywords (all kw slots are filled). + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_positional = args.len() - caller_kw_count; + let final_argc = caller_positional + kw_total_num as usize; + if final_argc + 1 > C_ARG_OPNDS.len() { // +1 for self + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + can_send +} + +/// A [`Function`], which is analogous to a Ruby ISeq, is a control-flow graph of [`Block`]s +/// containing instructions. +#[derive(Debug)] +pub struct Function { + // ISEQ this function refers to + iseq: *const rb_iseq_t, + /// The types for the parameters of this function. They are copied to the type + /// of entry block params after infer_types() fills Empty to all insn_types. + param_types: Vec<Type>, + + insns: Vec<Insn>, + union_find: std::cell::RefCell<UnionFind<InsnId>>, + insn_types: Vec<Type>, + blocks: Vec<Block>, + /// Entry block for the interpreter + entry_block: BlockId, + /// Entry block for JIT-to-JIT calls. Length will be `opt_num+1`, for callers + /// fulfilling `(0..=opt_num)` optional parameters. + jit_entry_blocks: Vec<BlockId>, + profiles: Option<ProfileOracle>, +} + +/// The kind of a value an ISEQ returns +enum IseqReturn { + Value(VALUE), + LocalVariable(u32), + Receiver, + // Builtin descriptor and return type (if known) + InvokeLeafBuiltin(rb_builtin_function, Option<Type>), +} + +unsafe extern "C" { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<InsnId>, ci_flags: u32) -> Option<IseqReturn> { + // Expect only two instructions and one possible operand + // NOTE: If an ISEQ has an optional keyword parameter with a default value that requires + // computation, the ISEQ will always have more than two instructions and won't be inlined. + + // Get the first two instructions + let first_insn = iseq_opcode_at_idx(iseq, 0); + let second_insn = iseq_opcode_at_idx(iseq, insn_len(first_insn as usize)); + + // Extract the return value if known + if second_insn != YARVINSN_leave { + return None; + } + match first_insn { + YARVINSN_getlocal_WC_0 => { + // Accept only cases where only positional arguments are used by both the callee and the caller. + // Keyword arguments may be specified by the callee or the caller but not used. + if captured_opnd.is_some() + // Equivalent to `VM_CALL_ARGS_SIMPLE - VM_CALL_KWARG - has_block_iseq` + || ci_flags & ( + VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING + ) != 0 + { + return None; + } + + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + + // Only inline if the local is a parameter (not a method-defined local) as we are indexing args. + let param_size = unsafe { rb_get_iseq_body_param_size(iseq) } as usize; + if local_idx >= param_size { + return None; + } + + if unsafe { rb_simple_iseq_p(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx.try_into().unwrap())); + } + + // TODO(max): Support only_kwparam case where the local_idx is a positional parameter + + None + } + YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), + YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), + YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), + YARVINSN_putobject_INT2FIX_1_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(1))), + // We don't support invokeblock for now. Such ISEQs are likely not used by blocks anyway. + YARVINSN_putself if captured_opnd.is_none() => Some(IseqReturn::Receiver), + YARVINSN_opt_invokebuiltin_delegate_leave => { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let argc = bf.argc as usize; + if argc != 0 { return None; } + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + if !leaf { return None; } + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + Some(IseqReturn::InvokeLeafBuiltin(bf, return_type)) + } + _ => None, + } +} + +impl Function { + fn new(iseq: *const rb_iseq_t) -> Function { + Function { + iseq, + insns: vec![], + insn_types: vec![], + union_find: UnionFind::new().into(), + blocks: vec![Block::default()], + entry_block: BlockId(0), + jit_entry_blocks: vec![], + param_types: vec![], + profiles: None, + } + } + + pub fn iseq(&self) -> *const rb_iseq_t { + self.iseq + } + + // Add an instruction to the function without adding it to any block + fn new_insn(&mut self, insn: Insn) -> InsnId { + let id = InsnId(self.insns.len()); + if insn.has_output() { + self.insn_types.push(types::Any); + } else { + self.insn_types.push(types::Empty); + } + self.insns.push(insn); + id + } + + // Add an instruction to an SSA block + pub fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { + let is_param = matches!(insn, Insn::Param); + let id = self.new_insn(insn); + if is_param { + self.blocks[block.0].params.push(id); + } else { + self.blocks[block.0].insns.push(id); + } + id + } + + // Add an instruction to an SSA block + fn push_insn_id(&mut self, block: BlockId, insn_id: InsnId) -> InsnId { + self.blocks[block.0].insns.push(insn_id); + insn_id + } + + /// Return the number of instructions + pub fn num_insns(&self) -> usize { + self.insns.len() + } + + /// Return a FrameState at the given instruction index. + pub fn frame_state(&self, insn_id: InsnId) -> FrameState { + match self.find(insn_id) { + Insn::Snapshot { state } => state, + insn => panic!("Unexpected non-Snapshot {insn} when looking up FrameState"), + } + } + + fn new_block(&mut self, insn_idx: u32) -> BlockId { + let id = BlockId(self.blocks.len()); + let block = Block { + insn_idx, + .. Block::default() + }; + self.blocks.push(block); + id + } + + fn remove_block(&mut self, block_id: BlockId) { + if BlockId(self.blocks.len() - 1) != block_id { + panic!("Can only remove the last block"); + } + self.blocks.pop(); + } + + /// Return a reference to the Block at the given index. + pub fn block(&self, block_id: BlockId) -> &Block { + &self.blocks[block_id.0] + } + + /// Return a reference to the entry block. + pub fn entry_block(&self) -> &Block { + &self.blocks[self.entry_block.0] + } + + /// Return the number of blocks + pub fn num_blocks(&self) -> usize { + self.blocks.len() + } + + pub fn assume_single_ractor_mode(&mut self, block: BlockId, state: InsnId) -> bool { + if unsafe { rb_jit_multi_ractor_p() } { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::SingleRactorMode, state }); + true + } + } + + /// Assume that objects of a given class will have no singleton class. + /// Returns true if safe to assume so and emits a PatchPoint. + /// Returns false if we've already seen a singleton class for this class, + /// to avoid an invalidation loop. + pub fn assume_no_singleton_classes(&mut self, block: BlockId, klass: VALUE, state: InsnId) -> bool { + if !klass.instance_can_have_singleton_class() { + // This class can never have a singleton class, so no patchpoint needed. + return true; + } + if has_singleton_class_of(klass) { + // We've seen a singleton class for this klass. Disable the optimization + // to avoid an invalidation loop. + return false; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state }); + true + } + + /// Return a copy of the instruction where the instruction and its operands have been read from + /// the union-find table (to find the current most-optimized version of this instruction). See + /// [`UnionFind`] for more. + /// + /// This is _the_ function for reading [`Insn`]. Use frequently. Example: + /// + /// ```rust + /// match func.find(insn_id) { + /// IfTrue { val, target } if func.is_truthy(val) => { + /// let jump = self.new_insn(Insn::Jump(target)); + /// func.make_equal_to(insn_id, jump); + /// } + /// _ => {} + /// } + /// ``` + pub fn find(&self, insn_id: InsnId) -> Insn { + macro_rules! find { + ( $x:expr ) => { + { + // TODO(max): Figure out why borrow_mut().find() causes `already borrowed: + // BorrowMutError` + self.union_find.borrow().find_const($x) + } + }; + } + macro_rules! find_vec { + ( $x:expr ) => { + { + $x.iter().map(|arg| find!(*arg)).collect() + } + }; + } + macro_rules! find_branch_edge { + ( $edge:ident ) => { + { + BranchEdge { + target: $edge.target, + args: find_vec!($edge.args), + } + } + }; + } + let insn_id = find!(insn_id); + use Insn::*; + match &self.insns[insn_id.0] { + result@(Const {..} + | Param + | GetConstantPath {..} + | PatchPoint {..} + | PutSpecialObject {..} + | GetGlobal {..} + | GetLocal {..} + | SideExit {..} + | EntryPoint {..} + | LoadPC + | LoadEC + | GetLEP + | LoadSelf + | IncrCounterPtr {..} + | IncrCounter(_)) => result.clone(), + &Snapshot { state: FrameState { iseq, insn_idx, pc, ref stack, ref locals } } => + Snapshot { + state: FrameState { + iseq, + insn_idx, + pc, + stack: find_vec!(stack), + locals: find_vec!(locals), + } + }, + &Return { val } => Return { val: find!(val) }, + &FixnumBitCheck { val, index } => FixnumBitCheck { val: find!(val), index }, + &Throw { throw_state, val, state } => Throw { throw_state, val: find!(val), state }, + &StringCopy { val, chilled, state } => StringCopy { val: find!(val), chilled, state }, + &StringIntern { val, state } => StringIntern { val: find!(val), state: find!(state) }, + &StringConcat { ref strings, state } => StringConcat { strings: find_vec!(strings), state: find!(state) }, + &StringGetbyte { string, index } => StringGetbyte { string: find!(string), index: find!(index) }, + &StringSetbyteFixnum { string, index, value } => StringSetbyteFixnum { string: find!(string), index: find!(index), value: find!(value) }, + &StringAppend { recv, other, state } => StringAppend { recv: find!(recv), other: find!(other), state: find!(state) }, + &StringAppendCodepoint { recv, other, state } => StringAppendCodepoint { recv: find!(recv), other: find!(other), state: find!(state) }, + &ToRegexp { opt, ref values, state } => ToRegexp { opt, values: find_vec!(values), state }, + &Test { val } => Test { val: find!(val) }, + &IsNil { val } => IsNil { val: find!(val) }, + &IsMethodCfunc { val, cd, cfunc, state } => IsMethodCfunc { val: find!(val), cd, cfunc, state }, + &IsBitEqual { left, right } => IsBitEqual { left: find!(left), right: find!(right) }, + &IsBitNotEqual { left, right } => IsBitNotEqual { left: find!(left), right: find!(right) }, + &BoxBool { val } => BoxBool { val: find!(val) }, + &BoxFixnum { val, state } => BoxFixnum { val: find!(val), state: find!(state) }, + &UnboxFixnum { val } => UnboxFixnum { val: find!(val) }, + &FixnumAref { recv, index } => FixnumAref { recv: find!(recv), index: find!(index) }, + Jump(target) => Jump(find_branch_edge!(target)), + &IfTrue { val, ref target } => IfTrue { val: find!(val), target: find_branch_edge!(target) }, + &IfFalse { val, ref target } => IfFalse { val: find!(val), target: find_branch_edge!(target) }, + &RefineType { val, new_type } => RefineType { val: find!(val), new_type }, + &GuardType { val, guard_type, state } => GuardType { val: find!(val), guard_type, state }, + &GuardTypeNot { val, guard_type, state } => GuardTypeNot { val: find!(val), guard_type, state }, + &GuardBitEquals { val, expected, reason, state } => GuardBitEquals { val: find!(val), expected, reason, state }, + &GuardShape { val, shape, state } => GuardShape { val: find!(val), shape, state }, + &GuardBlockParamProxy { level, state } => GuardBlockParamProxy { level, state: find!(state) }, + &GuardNotFrozen { recv, state } => GuardNotFrozen { recv: find!(recv), state }, + &GuardNotShared { recv, state } => GuardNotShared { recv: find!(recv), state }, + &GuardGreaterEq { left, right, state } => GuardGreaterEq { left: find!(left), right: find!(right), state }, + &GuardLess { left, right, state } => GuardLess { left: find!(left), right: find!(right), state }, + &GuardSuperMethodEntry { lep, cme, state } => GuardSuperMethodEntry { lep: find!(lep), cme, state }, + &GetBlockHandler { lep } => GetBlockHandler { lep: find!(lep) }, + &IsBlockGiven { lep } => IsBlockGiven { lep: find!(lep) }, + &FixnumAdd { left, right, state } => FixnumAdd { left: find!(left), right: find!(right), state }, + &FixnumSub { left, right, state } => FixnumSub { left: find!(left), right: find!(right), state }, + &FixnumMult { left, right, state } => FixnumMult { left: find!(left), right: find!(right), state }, + &FixnumDiv { left, right, state } => FixnumDiv { left: find!(left), right: find!(right), state }, + &FixnumMod { left, right, state } => FixnumMod { left: find!(left), right: find!(right), state }, + &FixnumNeq { left, right } => FixnumNeq { left: find!(left), right: find!(right) }, + &FixnumEq { left, right } => FixnumEq { left: find!(left), right: find!(right) }, + &FixnumGt { left, right } => FixnumGt { left: find!(left), right: find!(right) }, + &FixnumGe { left, right } => FixnumGe { left: find!(left), right: find!(right) }, + &FixnumLt { left, right } => FixnumLt { left: find!(left), right: find!(right) }, + &FixnumLe { left, right } => FixnumLe { left: find!(left), right: find!(right) }, + &FixnumAnd { left, right } => FixnumAnd { left: find!(left), right: find!(right) }, + &FixnumOr { left, right } => FixnumOr { left: find!(left), right: find!(right) }, + &FixnumXor { left, right } => FixnumXor { left: find!(left), right: find!(right) }, + &FixnumLShift { left, right, state } => FixnumLShift { left: find!(left), right: find!(right), state }, + &FixnumRShift { left, right } => FixnumRShift { left: find!(left), right: find!(right) }, + &ObjToString { val, cd, state } => ObjToString { + val: find!(val), + cd, + state, + }, + &AnyToString { val, str, state } => AnyToString { + val: find!(val), + str: find!(str), + state, + }, + &SendWithoutBlock { recv, cd, ref args, state, reason } => SendWithoutBlock { + recv: find!(recv), + cd, + args: find_vec!(args), + state, + reason, + }, + &SendWithoutBlockDirect { recv, cd, cme, iseq, ref args, kw_bits, state } => SendWithoutBlockDirect { + recv: find!(recv), + cd, + cme, + iseq, + args: find_vec!(args), + kw_bits, + state, + }, + &Send { recv, cd, blockiseq, ref args, state, reason } => Send { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, + }, + &SendForward { recv, cd, blockiseq, ref args, state, reason } => SendForward { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, + }, + &InvokeSuper { recv, cd, blockiseq, ref args, state, reason } => InvokeSuper { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, + }, + &InvokeBlock { cd, ref args, state, reason } => InvokeBlock { + cd, + args: find_vec!(args), + state, + reason, + }, + &InvokeProc { recv, ref args, state, kw_splat } => InvokeProc { + recv: find!(recv), + args: find_vec!(args), + state: find!(state), + kw_splat, + }, + &InvokeBuiltin { bf, recv, ref args, state, leaf, return_type } => InvokeBuiltin { bf, recv: find!(recv), args: find_vec!(args), state, leaf, return_type }, + &ArrayDup { val, state } => ArrayDup { val: find!(val), state }, + &HashDup { val, state } => HashDup { val: find!(val), state }, + &HashAref { hash, key, state } => HashAref { hash: find!(hash), key: find!(key), state }, + &HashAset { hash, key, val, state } => HashAset { hash: find!(hash), key: find!(key), val: find!(val), state }, + &ObjectAlloc { val, state } => ObjectAlloc { val: find!(val), state }, + &ObjectAllocClass { class, state } => ObjectAllocClass { class, state: find!(state) }, + &CCall { cfunc, recv, ref args, name, return_type, elidable } => CCall { cfunc, recv: find!(recv), args: find_vec!(args), name, return_type, elidable }, + &CCallWithFrame { cd, cfunc, recv, ref args, cme, name, state, return_type, elidable, blockiseq } => CCallWithFrame { + cd, + cfunc, + recv: find!(recv), + args: find_vec!(args), + cme, + name, + state: find!(state), + return_type, + elidable, + blockiseq, + }, + &CCallVariadic { cfunc, recv, ref args, cme, name, state, return_type, elidable, blockiseq } => CCallVariadic { + cfunc, recv: find!(recv), args: find_vec!(args), cme, name, state, return_type, elidable, blockiseq + }, + &Defined { op_type, obj, pushval, v, state } => Defined { op_type, obj, pushval, v: find!(v), state: find!(state) }, + &DefinedIvar { self_val, pushval, id, state } => DefinedIvar { self_val: find!(self_val), pushval, id, state }, + &NewArray { ref elements, state } => NewArray { elements: find_vec!(elements), state: find!(state) }, + &NewHash { ref elements, state } => NewHash { elements: find_vec!(elements), state: find!(state) }, + &NewRange { low, high, flag, state } => NewRange { low: find!(low), high: find!(high), flag, state: find!(state) }, + &NewRangeFixnum { low, high, flag, state } => NewRangeFixnum { low: find!(low), high: find!(high), flag, state: find!(state) }, + &ArrayAref { array, index } => ArrayAref { array: find!(array), index: find!(index) }, + &ArrayAset { array, index, val } => ArrayAset { array: find!(array), index: find!(index), val: find!(val) }, + &ArrayPop { array, state } => ArrayPop { array: find!(array), state: find!(state) }, + &ArrayLength { array } => ArrayLength { array: find!(array) }, + &ArrayMax { ref elements, state } => ArrayMax { elements: find_vec!(elements), state: find!(state) }, + &ArrayInclude { ref elements, target, state } => ArrayInclude { elements: find_vec!(elements), target: find!(target), state: find!(state) }, + &ArrayPackBuffer { ref elements, fmt, buffer, state } => ArrayPackBuffer { elements: find_vec!(elements), fmt: find!(fmt), buffer: find!(buffer), state: find!(state) }, + &DupArrayInclude { ary, target, state } => DupArrayInclude { ary, target: find!(target), state: find!(state) }, + &ArrayHash { ref elements, state } => ArrayHash { elements: find_vec!(elements), state }, + &SetGlobal { id, val, state } => SetGlobal { id, val: find!(val), state }, + &GetIvar { self_val, id, ic, state } => GetIvar { self_val: find!(self_val), id, ic, state }, + &LoadField { recv, id, offset, return_type } => LoadField { recv: find!(recv), id, offset, return_type }, + &StoreField { recv, id, offset, val } => StoreField { recv: find!(recv), id, offset, val: find!(val) }, + &WriteBarrier { recv, val } => WriteBarrier { recv: find!(recv), val: find!(val) }, + &SetIvar { self_val, id, ic, val, state } => SetIvar { self_val: find!(self_val), id, ic, val: find!(val), state }, + &GetClassVar { id, ic, state } => GetClassVar { id, ic, state }, + &SetClassVar { id, val, ic, state } => SetClassVar { id, val: find!(val), ic, state }, + &SetLocal { val, ep_offset, level } => SetLocal { val: find!(val), ep_offset, level }, + &GetSpecialSymbol { symbol_type, state } => GetSpecialSymbol { symbol_type, state }, + &GetSpecialNumber { nth, state } => GetSpecialNumber { nth, state }, + &ToArray { val, state } => ToArray { val: find!(val), state }, + &ToNewArray { val, state } => ToNewArray { val: find!(val), state }, + &ArrayExtend { left, right, state } => ArrayExtend { left: find!(left), right: find!(right), state }, + &ArrayPush { array, val, state } => ArrayPush { array: find!(array), val: find!(val), state }, + &CheckInterrupts { state } => CheckInterrupts { state }, + &IsA { val, class } => IsA { val: find!(val), class: find!(class) }, + } + } + + /// Update DynamicSendReason for the instruction at insn_id + fn set_dynamic_send_reason(&mut self, insn_id: InsnId, dynamic_send_reason: SendFallbackReason) { + use Insn::*; + if get_option!(stats) || get_option!(dump_hir_opt).is_some() || cfg!(test) { + match self.insns.get_mut(insn_id.0).unwrap() { + Send { reason, .. } + | SendForward { reason, .. } + | SendWithoutBlock { reason, .. } + | InvokeSuper { reason, .. } + | InvokeBlock { reason, .. } + => *reason = dynamic_send_reason, + _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) + } + } + } + + /// Replace `insn` with the new instruction `replacement`, which will get appended to `insns`. + fn make_equal_to(&mut self, insn: InsnId, replacement: InsnId) { + assert!(self.insns[insn.0].has_output(), + "Don't use make_equal_to for instruction with no output"); + assert!(self.insns[replacement.0].has_output(), + "Can't replace instruction that has output with instruction that has no output"); + // Don't push it to the block + self.union_find.borrow_mut().make_equal_to(insn, replacement); + } + + pub fn type_of(&self, insn: InsnId) -> Type { + assert!(self.insns[insn.0].has_output()); + self.insn_types[self.union_find.borrow_mut().find(insn).0] + } + + /// Check if the type of `insn` is a subtype of `ty`. + pub fn is_a(&self, insn: InsnId, ty: Type) -> bool { + self.type_of(insn).is_subtype(ty) + } + + fn infer_type(&self, insn: InsnId) -> Type { + assert!(self.insns[insn.0].has_output()); + match &self.insns[insn.0] { + Insn::Param => unimplemented!("params should not be present in block.insns"), + Insn::SetGlobal { .. } | Insn::Jump(_) | Insn::EntryPoint { .. } + | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } | Insn::Throw { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } + | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => + panic!("Cannot infer type of instruction with no output: {}. See Insn::has_output().", self.insns[insn.0]), + Insn::Const { val: Const::Value(val) } => Type::from_value(*val), + Insn::Const { val: Const::CBool(val) } => Type::from_cbool(*val), + Insn::Const { val: Const::CInt8(val) } => Type::from_cint(types::CInt8, *val as i64), + Insn::Const { val: Const::CInt16(val) } => Type::from_cint(types::CInt16, *val as i64), + Insn::Const { val: Const::CInt32(val) } => Type::from_cint(types::CInt32, *val as i64), + Insn::Const { val: Const::CInt64(val) } => Type::from_cint(types::CInt64, *val), + Insn::Const { val: Const::CUInt8(val) } => Type::from_cint(types::CUInt8, *val as i64), + Insn::Const { val: Const::CUInt16(val) } => Type::from_cint(types::CUInt16, *val as i64), + Insn::Const { val: Const::CUInt32(val) } => Type::from_cint(types::CUInt32, *val as i64), + Insn::Const { val: Const::CShape(val) } => Type::from_cint(types::CShape, val.0 as i64), + Insn::Const { val: Const::CUInt64(val) } => Type::from_cint(types::CUInt64, *val as i64), + Insn::Const { val: Const::CPtr(val) } => Type::from_cptr(*val), + Insn::Const { val: Const::CDouble(val) } => Type::from_double(*val), + Insn::Test { val } if self.type_of(*val).is_known_falsy() => Type::from_cbool(false), + Insn::Test { val } if self.type_of(*val).is_known_truthy() => Type::from_cbool(true), + Insn::Test { .. } => types::CBool, + Insn::IsNil { val } if self.is_a(*val, types::NilClass) => Type::from_cbool(true), + Insn::IsNil { val } if !self.type_of(*val).could_be(types::NilClass) => Type::from_cbool(false), + Insn::IsNil { .. } => types::CBool, + Insn::IsMethodCfunc { .. } => types::CBool, + Insn::IsBitEqual { .. } => types::CBool, + Insn::IsBitNotEqual { .. } => types::CBool, + Insn::BoxBool { .. } => types::BoolExact, + Insn::BoxFixnum { .. } => types::Fixnum, + Insn::UnboxFixnum { val } => self + .type_of(*val) + .fixnum_value() + .map_or(types::CInt64, |fixnum| Type::from_cint(types::CInt64, fixnum)), + Insn::FixnumAref { .. } => types::Fixnum, + Insn::StringCopy { .. } => types::StringExact, + Insn::StringIntern { .. } => types::Symbol, + Insn::StringConcat { .. } => types::StringExact, + Insn::StringGetbyte { .. } => types::Fixnum, + Insn::StringSetbyteFixnum { .. } => types::Fixnum, + Insn::StringAppend { .. } => types::StringExact, + Insn::StringAppendCodepoint { .. } => types::StringExact, + Insn::ToRegexp { .. } => types::RegexpExact, + Insn::NewArray { .. } => types::ArrayExact, + Insn::ArrayDup { .. } => types::ArrayExact, + Insn::ArrayAref { .. } => types::BasicObject, + Insn::ArrayPop { .. } => types::BasicObject, + Insn::ArrayLength { .. } => types::CInt64, + Insn::HashAref { .. } => types::BasicObject, + Insn::NewHash { .. } => types::HashExact, + Insn::HashDup { .. } => types::HashExact, + Insn::NewRange { .. } => types::RangeExact, + Insn::NewRangeFixnum { .. } => types::RangeExact, + Insn::ObjectAlloc { .. } => types::HeapBasicObject, + Insn::ObjectAllocClass { class, .. } => Type::from_class(*class), + &Insn::CCallWithFrame { return_type, .. } => return_type, + Insn::CCall { return_type, .. } => *return_type, + &Insn::CCallVariadic { return_type, .. } => return_type, + Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), + Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), + Insn::GuardTypeNot { .. } => types::BasicObject, + Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), + Insn::GuardShape { val, .. } => self.type_of(*val), + Insn::GuardNotFrozen { recv, .. } | Insn::GuardNotShared { recv, .. } => self.type_of(*recv), + Insn::GuardLess { left, .. } => self.type_of(*left), + Insn::GuardGreaterEq { left, .. } => self.type_of(*left), + Insn::FixnumAdd { .. } => types::Fixnum, + Insn::FixnumSub { .. } => types::Fixnum, + Insn::FixnumMult { .. } => types::Fixnum, + Insn::FixnumDiv { .. } => types::Fixnum, + Insn::FixnumMod { .. } => types::Fixnum, + Insn::FixnumEq { .. } => types::BoolExact, + Insn::FixnumNeq { .. } => types::BoolExact, + Insn::FixnumLt { .. } => types::BoolExact, + Insn::FixnumLe { .. } => types::BoolExact, + Insn::FixnumGt { .. } => types::BoolExact, + Insn::FixnumGe { .. } => types::BoolExact, + Insn::FixnumAnd { .. } => types::Fixnum, + Insn::FixnumOr { .. } => types::Fixnum, + Insn::FixnumXor { .. } => types::Fixnum, + Insn::FixnumLShift { .. } => types::Fixnum, + Insn::FixnumRShift { .. } => types::Fixnum, + Insn::PutSpecialObject { .. } => types::BasicObject, + Insn::SendWithoutBlock { .. } => types::BasicObject, + Insn::SendWithoutBlockDirect { .. } => types::BasicObject, + Insn::Send { .. } => types::BasicObject, + Insn::SendForward { .. } => types::BasicObject, + Insn::InvokeSuper { .. } => types::BasicObject, + Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeProc { .. } => types::BasicObject, + Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), + Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::DefinedIvar { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::GetConstantPath { .. } => types::BasicObject, + Insn::IsBlockGiven { .. } => types::BoolExact, + Insn::FixnumBitCheck { .. } => types::BoolExact, + Insn::ArrayMax { .. } => types::BasicObject, + Insn::ArrayInclude { .. } => types::BoolExact, + Insn::ArrayPackBuffer { .. } => types::String, + Insn::DupArrayInclude { .. } => types::BoolExact, + Insn::ArrayHash { .. } => types::Fixnum, + Insn::GetGlobal { .. } => types::BasicObject, + Insn::GetIvar { .. } => types::BasicObject, + Insn::LoadPC => types::CPtr, + Insn::LoadEC => types::CPtr, + Insn::GetLEP => types::CPtr, + Insn::LoadSelf => types::BasicObject, + &Insn::LoadField { return_type, .. } => return_type, + Insn::GetSpecialSymbol { .. } => types::BasicObject, + Insn::GetSpecialNumber { .. } => types::BasicObject, + Insn::GetClassVar { .. } => types::BasicObject, + Insn::ToNewArray { .. } => types::ArrayExact, + Insn::ToArray { .. } => types::ArrayExact, + Insn::ObjToString { .. } => types::BasicObject, + Insn::AnyToString { .. } => types::String, + Insn::GetLocal { rest_param: true, .. } => types::ArrayExact, + Insn::GetLocal { .. } => types::BasicObject, + Insn::GetBlockHandler { .. } => types::RubyValue, + // The type of Snapshot doesn't really matter; it's never materialized. It's used only + // as a reference for FrameState, which we use to generate side-exit code. + Insn::Snapshot { .. } => types::Any, + Insn::IsA { .. } => types::BoolExact, + } + } + + /// Set self.param_types. They are copied to the param types of jit_entry_blocks. + fn set_param_types(&mut self) { + let iseq = self.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + self.param_types.push(types::BasicObject); // self + for local_idx in 0..param_size { + let param_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact // Rest parameters are always ArrayExact + } else { + types::BasicObject + }; + self.param_types.push(param_type); + } + } + + /// Copy self.param_types to the param types of jit_entry_blocks. + fn copy_param_types(&mut self) { + for jit_entry_block in self.jit_entry_blocks.iter() { + let entry_params = self.blocks[jit_entry_block.0].params.iter(); + let param_types = self.param_types.iter(); + assert!( + param_types.len() >= entry_params.len(), + "param types should be initialized before type inference", + ); + for (param, param_type) in std::iter::zip(entry_params, param_types) { + // We know that function parameters are BasicObject or some subclass + self.insn_types[param.0] = *param_type; + } + } + } + + fn infer_types(&mut self) { + // Reset all types + self.insn_types.fill(types::Empty); + + // Fill entry parameter types + self.copy_param_types(); + + let mut reachable = BlockSet::with_capacity(self.blocks.len()); + for entry_block in self.entry_blocks() { + reachable.insert(entry_block); + } + + // Walk the graph, computing types until fixpoint + let rpo = self.rpo(); + loop { + let mut changed = false; + for &block in &rpo { + if !reachable.get(block) { continue; } + for insn_id in &self.blocks[block.0].insns { + let insn_type = match self.find(*insn_id) { + Insn::IfTrue { val, target: BranchEdge { target, args } } => { + assert!(!self.type_of(val).bit_equal(types::Empty)); + if self.type_of(val).could_be(Type::from_cbool(true)) { + reachable.insert(target); + for (idx, arg) in args.iter().enumerate() { + let param = self.blocks[target.0].params[idx]; + self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + } + } + continue; + } + Insn::IfFalse { val, target: BranchEdge { target, args } } => { + assert!(!self.type_of(val).bit_equal(types::Empty)); + if self.type_of(val).could_be(Type::from_cbool(false)) { + reachable.insert(target); + for (idx, arg) in args.iter().enumerate() { + let param = self.blocks[target.0].params[idx]; + self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + } + } + continue; + } + Insn::Jump(BranchEdge { target, args }) => { + reachable.insert(target); + for (idx, arg) in args.iter().enumerate() { + let param = self.blocks[target.0].params[idx]; + self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + } + continue; + } + insn if insn.has_output() => self.infer_type(*insn_id), + _ => continue, + }; + if !self.type_of(*insn_id).bit_equal(insn_type) { + self.insn_types[insn_id.0] = insn_type; + changed = true; + } + } + } + if !changed { + break; + } + } + } + + fn chase_insn(&self, insn: InsnId) -> InsnId { + let id = self.union_find.borrow().find_const(insn); + match self.insns[id.0] { + Insn::GuardType { val, .. } + | Insn::GuardTypeNot { val, .. } + | Insn::GuardShape { val, .. } + | Insn::GuardBitEquals { val, .. } => self.chase_insn(val), + | Insn::RefineType { val, .. } => self.chase_insn(val), + _ => id, + } + } + + /// Return the profiled type of the HIR instruction at the given ISEQ instruction + /// index, if it is known to be monomorphic or skewed polymorphic. This historical type + /// record is not a guarantee and must be checked with a GuardType or similar. + fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: usize) -> Option<ProfiledType> { + match self.resolve_receiver_type_from_profile(insn, iseq_insn_idx) { + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => Some(profiled_type), + _ => None, + } + } + + /// Prepare arguments for a direct send, handling keyword argument reordering and default synthesis. + /// Returns the (state, processed_args, kw_bits) to use for the SendWithoutBlockDirect instruction, + /// or Err with the fallback reason if direct send isn't possible. + fn prepare_direct_send_args( + &mut self, + block: BlockId, + args: &[InsnId], + ci: *const rb_callinfo, + iseq: IseqPtr, + state: InsnId, + ) -> Result<(InsnId, Vec<InsnId>, u32), SendFallbackReason> { + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (processed_args, caller_argc, kw_bits) = self.setup_keyword_arguments(block, args, kwarg, iseq)?; + + // If args were reordered or synthesized, create a new snapshot with the updated stack + let send_state = if processed_args != args { + let new_state = self.frame_state(state).with_replaced_args(&processed_args, caller_argc); + self.push_insn(block, Insn::Snapshot { state: new_state }) + } else { + state + }; + + Ok((send_state, processed_args, kw_bits)) + } + + /// Reorder keyword arguments to match the callee's expected order, and synthesize + /// default values for any optional keywords not provided by the caller. + /// + /// The output always contains all of the callee's keyword arguments (required + optional), + /// so the returned vec may be larger than the input args. + /// + /// Returns Ok with (processed_args, caller_argc, kw_bits) if successful, or Err with the fallback reason if not. + /// - caller_argc: number of arguments the caller actually pushed (for stack calculations) + /// - kw_bits: bitmask indicating which optional keywords were NOT provided by the caller + /// (used by checkkeyword to determine if non-constant defaults need evaluation) + fn setup_keyword_arguments( + &mut self, + block: BlockId, + args: &[InsnId], + kwarg: *const rb_callinfo_kwarg, + iseq: IseqPtr, + ) -> Result<(Vec<InsnId>, usize, u32), SendFallbackReason> { + let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if callee_keyword.is_null() { + if !kwarg.is_null() { + // Caller is passing kwargs but callee doesn't expect them. + return Err(SendWithoutBlockDirectKeywordMismatch); + } + // Neither caller nor callee have keywords - nothing to do + return Ok((args.to_vec(), args.len(), 0)); + } + + // kwarg may be null if caller passes no keywords but callee has optional keywords + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let callee_kw_count = unsafe { (*callee_keyword).num } as usize; + + // When there are 31+ keywords, CRuby uses a hash instead of a fixnum bitmask + // for kw_bits. Fall back to VM dispatch for this rare case. + if callee_kw_count >= VM_KW_SPECIFIED_BITS_MAX as usize { + return Err(SendWithoutBlockDirectTooManyKeywords); + } + + let callee_kw_required = unsafe { (*callee_keyword).required_num } as usize; + let callee_kw_table = unsafe { (*callee_keyword).table }; + let default_values = unsafe { (*callee_keyword).default_values }; + + // Caller can't provide more keywords than callee expects (no **kwrest support yet). + if caller_kw_count > callee_kw_count { + return Err(SendWithoutBlockDirectKeywordCountMismatch); + } + + // The keyword arguments are the last arguments in the args vector. + let kw_args_start = args.len() - caller_kw_count; + + // Build a mapping from caller keywords to their positions. + let mut caller_kw_order: Vec<ID> = Vec::with_capacity(caller_kw_count); + if !kwarg.is_null() { + for i in 0..caller_kw_count { + let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; + let id = unsafe { rb_sym2id(sym) }; + caller_kw_order.push(id); + } + } + + // Verify all caller keywords are expected by callee (no unknown keywords). + // Without **kwrest, unexpected keywords should raise ArgumentError at runtime. + for &caller_id in &caller_kw_order { + let mut found = false; + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + if caller_id == expected_id { + found = true; + break; + } + } + if !found { + // Caller is passing an unknown keyword - this will raise ArgumentError. + // Fall back to VM dispatch to handle the error. + return Err(SendWithoutBlockDirectKeywordMismatch); + } + } + + // Reorder keyword arguments to match callee expectation. + // Track which optional keywords were not provided via kw_bits. + let mut kw_bits: u32 = 0; + let mut reordered_kw_args: Vec<InsnId> = Vec::with_capacity(callee_kw_count); + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + + // Find where this keyword is in the caller's order + let mut found = false; + for (j, &caller_id) in caller_kw_order.iter().enumerate() { + if caller_id == expected_id { + reordered_kw_args.push(args[kw_args_start + j]); + found = true; + break; + } + } + + if !found { + // Required keyword not provided by caller which will raise an ArgumentError. + if i < callee_kw_required { + return Err(SendWithoutBlockDirectMissingKeyword); + } + + // Optional keyword not provided - use default value + let default_idx = i - callee_kw_required; + let default_value = unsafe { *default_values.add(default_idx) }; + + if default_value == Qundef { + // Non-constant default (e.g., `def foo(a: compute())`). + // Set the bit so checkkeyword knows to evaluate the default at runtime. + // Push Qnil as a placeholder; the callee's checkkeyword will detect this + // and branch to evaluate the default expression. + kw_bits |= 1 << default_idx; + let nil_insn = self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + reordered_kw_args.push(nil_insn); + } else { + // Constant default value - use it directly + let const_insn = self.push_insn(block, Insn::Const { val: Const::Value(default_value) }); + reordered_kw_args.push(const_insn); + } + } + } + + // Replace the keyword arguments with the reordered ones. + // Keep track of the original caller argc for stack calculations. + let caller_argc = args.len(); + let mut processed_args = args[..kw_args_start].to_vec(); + processed_args.extend(reordered_kw_args); + Ok((processed_args, caller_argc, kw_bits)) + } + + /// Resolve the receiver type for method dispatch optimization. + /// + /// Takes the receiver's Type, receiver HIR instruction, and ISEQ instruction index. + /// First checks if the receiver's class is statically known, otherwise consults profile data. + /// + /// Returns: + /// - `StaticallyKnown` if the receiver's exact class is known at compile-time + /// - Result of [`Self::resolve_receiver_type_from_profile`] if we need to check profile data + fn resolve_receiver_type(&self, recv: InsnId, recv_type: Type, insn_idx: usize) -> ReceiverTypeResolution { + if let Some(class) = recv_type.runtime_exact_ruby_class() { + return ReceiverTypeResolution::StaticallyKnown { class }; + } + self.resolve_receiver_type_from_profile(recv, insn_idx) + } + + /// Resolve the receiver type for method dispatch optimization from profile data. + /// + /// Returns: + /// - `Monomorphic`/`SkewedPolymorphic` if we have usable profile data + /// - `Polymorphic` if the receiver has multiple types + /// - `Megamorphic`/`SkewedMegamorphic` if the receiver has too many types to optimize + /// (SkewedMegamorphic may be optimized in the future, but for now we don't) + /// - `NoProfile` if we have no type information + fn resolve_receiver_type_from_profile(&self, recv: InsnId, insn_idx: usize) -> ReceiverTypeResolution { + let Some(profiles) = self.profiles.as_ref() else { + return ReceiverTypeResolution::NoProfile; + }; + let Some(entries) = profiles.types.get(&insn_idx) else { + return ReceiverTypeResolution::NoProfile; + }; + let recv = self.chase_insn(recv); + + for (entry_insn, entry_type_summary) in entries { + if self.union_find.borrow().find_const(*entry_insn) == recv { + if entry_type_summary.is_monomorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::Monomorphic { profiled_type }; + } else if entry_type_summary.is_skewed_polymorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedPolymorphic { profiled_type }; + } else if entry_type_summary.is_skewed_megamorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedMegamorphic { profiled_type }; + } else if entry_type_summary.is_polymorphic() { + return ReceiverTypeResolution::Polymorphic; + } else if entry_type_summary.is_megamorphic() { + return ReceiverTypeResolution::Megamorphic; + } + } + } + + ReceiverTypeResolution::NoProfile + } + + pub fn assume_expected_cfunc(&mut self, block: BlockId, class: VALUE, method_id: ID, cfunc: *mut c_void, state: InsnId) -> bool { + let cme = unsafe { rb_callable_method_entry(class, method_id) }; + if cme.is_null() { return false; } + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { return false; } + if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc { + return false; + } + self.gen_patch_points_for_optimized_ccall(block, class, method_id, cme, state); + if !self.assume_no_singleton_classes(block, class, state) { + return false; + } + true + } + + pub fn likely_a(&self, val: InsnId, ty: Type, state: InsnId) -> bool { + if self.type_of(val).is_subtype(ty) { + return true; + } + let frame_state = self.frame_state(state); + let iseq_insn_idx = frame_state.insn_idx; + let Some(profiled_type) = self.profiled_type_of_at(val, iseq_insn_idx) else { + return false; + }; + Type::from_profiled_type(profiled_type).is_subtype(ty) + } + + pub fn coerce_to(&mut self, block: BlockId, val: InsnId, guard_type: Type, state: InsnId) -> InsnId { + if self.is_a(val, guard_type) { return val; } + self.push_insn(block, Insn::GuardType { val, guard_type, state }) + } + + fn count_complex_call_features(&mut self, block: BlockId, ci_flags: c_uint) { + use Counter::*; + if 0 != ci_flags & VM_CALL_ARGS_SPLAT { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_splat)); } + if 0 != ci_flags & VM_CALL_ARGS_BLOCKARG { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_blockarg)); } + if 0 != ci_flags & VM_CALL_KWARG { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_kwarg)); } + if 0 != ci_flags & VM_CALL_KW_SPLAT { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_kw_splat)); } + if 0 != ci_flags & VM_CALL_TAILCALL { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_tailcall)); } + if 0 != ci_flags & VM_CALL_SUPER { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_super)); } + if 0 != ci_flags & VM_CALL_ZSUPER { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_zsuper)); } + if 0 != ci_flags & VM_CALL_FORWARDING { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_forwarding)); } + } + + fn rewrite_if_frozen(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, klass: u32, bop: u32, state: InsnId) { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + // If the basic operation is already redefined, we cannot optimize it. + self.set_dynamic_send_reason(orig_insn_id, SendWithoutBlockBopRedefined); + self.push_insn_id(block, orig_insn_id); + return; + } + let self_type = self.type_of(self_val); + if let Some(obj) = self_type.ruby_object() { + if obj.is_frozen() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state }); + self.make_equal_to(orig_insn_id, self_val); + return; + } + } + self.push_insn_id(block, orig_insn_id); + } + + fn try_rewrite_freeze(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::ArrayExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::HashExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, HASH_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else { + self.push_insn_id(block, orig_insn_id); + } + } + + fn try_rewrite_uminus(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_UMINUS, state); + } else { + self.push_insn_id(block, orig_insn_id); + } + } + + fn is_metaclass(&self, object: VALUE) -> bool { + unsafe { + if RB_TYPE_P(object, RUBY_T_CLASS) && rb_zjit_singleton_class_p(object) { + let attached = rb_class_attached_object(object); + RB_TYPE_P(attached, RUBY_T_CLASS) || RB_TYPE_P(attached, RUBY_T_MODULE) + } else { + false + } + } + } + + /// Rewrite SendWithoutBlock opcodes into SendWithoutBlockDirect opcodes if we know the target + /// ISEQ statically. This removes run-time method lookups and opens the door for inlining. + /// Also try and inline constant caches, specialize object allocations, and more. + fn type_specialize(&mut self) { + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.is_empty() => + self.try_rewrite_freeze(block, insn_id, recv, state), + Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => + self.try_rewrite_uminus(block, insn_id, recv, state), + Insn::SendWithoutBlock { mut recv, cd, args, state, .. } => { + let frame_state = self.frame_state(state); + let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockMegamorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockPolymorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNoProfiles); + } + self.push_insn_id(block, insn_id); + continue; + } + }; + let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + + let flags = unsafe { rb_vm_ci_flag(ci) }; + + let mid = unsafe { vm_ci_mid(ci) }; + // Do method lookup + let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; + if cme.is_null() { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); + self.push_insn_id(block, insn_id); continue; + } + // Load an overloaded cme if applicable. See vm_search_cc(). + // It allows you to use a faster ISEQ if possible. + cme = unsafe { rb_check_overloaded_cme(cme, ci) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedNeedPermission); + self.push_insn_id(block, insn_id); continue; + } + } + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + + // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendWithoutBlockDirect`. + // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) handle their own argument constraints (e.g., kw_splat for Proc call). + if def_type != VM_METHOD_TYPE_OPTIMIZED && unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + + if def_type == VM_METHOD_TYPE_ISEQ { + // TODO(max): Allow non-iseq; cache cme + // Only specialize positional-positional calls + // TODO(max): Handle other kinds of parameter passing + let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state }); + self.make_equal_to(insn_id, send_direct); + } else if def_type == VM_METHOD_TYPE_BMETHOD { + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + // Target ISEQ bmethods. Can't handle for example, `define_method(:foo, &:foo)` + // which makes a `block_type_symbol` bmethod. + if proc_block.type_ != block_type_iseq { + self.set_dynamic_send_reason(insn_id, BmethodNonIseqProc); + self.push_insn_id(block, insn_id); continue; + } + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let iseq = unsafe { *capture.code.iseq.as_ref() }; + + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); continue; + } + // Can't pass a block to a block for now + assert!((unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) == 0, "SendWithoutBlock but has a block arg"); + + // Patch points: + // Check for "defined with an un-shareable Proc in a different Ractor" + if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { + // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state }); + self.make_equal_to(insn_id, send_direct); + } else if def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if self.is_metaclass(klass) && !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + let getivar = self.push_insn(block, Insn::GetIvar { self_val: recv, id, ic: std::ptr::null(), state }); + self.make_equal_to(insn_id, getivar); + } else if let (VM_METHOD_TYPE_ATTRSET, &[val]) = (def_type, args.as_slice()) { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if self.is_metaclass(klass) && !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state }); + self.make_equal_to(insn_id, val); + } else if def_type == VM_METHOD_TYPE_OPTIMIZED { + let opt_type: OptimizedMethodType = unsafe { get_cme_def_body_optimized_type(cme) }.into(); + match (opt_type, args.as_slice()) { + (OptimizedMethodType::Call, _) => { + if flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KWARG) != 0 { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let kw_splat = flags & VM_CALL_KW_SPLAT != 0; + let invoke_proc = self.push_insn(block, Insn::InvokeProc { recv, args: args.clone(), state, kw_splat }); + self.make_equal_to(insn_id, invoke_proc); + } + (OptimizedMethodType::StructAref, &[]) | (OptimizedMethodType::StructAset, &[_]) => { + if unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + let index: i32 = unsafe { get_cme_def_body_optimized_index(cme) } + .try_into() + .unwrap(); + // We are going to use an encoding that takes a 4-byte immediate which + // limits the offset to INT32_MAX. + { + let native_index = (index as i64) * (SIZEOF_VALUE as i64); + if native_index > (i32::MAX as i64) { + self.push_insn_id(block, insn_id); continue; + } + } + // Get the profiled type to check if the fields is embedded or heap allocated. + let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn_id(block, insn_id); continue; + }; + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + // All structs from the same Struct class should have the same + // length. So if our recv is embedded all runtime + // structs of the same class should be as well, and the same is + // true of the converse. + // + // No need for a GuardShape. + if let OptimizedMethodType::StructAset = opt_type { + // We know that all Struct are HeapObject, so no need to insert a GuardType(HeapObject). + recv = self.push_insn(block, Insn::GuardNotFrozen { recv, state }); + } + + let (target, offset) = if is_embedded { + let offset = RUBY_OFFSET_RSTRUCT_AS_ARY + (SIZEOF_VALUE_I32 * index); + (recv, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv, id: ID!(_as_heap), offset: RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * index; + (as_heap, offset) + }; + + let replacement = if let (OptimizedMethodType::StructAset, &[val]) = (opt_type, args.as_slice()) { + self.push_insn(block, Insn::StoreField { recv: target, id: mid, offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv, val }); + val + } else { // StructAref + self.push_insn(block, Insn::LoadField { recv: target, id: mid, offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + }, + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType::from(opt_type))); + self.push_insn_id(block, insn_id); continue; + }, + }; + } else { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::from(def_type))); + self.push_insn_id(block, insn_id); continue; + } + } + // This doesn't actually optimize Send yet, just replaces the fallback reason to be more precise. + // The actual optimization is done in reduce_send_to_ccall. + Insn::Send { recv, cd, state, .. } => { + let frame_state = self.frame_state(state); + let klass = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => class, + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => profiled_type.class(), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendMegamorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendPolymorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendNoProfiles); + } + self.push_insn_id(block, insn_id); + continue; + } + }; + let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + let mid = unsafe { vm_ci_mid(ci) }; + // Do method lookup + let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; + if cme.is_null() { + self.set_dynamic_send_reason(insn_id, SendNotOptimizedMethodType(MethodType::Null)); + self.push_insn_id(block, insn_id); continue; + } + // Load an overloaded cme if applicable. See vm_search_cc(). + // It allows you to use a faster ISEQ if possible. + cme = unsafe { rb_check_overloaded_cme(cme, ci) }; + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + self.set_dynamic_send_reason(insn_id, SendNotOptimizedMethodType(MethodType::from(def_type))); + self.push_insn_id(block, insn_id); continue; + } + Insn::GetConstantPath { ic, state, .. } => { + let idlist: *const ID = unsafe { (*ic).segments }; + let ice = unsafe { (*ic).entry }; + if ice.is_null() { + self.push_insn_id(block, insn_id); continue; + } + let cref_sensitive = !unsafe { (*ice).ic_cref }.is_null(); + if cref_sensitive || !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + // Invalidate output code on any constant writes associated with constants + // referenced after the PatchPoint. + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::StableConstantNames { idlist }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(unsafe { (*ice).value }) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjToString { val, cd, state, .. } => { + if self.is_a(val, types::String) { + // behaves differently from `SendWithoutBlock` with `mid:to_s` because ObjToString should not have a patch point for String to_s being redefined + self.make_equal_to(insn_id, val); continue; + } + + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(val, frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); continue + }; + + if recv_type.is_string() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_type.class() }, state }); + let guard = self.push_insn(block, Insn::GuardType { val, guard_type: types::String, state }); + // Infer type so AnyToString can fold off this + self.insn_types[guard.0] = self.infer_type(guard); + self.make_equal_to(insn_id, guard); + } else { + let recv = self.push_insn(block, Insn::GuardType { val, guard_type: Type::from_profiled_type(recv_type), state}); + let send_to_s = self.push_insn(block, Insn::SendWithoutBlock { recv, cd, args: vec![], state, reason: ObjToStringNotString }); + self.make_equal_to(insn_id, send_to_s); + } + } + Insn::AnyToString { str, .. } => { + if self.is_a(str, types::String) { + self.make_equal_to(insn_id, str); + } else { + self.push_insn_id(block, insn_id); + } + } + Insn::IsMethodCfunc { val, cd, cfunc, state } if self.type_of(val).ruby_object_known() => { + let class = self.type_of(val).ruby_object().unwrap(); + let cme = unsafe { rb_zjit_vm_search_method(self.iseq.into(), cd as *mut rb_call_data, class) }; + let is_expected_cfunc = unsafe { rb_zjit_cme_is_cfunc(cme, cfunc as *const c_void) }; + let method = unsafe { rb_vm_ci_mid((*cd).ci) }; + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: class, method, cme }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::CBool(is_expected_cfunc) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjectAlloc { val, state } => { + let val_type = self.type_of(val); + if !val_type.is_subtype(types::Class) { + self.push_insn_id(block, insn_id); continue; + } + let Some(class) = val_type.ruby_object() else { + self.push_insn_id(block, insn_id); continue; + }; + // See class_get_alloc_func in object.c; if the class isn't initialized, is + // a singleton class, or has a custom allocator, ObjectAlloc might raise an + // exception or run arbitrary code. + // + // We also need to check if the class is initialized or a singleton before + // trying to read the allocator, otherwise it might raise. + if !unsafe { rb_zjit_class_initialized_p(class) } { + self.push_insn_id(block, insn_id); continue; + } + if unsafe { rb_zjit_singleton_class_p(class) } { + self.push_insn_id(block, insn_id); continue; + } + if !class_has_leaf_allocator(class) { + // Custom, known unsafe, or NULL allocator; could run arbitrary code. + self.push_insn_id(block, insn_id); continue; + } + let replacement = self.push_insn(block, Insn::ObjectAllocClass { class, state }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::NewRange { low, high, flag, state } => { + let low_is_fix = self.is_a(low, types::Fixnum); + let high_is_fix = self.is_a(high, types::Fixnum); + + if low_is_fix || high_is_fix { + let low_fix = self.coerce_to(block, low, types::Fixnum, state); + let high_fix = self.coerce_to(block, high, types::Fixnum, state); + let replacement = self.push_insn(block, Insn::NewRangeFixnum { low: low_fix, high: high_fix, flag, state }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + } else { + self.push_insn_id(block, insn_id); + }; + } + Insn::InvokeSuper { recv, cd, blockiseq, args, state, .. } => { + // Don't handle calls with literal blocks (e.g., super { ... }) + if !blockiseq.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperCallWithBlock); + continue; + } + + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert!(flags & VM_CALL_FCALL != 0); + + // Reject calls with complex argument handling. + let complex_arg_types = VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_KWARG + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING; + + if (flags & complex_arg_types) != 0 { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperComplexArgsPass); + continue; + } + + let frame_state = self.frame_state(state); + + // Get the profiled CME from the current method. + let Some(profiles) = self.profiles.as_ref() else { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNoProfiles); + continue; + }; + + let Some(current_cme) = profiles.payload.profile.get_super_method_entry(frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); + + // The absence of the super CME could be due to a missing profile, but + // if we've made it this far the value would have been deleted, indicating + // that the call is at least polymorphic and possibly megamorphic. + self.set_dynamic_send_reason(insn_id, SuperPolymorphic); + continue; + }; + + // Get defined_class and method ID from the profiled CME. + let current_defined_class = unsafe { (*current_cme).defined_class }; + let mid = unsafe { get_def_original_id((*current_cme).def) }; + + // Compute superclass: RCLASS_SUPER(RCLASS_ORIGIN(defined_class)) + let superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; + if superclass.nil_p() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperClassNotFound); + continue; + } + + // Look up the super method. + let super_cme = unsafe { rb_callable_method_entry(superclass, mid) }; + if super_cme.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetNotFound); + continue; + } + + // Check if it's an ISEQ method; bail if it isn't. + let def_type = unsafe { get_cme_def_type(super_cme) }; + if def_type != VM_METHOD_TYPE_ISEQ { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::from(def_type))); + continue; + } + + // Check if the super method's parameters support direct send. + // If not, we can't do direct dispatch. + let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; + if !can_direct_send(self, block, super_iseq, ci, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetComplexArgsPass); + continue; + } + + // Add PatchPoint for method redefinition. + self.push_insn(block, Insn::PatchPoint { + invariant: Invariant::MethodRedefined { + klass: unsafe { (*super_cme).defined_class }, + method: mid, + cme: super_cme + }, + state + }); + + // Guard that we're calling `super` from the expected method context. + let lep = self.push_insn(block, Insn::GetLEP); + self.push_insn(block, Insn::GuardSuperMethodEntry { + lep, + cme: current_cme, + state + }); + + // Guard that no block is being passed (implicit or explicit). + let block_handler = self.push_insn(block, Insn::GetBlockHandler { lep }); + self.push_insn(block, Insn::GuardBitEquals { + val: block_handler, + expected: Const::Value(VALUE(VM_BLOCK_HANDLER_NONE as usize)), + reason: SideExitReason::UnhandledBlockArg, + state + }); + + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, super_iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + // Use SendWithoutBlockDirect with the super method's CME and ISEQ. + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { + recv, + cd, + cme: super_cme, + iseq: super_iseq, + args: processed_args, + kw_bits, + state: send_state + }); + self.make_equal_to(insn_id, send_direct); + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + self.infer_types(); + } + + fn inline(&mut self) { + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + // Reject block ISEQs to avoid autosplat and other block parameter complications. + Insn::SendWithoutBlockDirect { recv, iseq, cd, args, state, .. } => { + let call_info = unsafe { (*cd).ci }; + let ci_flags = unsafe { vm_ci_flag(call_info) }; + // .send call is not currently supported for builtins + if ci_flags & VM_CALL_OPT_SEND != 0 { + self.push_insn_id(block, insn_id); continue; + } + let Some(value) = iseq_get_return_value(iseq, None, ci_flags) else { + self.push_insn_id(block, insn_id); continue; + }; + match value { + IseqReturn::LocalVariable(idx) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + self.make_equal_to(insn_id, args[idx as usize]); + } + IseqReturn::Value(value) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(value) }); + self.make_equal_to(insn_id, replacement); + } + IseqReturn::Receiver => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + self.make_equal_to(insn_id, recv); + } + IseqReturn::InvokeLeafBuiltin(bf, return_type) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + let replacement = self.push_insn(block, Insn::InvokeBuiltin { + bf, + recv, + args: vec![recv], + state, + leaf: true, + return_type, + }); + self.make_equal_to(insn_id, replacement); + } + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + self.infer_types(); + } + + fn load_shape(&mut self, block: BlockId, recv: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { + recv, + id: ID!(_shape_id), + offset: unsafe { rb_shape_id_offset() } as i32, + return_type: types::CShape + }) + } + + fn guard_shape(&mut self, block: BlockId, val: InsnId, expected: ShapeId, state: InsnId) -> InsnId { + self.push_insn(block, Insn::GuardBitEquals { + val, + expected: Const::CShape(expected), + reason: SideExitReason::GuardShape(expected), + state + }) + } + + fn optimize_getivar(&mut self) { + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::GetIvar { self_val, id, ic: _, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + let mut ivar_index: u16 = 0; + let replacement = if ! unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + } else if !recv_type.flags().is_t_object() { + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + let ivar_index_insn: InsnId = self.push_insn(block, Insn::Const { val: Const::CUInt16(ivar_index as u16) }); + self.push_insn(block, Insn::CCall { + cfunc: rb_ivar_get_at_no_ractor_check as *const u8, + recv: self_val, + args: vec![ivar_index_insn], + name: ID!(rb_ivar_get_at_no_ractor_check), + return_type: types::BasicObject, + elidable: true }) + } else if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + self.push_insn(block, Insn::LoadField { recv: self_val, id, offset, return_type: types::BasicObject }) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: ID!(_as_heap), offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + self.push_insn(block, Insn::LoadField { recv: as_heap, id, offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::DefinedIvar { self_val, id, pushval, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_not_t_object)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + let mut ivar_index: u16 = 0; + let replacement = if unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + self.push_insn(block, Insn::Const { val: Const::Value(pushval) }) + } else { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::SetIvar { self_val, id, val, state, ic: _ } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_not_t_object)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_frozen() { + // Can't set ivars on frozen objects + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_frozen)); + self.push_insn_id(block, insn_id); continue; + } + let mut ivar_index: u16 = 0; + let mut next_shape_id = recv_type.shape(); + if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // Current shape does not contain this ivar; do a shape transition. + let current_shape_id = recv_type.shape(); + let class = recv_type.class(); + // We're only looking at T_OBJECT so ignore all of the imemo stuff. + assert!(recv_type.flags().is_t_object()); + next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(class, current_shape_id.0, id) }); + // If the VM ran out of shapes, or this class generated too many leaf, + // it may be de-optimized into OBJ_TOO_COMPLEX_SHAPE (hash-table). + let new_shape_too_complex = unsafe { rb_jit_shape_too_complex_p(next_shape_id.0) }; + // TODO(max): Is it OK to bail out here after making a shape transition? + if new_shape_too_complex { + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_new_shape_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) }; + assert!(ivar_result, "New shape must have the ivar index"); + let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) }; + let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) }; + // If the new shape has a different capacity, or is TOO_COMPLEX, we'll have to + // reallocate it. + let needs_extension = next_capacity != current_capacity; + if needs_extension { + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_new_shape_needs_extension)); + self.push_insn_id(block, insn_id); continue; + } + // Fall through to emitting the ivar write + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + // Current shape contains this ivar + let (ivar_storage, offset) = if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + (self_val, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: ID!(_as_heap), offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + (as_heap, offset) + }; + self.push_insn(block, Insn::StoreField { recv: ivar_storage, id, offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv: self_val, val }); + if next_shape_id != recv_type.shape() { + // Write the new shape ID + let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) }); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + self.push_insn(block, Insn::StoreField { recv: self_val, id: ID!(_shape_id), offset: shape_id_offset, val: shape_id }); + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + self.infer_types(); + } + + fn gen_patch_points_for_optimized_ccall(&mut self, block: BlockId, recv_class: VALUE, method_id: ID, cme: *const rb_callable_method_entry_struct, state: InsnId) { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme }, state }); + } + + /// Optimize SendWithoutBlock that land in a C method to a direct CCall without + /// runtime lookup. + fn optimize_c_calls(&mut self) { + if unsafe { rb_zjit_method_tracing_currently_enabled() } { + return; + } + + // Try to reduce a Send insn to a CCallWithFrame + fn reduce_send_to_ccall( + fun: &mut Function, + block: BlockId, + self_type: Type, + send: Insn, + send_insn_id: InsnId, + ) -> Result<(), ()> { + let Insn::Send { mut recv, cd, blockiseq, args, state, .. } = send else { + return Err(()); + }; + + let call_info = unsafe { (*cd).ci }; + let argc = unsafe { vm_ci_argc(call_info) }; + let method_id = unsafe { rb_vm_ci_mid(call_info) }; + + // If we have info about the class of the receiver + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type} => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), + }; + + // Do method lookup + let cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); + return Err(()); + } + + // Filter for C methods + // TODO(max): Handle VM_METHOD_TYPE_ALIAS + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { + return Err(()); + } + + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedNeedPermission); + return Err(()); + } + } + + // When seeing &block argument, fall back to dynamic dispatch for now + // TODO: Support block forwarding + if unspecializable_c_call_type(ci_flags) { + fun.count_complex_call_features(block, ci_flags); + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + let blockiseq = if blockiseq.is_null() { None } else { Some(blockiseq) }; + + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; + // Find the `argc` (arity) of the C method, which describes the parameters it expects + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + + match cfunc_argc { + 0.. => { + // (self, arg0, arg1, ..., argc) form + // + // Bail on argc mismatch + if argc != cfunc_argc as u32 { + return Err(()); + } + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Emit a call + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc, + recv, + args, + cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + blockiseq, + }); + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + // Variadic method + -1 => { + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args, + cme, + name: method_id, + state, + return_type: types::BasicObject, + elidable: false, + blockiseq + }); + + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + -2 => { + // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); + Err(()) + } + _ => unreachable!("unknown cfunc kind: argc={argc}") + } + } + + // Try to reduce a SendWithoutBlock insn to a CCall/CCallWithFrame + fn reduce_send_without_block_to_ccall( + fun: &mut Function, + block: BlockId, + self_type: Type, + send: Insn, + send_insn_id: InsnId, + ) -> Result<(), ()> { + let Insn::SendWithoutBlock { mut recv, cd, args, state, .. } = send else { + return Err(()); + }; + + let call_info = unsafe { (*cd).ci }; + let argc = unsafe { vm_ci_argc(call_info) }; + let method_id = unsafe { rb_vm_ci_mid(call_info) }; + + // If we have info about the class of the receiver + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), + }; + + // Do method lookup + let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); + return Err(()); + } + + // Filter for C methods + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + if def_type != VM_METHOD_TYPE_CFUNC { + return Err(()); + } + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedNeedPermission); + return Err(()); + } + } + + // Find the `argc` (arity) of the C method, which describes the parameters it expects + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + match cfunc_argc { + 0.. => { + // (self, arg0, arg1, ..., argc) form + // + // Bail on argc mismatch + if argc != cfunc_argc as u32 { + return Err(()); + } + + // Filter for simple call sites (i.e. no splats etc.) + if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { + fun.count_complex_call_features(block, ci_flags); + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + count_not_annotated_cfunc(fun, block, cme); + } + let props = props.unwrap_or_default(); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // No inlining; emit a call + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + if props.leaf && props.no_gc { + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + let ccall = fun.push_insn(block, Insn::CCall { cfunc, recv, args, name, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + } else { + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc, + recv, + args, + cme, + name, + state, + return_type, + elidable, + blockiseq: None, + }); + fun.make_equal_to(send_insn_id, ccall); + } + + return Ok(()); + } + // Variadic method + -1 => { + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + let ci_flags = unsafe { vm_ci_flag(call_info) }; + if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { + // TODO(alan): Add fun.count_complex_call_features() here without double + // counting splat + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } else { + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + count_not_annotated_cfunc(fun, block, cme); + } + let props = props.unwrap_or_default(); + + // Try inlining the cfunc into HIR + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // No inlining; emit a call + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + let return_type = props.return_type; + let elidable = props.elidable; + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc, + recv, + args, + cme, + name, + state, + return_type, + elidable, + blockiseq: None, + }); + + fun.make_equal_to(send_insn_id, ccall); + return Ok(()) + } + + // Fall through for complex cases (splat, kwargs, etc.) + } + -2 => { + // (self, args_ruby_array) parameter form + // Falling through for now + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockCfuncArrayVariadic); + } + _ => unreachable!("unknown cfunc kind: argc={argc}") + } + + Err(()) + } + + fn qualified_method_name(class: VALUE, method_id: ID) -> String { + let method_name = method_id.contents_lossy(); + // rb_zjit_singleton_class_p also checks if it's a class + if unsafe { rb_zjit_singleton_class_p(class) } { + let class_name = get_class_name(unsafe { rb_class_attached_object(class) }); + format!("{class_name}.{method_name}") + } else { + let class_name = get_class_name(class); + format!("{class_name}#{method_name}") + } + } + + fn count_not_inlined_cfunc(fun: &mut Function, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_inlined_cfunc_counter_pointers = ZJITState::get_not_inlined_cfunc_counter_pointers(); + let counter_ptr = not_inlined_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + fun.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + fn count_not_annotated_cfunc(fun: &mut Function, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_annotated_cfunc_counter_pointers = ZJITState::get_not_annotated_cfunc_counter_pointers(); + let counter_ptr = not_annotated_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + fun.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + let send = self.find(insn_id); + match send { + send @ Insn::SendWithoutBlock { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_without_block_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } + } + send @ Insn::Send { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } + } + Insn::InvokeBuiltin { bf, recv, args, state, .. } => { + let props = ZJITState::get_method_annotations().get_builtin_properties(&bf).unwrap_or_default(); + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + } + _ => {} + } + self.push_insn_id(block, insn_id); + } + } + self.infer_types(); + } + + /// Fold a binary operator on fixnums. + fn fold_fixnum_bop(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<i64>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .filter(|&n| n >= (RUBY_FIXNUM_MIN as i64) && n <= RUBY_FIXNUM_MAX as i64) + .map(|n| self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_isize(n as isize)) })) + .unwrap_or(insn_id) + } + + /// Fold a binary predicate on fixnums. + fn fold_fixnum_pred(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<bool>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .map(|b| if b { Qtrue } else { Qfalse }) + .map(|b| self.new_insn(Insn::Const { val: Const::Value(b) })) + .unwrap_or(insn_id) + } + + /// Use type information left by `infer_types` to fold away operations that can be evaluated at compile-time. + /// + /// It can fold fixnum math, truthiness tests, and branches with constant conditionals. + fn fold_constants(&mut self) { + // TODO(max): Determine if it's worth it for us to reflow types after each branch + // simplification. This means that we can have nice cascading optimizations if what used to + // be a union of two different basic block arguments now has a single value. + // + // This would require 1) fixpointing, 2) worklist, or 3) (slightly less powerful) calling a + // function-level infer_types after each pruned branch. + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + let mut new_insns = vec![]; + for insn_id in old_insns { + let replacement_id = match self.find(insn_id) { + Insn::GuardType { val, guard_type, .. } if self.is_a(val, guard_type) => { + self.make_equal_to(insn_id, val); + // Don't bother re-inferring the type of val; we already know it. + continue; + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::BasicObject) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const VALUE; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::CShape) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const u32; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::CShape(ShapeId(val)) }) + } + _ => insn_id, + } + } + Insn::GuardBitEquals { val, expected, .. } => { + let recv_type = self.type_of(val); + if recv_type.has_value(expected) { + continue; + } else { + insn_id + } + } + Insn::AnyToString { str, .. } if self.is_a(str, types::String) => { + self.make_equal_to(insn_id, str); + // Don't bother re-inferring the type of str; we already know it. + continue; + } + Insn::FixnumAdd { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_add(r), + _ => None, + }) + } + Insn::FixnumSub { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_sub(r), + _ => None, + }) + } + Insn::FixnumMult { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_mul(r), + (Some(0), _) | (_, Some(0)) => Some(0), + _ => None, + }) + } + Insn::FixnumEq { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l == r), + _ => None, + }) + } + Insn::FixnumNeq { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l != r), + _ => None, + }) + } + Insn::FixnumLt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l < r), + _ => None, + }) + } + Insn::FixnumLe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l <= r), + _ => None, + }) + } + Insn::FixnumGt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l > r), + _ => None, + }) + } + Insn::FixnumGe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l >= r), + _ => None, + }) + } + Insn::ArrayAref { array, index } + if self.type_of(array).ruby_object_known() + && self.type_of(index).is_subtype(types::CInt64) => { + let array_obj = self.type_of(array).ruby_object().unwrap(); + match (array_obj.is_frozen(), self.type_of(index).cint64_value()) { + (true, Some(index)) => { + let val = unsafe { rb_yarv_ary_entry_internal(array_obj, index) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::Test { val } if self.type_of(val).is_known_falsy() => { + self.new_insn(Insn::Const { val: Const::CBool(false) }) + } + Insn::Test { val } if self.type_of(val).is_known_truthy() => { + self.new_insn(Insn::Const { val: Const::CBool(true) }) + } + Insn::IfTrue { val, target } if self.is_a(val, Type::from_cbool(true)) => { + self.new_insn(Insn::Jump(target)) + } + Insn::IfFalse { val, target } if self.is_a(val, Type::from_cbool(false)) => { + self.new_insn(Insn::Jump(target)) + } + // If we know that the branch condition is never going to cause a branch, + // completely drop the branch from the block. + Insn::IfTrue { val, .. } if self.is_a(val, Type::from_cbool(false)) => continue, + Insn::IfFalse { val, .. } if self.is_a(val, Type::from_cbool(true)) => continue, + _ => insn_id, + }; + // If we're adding a new instruction, mark the two equivalent in the union-find and + // do an incremental flow typing of the new instruction. + if insn_id != replacement_id && self.insns[replacement_id.0].has_output() { + self.make_equal_to(insn_id, replacement_id); + self.insn_types[replacement_id.0] = self.infer_type(replacement_id); + } + new_insns.push(replacement_id); + // If we've just folded an IfTrue into a Jump, for example, don't bother copying + // over unreachable instructions afterward. + if self.insns[replacement_id.0].is_terminator() { + break; + } + } + self.blocks[block.0].insns = new_insns; + } + } + + fn worklist_traverse_single_insn(&self, insn: &Insn, worklist: &mut VecDeque<InsnId>) { + match insn { + &Insn::Const { .. } + | &Insn::Param + | &Insn::EntryPoint { .. } + | &Insn::LoadPC + | &Insn::LoadEC + | &Insn::GetLEP + | &Insn::LoadSelf + | &Insn::GetLocal { .. } + | &Insn::PutSpecialObject { .. } + | &Insn::IncrCounter(_) + | &Insn::IncrCounterPtr { .. } => + {} + &Insn::GetBlockHandler { lep } + | &Insn::IsBlockGiven { lep } => { + worklist.push_back(lep); + } + &Insn::PatchPoint { state, .. } + | &Insn::CheckInterrupts { state } + | &Insn::GetConstantPath { ic: _, state } => { + worklist.push_back(state); + } + &Insn::FixnumBitCheck { val, index: _ } => { + worklist.push_back(val) + } + &Insn::ArrayMax { ref elements, state } + | &Insn::ArrayHash { ref elements, state } + | &Insn::NewHash { ref elements, state } + | &Insn::NewArray { ref elements, state } => { + worklist.extend(elements); + worklist.push_back(state); + } + &Insn::ArrayInclude { ref elements, target, state } => { + worklist.extend(elements); + worklist.push_back(target); + worklist.push_back(state); + } + &Insn::ArrayPackBuffer { ref elements, fmt, buffer, state } => { + worklist.extend(elements); + worklist.push_back(fmt); + worklist.push_back(buffer); + worklist.push_back(state); + } + &Insn::DupArrayInclude { target, state, .. } => { + worklist.push_back(target); + worklist.push_back(state); + } + &Insn::NewRange { low, high, state, .. } + | &Insn::NewRangeFixnum { low, high, state, .. } => { + worklist.push_back(low); + worklist.push_back(high); + worklist.push_back(state); + } + &Insn::StringConcat { ref strings, state, .. } => { + worklist.extend(strings); + worklist.push_back(state); + } + &Insn::StringGetbyte { string, index } => { + worklist.push_back(string); + worklist.push_back(index); + } + &Insn::StringSetbyteFixnum { string, index, value } => { + worklist.push_back(string); + worklist.push_back(index); + worklist.push_back(value); + } + &Insn::StringAppend { recv, other, state } + | &Insn::StringAppendCodepoint { recv, other, state } + => { + worklist.push_back(recv); + worklist.push_back(other); + worklist.push_back(state); + } + &Insn::ToRegexp { ref values, state, .. } => { + worklist.extend(values); + worklist.push_back(state); + } + | &Insn::RefineType { val, .. } + | &Insn::Return { val } + | &Insn::Test { val } + | &Insn::SetLocal { val, .. } + | &Insn::BoxBool { val } + | &Insn::IsNil { val } => + worklist.push_back(val), + &Insn::SetGlobal { val, state, .. } + | &Insn::Defined { v: val, state, .. } + | &Insn::StringIntern { val, state } + | &Insn::StringCopy { val, state, .. } + | &Insn::ObjectAlloc { val, state } + | &Insn::GuardType { val, state, .. } + | &Insn::GuardTypeNot { val, state, .. } + | &Insn::GuardBitEquals { val, state, .. } + | &Insn::GuardShape { val, state, .. } + | &Insn::GuardNotFrozen { recv: val, state } + | &Insn::GuardNotShared { recv: val, state } + | &Insn::ToArray { val, state } + | &Insn::IsMethodCfunc { val, state, .. } + | &Insn::ToNewArray { val, state } + | &Insn::BoxFixnum { val, state } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::GuardGreaterEq { left, right, state } => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + &Insn::GuardLess { left, right, state } => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + Insn::Snapshot { state } => { + worklist.extend(&state.stack); + worklist.extend(&state.locals); + } + &Insn::FixnumAdd { left, right, state } + | &Insn::FixnumSub { left, right, state } + | &Insn::FixnumMult { left, right, state } + | &Insn::FixnumDiv { left, right, state } + | &Insn::FixnumMod { left, right, state } + | &Insn::ArrayExtend { left, right, state } + | &Insn::FixnumLShift { left, right, state } + => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + &Insn::FixnumLt { left, right } + | &Insn::FixnumLe { left, right } + | &Insn::FixnumGt { left, right } + | &Insn::FixnumGe { left, right } + | &Insn::FixnumEq { left, right } + | &Insn::FixnumNeq { left, right } + | &Insn::FixnumAnd { left, right } + | &Insn::FixnumOr { left, right } + | &Insn::FixnumXor { left, right } + | &Insn::FixnumRShift { left, right } + | &Insn::IsBitEqual { left, right } + | &Insn::IsBitNotEqual { left, right } + => { + worklist.push_back(left); + worklist.push_back(right); + } + &Insn::Jump(BranchEdge { ref args, .. }) => worklist.extend(args), + &Insn::IfTrue { val, target: BranchEdge { ref args, .. } } | &Insn::IfFalse { val, target: BranchEdge { ref args, .. } } => { + worklist.push_back(val); + worklist.extend(args); + } + &Insn::ArrayDup { val, state } + | &Insn::Throw { val, state, .. } + | &Insn::HashDup { val, state } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ArrayAref { array, index } => { + worklist.push_back(array); + worklist.push_back(index); + } + &Insn::ArrayAset { array, index, val } => { + worklist.push_back(array); + worklist.push_back(index); + worklist.push_back(val); + } + &Insn::ArrayPop { array, state } => { + worklist.push_back(array); + worklist.push_back(state); + } + &Insn::ArrayLength { array } => { + worklist.push_back(array); + } + &Insn::HashAref { hash, key, state } => { + worklist.push_back(hash); + worklist.push_back(key); + worklist.push_back(state); + } + &Insn::HashAset { hash, key, val, state } => { + worklist.push_back(hash); + worklist.push_back(key); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::Send { recv, ref args, state, .. } + | &Insn::SendForward { recv, ref args, state, .. } + | &Insn::SendWithoutBlock { recv, ref args, state, .. } + | &Insn::CCallVariadic { recv, ref args, state, .. } + | &Insn::CCallWithFrame { recv, ref args, state, .. } + | &Insn::SendWithoutBlockDirect { recv, ref args, state, .. } + | &Insn::InvokeBuiltin { recv, ref args, state, .. } + | &Insn::InvokeSuper { recv, ref args, state, .. } + | &Insn::InvokeProc { recv, ref args, state, .. } => { + worklist.push_back(recv); + worklist.extend(args); + worklist.push_back(state); + } + &Insn::InvokeBlock { ref args, state, .. } => { + worklist.extend(args); + worklist.push_back(state) + } + &Insn::CCall { recv, ref args, .. } => { + worklist.push_back(recv); + worklist.extend(args); + } + &Insn::GetIvar { self_val, state, .. } | &Insn::DefinedIvar { self_val, state, .. } => { + worklist.push_back(self_val); + worklist.push_back(state); + } + &Insn::SetIvar { self_val, val, state, .. } => { + worklist.push_back(self_val); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::GetClassVar { state, .. } => { + worklist.push_back(state); + } + &Insn::SetClassVar { val, state, .. } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ArrayPush { array, val, state } => { + worklist.push_back(array); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ObjToString { val, state, .. } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::AnyToString { val, str, state, .. } => { + worklist.push_back(val); + worklist.push_back(str); + worklist.push_back(state); + } + &Insn::LoadField { recv, .. } => { + worklist.push_back(recv); + } + &Insn::StoreField { recv, val, .. } + | &Insn::WriteBarrier { recv, val } => { + worklist.push_back(recv); + worklist.push_back(val); + } + &Insn::GuardBlockParamProxy { state, .. } | + &Insn::GetGlobal { state, .. } | + &Insn::GetSpecialSymbol { state, .. } | + &Insn::GetSpecialNumber { state, .. } | + &Insn::ObjectAllocClass { state, .. } | + &Insn::SideExit { state, .. } => worklist.push_back(state), + &Insn::GuardSuperMethodEntry { lep, state, .. } => { + worklist.push_back(lep); + worklist.push_back(state); + } + &Insn::UnboxFixnum { val } => worklist.push_back(val), + &Insn::FixnumAref { recv, index } => { + worklist.push_back(recv); + worklist.push_back(index); + } + &Insn::IsA { val, class } => { + worklist.push_back(val); + worklist.push_back(class); + } + } + } + + /// Remove instructions that do not have side effects and are not referenced by any other + /// instruction. + fn eliminate_dead_code(&mut self) { + let rpo = self.rpo(); + let mut worklist = VecDeque::new(); + // Find all of the instructions that have side effects, are control instructions, or are + // otherwise necessary to keep around + for block_id in &rpo { + for insn_id in &self.blocks[block_id.0].insns { + if !&self.insns[insn_id.0].is_elidable() { + worklist.push_back(*insn_id); + } + } + } + let mut necessary = InsnSet::with_capacity(self.insns.len()); + // Now recursively traverse their data dependencies and mark those as necessary + while let Some(insn_id) = worklist.pop_front() { + if necessary.get(insn_id) { continue; } + necessary.insert(insn_id); + self.worklist_traverse_single_insn(&self.find(insn_id), &mut worklist); + } + // Now remove all unnecessary instructions + for block_id in &rpo { + self.blocks[block_id.0].insns.retain(|&insn_id| necessary.get(insn_id)); + } + } + + fn absorb_dst_block(&mut self, num_in_edges: &[u32], block: BlockId) -> bool { + let Some(terminator_id) = self.blocks[block.0].insns.last() + else { return false }; + let Insn::Jump(BranchEdge { target, args }) = self.find(*terminator_id) + else { return false }; + if target == block { + // Can't absorb self + return false; + } + if num_in_edges[target.0] != 1 { + // Can't absorb block if it's the target of more than one branch + return false; + } + // Link up params with block args + let params = std::mem::take(&mut self.blocks[target.0].params); + assert_eq!(args.len(), params.len()); + for (arg, param) in args.iter().zip(params) { + self.make_equal_to(param, *arg); + } + // Remove branch instruction + self.blocks[block.0].insns.pop(); + // Move target instructions into block + let target_insns = std::mem::take(&mut self.blocks[target.0].insns); + self.blocks[block.0].insns.extend(target_insns); + true + } + + /// Clean up linked lists of blocks A -> B -> C into A (with B's and C's instructions). + fn clean_cfg(&mut self) { + // num_in_edges is invariant throughout cleaning the CFG: + // * we don't allocate new blocks + // * blocks that get absorbed are not in RPO anymore + // * blocks pointed to by blocks that get absorbed retain the same number of in-edges + let mut num_in_edges = vec![0; self.blocks.len()]; + for block in self.rpo() { + for &insn in &self.blocks[block.0].insns { + if let Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } | Insn::Jump(target) = self.find(insn) { + num_in_edges[target.target.0] += 1; + } + } + } + let mut changed = false; + loop { + let mut iter_changed = false; + for block in self.rpo() { + // Ignore transient empty blocks + if self.blocks[block.0].insns.is_empty() { continue; } + loop { + let absorbed = self.absorb_dst_block(&num_in_edges, block); + if !absorbed { break; } + iter_changed = true; + } + } + if !iter_changed { break; } + changed = true; + } + if changed { + self.infer_types(); + } + } + + /// Return a list that has entry_block and then jit_entry_blocks + fn entry_blocks(&self) -> Vec<BlockId> { + let mut entry_blocks = self.jit_entry_blocks.clone(); + entry_blocks.insert(0, self.entry_block); + entry_blocks + } + + pub fn is_entry_block(&self, block_id: BlockId) -> bool { + self.entry_block == block_id || self.jit_entry_blocks.contains(&block_id) + } + + /// Return a traversal of the `Function`'s `BlockId`s in reverse post-order. + pub fn rpo(&self) -> Vec<BlockId> { + let mut result = self.po_from(self.entry_blocks()); + result.reverse(); + result + } + + fn po_from(&self, starts: Vec<BlockId>) -> Vec<BlockId> { + #[derive(PartialEq)] + enum Action { + VisitEdges, + VisitSelf, + } + let mut result = vec![]; + let mut seen = BlockSet::with_capacity(self.blocks.len()); + let mut stack: Vec<_> = starts.iter().map(|&start| (start, Action::VisitEdges)).collect(); + while let Some((block, action)) = stack.pop() { + if action == Action::VisitSelf { + result.push(block); + continue; + } + if !seen.insert(block) { continue; } + stack.push((block, Action::VisitSelf)); + for insn_id in &self.blocks[block.0].insns { + let insn = self.find(*insn_id); + if let Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } | Insn::Jump(target) = insn { + stack.push((target.target, Action::VisitEdges)); + } + } + } + result + } + + fn assert_validates(&self) { + if let Err(err) = self.validate() { + eprintln!("Function failed validation."); + eprintln!("Err: {err:?}"); + eprintln!("{}", FunctionPrinter::with_snapshot(self)); + panic!("Aborting..."); + } + } + + /// Helper function to make an Iongraph JSON "instruction". + /// `uses`, `memInputs` and `attributes` are left empty for now, but may be populated + /// in the future. + fn make_iongraph_instr(id: InsnId, inputs: Vec<Json>, opcode: &str, ty: &str) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("opcode", opcode) + .insert("attributes", Json::empty_array()) + .insert("inputs", Json::Array(inputs)) + .insert("uses", Json::empty_array()) + .insert("memInputs", Json::empty_array()) + .insert("type", ty) + .build() + } + + /// Helper function to make an Iongraph JSON "block". + fn make_iongraph_block(id: BlockId, predecessors: Vec<BlockId>, successors: Vec<BlockId>, instructions: Vec<Json>, attributes: Vec<&str>, loop_depth: u32) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("loopDepth", loop_depth) + .insert("attributes", Json::array(attributes)) + .insert("predecessors", Json::array(predecessors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("successors", Json::array(successors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("instructions", Json::array(instructions)) + .build() + } + + /// Helper function to make an Iongraph JSON "function". + /// Note that `lir` is unpopulated right now as ZJIT doesn't use its functionality. + fn make_iongraph_function(pass_name: &str, hir_blocks: Vec<Json>) -> Json { + Json::object() + .insert("name", pass_name) + .insert("mir", Json::object() + .insert("blocks", Json::array(hir_blocks)) + .build() + ) + .insert("lir", Json::object() + .insert("blocks", Json::empty_array()) + .build() + ) + .build() + } + + /// Generate an iongraph JSON pass representation for this function. + pub fn to_iongraph_pass(&self, pass_name: &str) -> Json { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + + let mut hir_blocks = Vec::new(); + let cfi = ControlFlowInfo::new(self); + let dominators = Dominators::new(self); + let loop_info = LoopInfo::new(&cfi, &dominators); + + // Push each block from the iteration in reverse post order to `hir_blocks`. + for block_id in self.rpo() { + // Create the block with instructions. + let block = &self.blocks[block_id.0]; + let predecessors = cfi.predecessors(block_id).collect(); + let successors = cfi.successors(block_id).collect(); + let mut instructions = Vec::new(); + + // Process all instructions (parameters and body instructions). + // Parameters are currently guaranteed to be Parameter instructions, but in the future + // they might be refined to other instruction kinds by the optimizer. + for insn_id in block.params.iter().chain(block.insns.iter()) { + let insn_id = self.union_find.borrow().find_const(*insn_id); + let insn = self.find(insn_id); + + // Snapshots are not serialized, so skip them. + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + + // Instructions with no output or an empty type should have an empty type field. + let type_str = if insn.has_output() { + let insn_type = self.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + String::new() + } else { + insn_type.print(&ptr_map).to_string() + } + } else { + String::new() + }; + + + let opcode = insn.print(&ptr_map, Some(self.iseq)).to_string(); + + // Traverse the worklist to get inputs for a given instruction. + let mut inputs = VecDeque::new(); + self.worklist_traverse_single_insn(&insn, &mut inputs); + let inputs: Vec<Json> = inputs.into_iter().map(|x| x.0.into()).collect(); + + instructions.push( + Self::make_iongraph_instr( + insn_id, + inputs, + &opcode, + &type_str + ) + ); + } + + let mut attributes = vec![]; + if loop_info.is_back_edge_source(block_id) { + attributes.push("backedge"); + } + if loop_info.is_loop_header(block_id) { + attributes.push("loopheader"); + } + let loop_depth = loop_info.loop_depth(block_id); + + hir_blocks.push(Self::make_iongraph_block( + block_id, + predecessors, + successors, + instructions, + attributes, + loop_depth, + )); + } + + Self::make_iongraph_function(pass_name, hir_blocks) + } + + /// Run all the optimization passes we have. + pub fn optimize(&mut self) { + let mut passes: Vec<Json> = Vec::new(); + let should_dump = get_option!(dump_hir_iongraph); + + macro_rules! run_pass { + ($name:ident) => { + self.$name(); + #[cfg(debug_assertions)] self.assert_validates(); + if should_dump { + passes.push( + self.to_iongraph_pass(stringify!($name)) + ); + } + } + } + + if should_dump { + passes.push(self.to_iongraph_pass("unoptimized")); + } + + // Function is assumed to have types inferred already + run_pass!(type_specialize); + run_pass!(inline); + run_pass!(optimize_getivar); + run_pass!(optimize_c_calls); + run_pass!(fold_constants); + run_pass!(clean_cfg); + run_pass!(eliminate_dead_code); + + if should_dump { + let iseq_name = iseq_get_location(self.iseq, 0); + self.dump_iongraph(&iseq_name, passes); + } + } + + /// Dump HIR passed to codegen if specified by options. + pub fn dump_hir(&self) { + // Dump HIR after optimization + match get_option!(dump_hir_opt) { + Some(DumpHIR::WithoutSnapshot) => println!("Optimized HIR:\n{}", FunctionPrinter::without_snapshot(self)), + Some(DumpHIR::All) => println!("Optimized HIR:\n{}", FunctionPrinter::with_snapshot(self)), + Some(DumpHIR::Debug) => println!("Optimized HIR:\n{:#?}", &self), + None => {}, + } + + if let Some(filename) = &get_option!(dump_hir_graphviz) { + use std::fs::OpenOptions; + use std::io::Write; + let mut file = OpenOptions::new().append(true).open(filename).unwrap(); + writeln!(file, "{}", FunctionGraphvizPrinter::new(self)).unwrap(); + } + } + + pub fn dump_iongraph(&self, function_name: &str, passes: Vec<Json>) { + fn sanitize_for_filename(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == '-' { + c + } else { + '_' + } + }) + .collect() + } + + use std::io::Write; + let dir = format!("/tmp/zjit-iongraph-{}", std::process::id()); + std::fs::create_dir_all(&dir).expect("Unable to create directory."); + let sanitized = sanitize_for_filename(function_name); + let path = format!("{dir}/func_{sanitized}.json"); + let mut file = std::fs::File::create(path).unwrap(); + let json = Json::object() + .insert("name", function_name) + .insert("passes", passes) + .build(); + writeln!(file, "{json}").unwrap(); + } + + /// Validates the following: + /// 1. Basic block jump args match parameter arity. + /// 2. Every terminator must be in the last position. + /// 3. Every block must have a terminator. + fn validate_block_terminators_and_jumps(&self) -> Result<(), ValidationError> { + for block_id in self.rpo() { + let mut block_has_terminator = false; + let insns = &self.blocks[block_id.0].insns; + for (idx, insn_id) in insns.iter().enumerate() { + let insn = self.find(*insn_id); + match &insn { + Insn::Jump(BranchEdge{target, args}) + | Insn::IfTrue { val: _, target: BranchEdge{target, args} } + | Insn::IfFalse { val: _, target: BranchEdge{target, args}} => { + let target_block = &self.blocks[target.0]; + let target_len = target_block.params.len(); + let args_len = args.len(); + if target_len != args_len { + return Err(ValidationError::MismatchedBlockArity(block_id, target_len, args_len)) + } + } + _ => {} + } + if !insn.is_terminator() { + continue; + } + block_has_terminator = true; + if idx != insns.len() - 1 { + return Err(ValidationError::TerminatorNotAtEnd(block_id, *insn_id, idx)); + } + } + if !block_has_terminator { + return Err(ValidationError::BlockHasNoTerminator(block_id)); + } + } + Ok(()) + } + + // This performs a dataflow def-analysis over the entire CFG to detect any + // possibly undefined instruction operands. + fn validate_definite_assignment(&self) -> Result<(), ValidationError> { + // Map of block ID -> InsnSet + // Initialize with all missing values at first, to catch if a jump target points to a + // missing location. + let mut assigned_in = vec![None; self.num_blocks()]; + let rpo = self.rpo(); + // Begin with every block having every variable defined, except for the entry blocks, which + // start with nothing defined. + let entry_blocks = self.entry_blocks(); + for &block in &rpo { + if entry_blocks.contains(&block) { + assigned_in[block.0] = Some(InsnSet::with_capacity(self.insns.len())); + } else { + let mut all_ones = InsnSet::with_capacity(self.insns.len()); + all_ones.insert_all(); + assigned_in[block.0] = Some(all_ones); + } + } + let mut worklist = VecDeque::with_capacity(self.num_blocks()); + worklist.push_back(self.entry_block); + while let Some(block) = worklist.pop_front() { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + match self.find(insn_id) { + Insn::Jump(target) | Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } => { + let Some(block_in) = assigned_in[target.target.0].as_mut() else { + return Err(ValidationError::JumpTargetNotInRPO(target.target)); + }; + // jump target's block_in was modified, we need to queue the block for processing. + if block_in.intersect_with(&assigned) { + worklist.push_back(target.target); + } + } + insn if insn.has_output() => { + assigned.insert(insn_id); + } + _ => {} + } + } + } + // Check that each instruction's operands are assigned + for &block in &rpo { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + let mut operands = VecDeque::new(); + let insn = self.find(insn_id); + self.worklist_traverse_single_insn(&insn, &mut operands); + for operand in operands { + if !assigned.get(operand) { + return Err(ValidationError::OperandNotDefined(block, insn_id, operand)); + } + } + if insn.has_output() { + assigned.insert(insn_id); + } + } + } + Ok(()) + } + + /// Checks that each instruction('s representative) appears only once in the CFG. + fn validate_insn_uniqueness(&self) -> Result<(), ValidationError> { + let mut seen = InsnSet::with_capacity(self.insns.len()); + for block_id in self.rpo() { + for &insn_id in &self.blocks[block_id.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + if !seen.insert(insn_id) { + return Err(ValidationError::DuplicateInstruction(block_id, insn_id)); + } + } + } + Ok(()) + } + + fn assert_subtype(&self, user: InsnId, operand: InsnId, expected: Type) -> Result<(), ValidationError> { + let actual = self.type_of(operand); + if !actual.is_subtype(expected) { + return Err(ValidationError::MismatchedOperandType(user, operand, format!("{expected}"), format!("{actual}"))); + } + Ok(()) + } + + fn validate_insn_type(&self, insn_id: InsnId) -> Result<(), ValidationError> { + let insn_id = self.union_find.borrow().find_const(insn_id); + let insn = self.find(insn_id); + match insn { + // Instructions with no InsnId operands (except state) or nothing to assert + Insn::Const { .. } + | Insn::Param + | Insn::PutSpecialObject { .. } + | Insn::LoadField { .. } + | Insn::GetConstantPath { .. } + | Insn::IsBlockGiven { .. } + | Insn::GetGlobal { .. } + | Insn::LoadPC + | Insn::LoadEC + | Insn::GetLEP + | Insn::LoadSelf + | Insn::Snapshot { .. } + | Insn::Jump { .. } + | Insn::EntryPoint { .. } + | Insn::GuardBlockParamProxy { .. } + | Insn::GuardSuperMethodEntry { .. } + | Insn::GetBlockHandler { .. } + | Insn::PatchPoint { .. } + | Insn::SideExit { .. } + | Insn::IncrCounter { .. } + | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } + | Insn::GetClassVar { .. } + | Insn::GetSpecialNumber { .. } + | Insn::GetSpecialSymbol { .. } + | Insn::GetLocal { .. } + | Insn::StoreField { .. } => { + Ok(()) + } + // Instructions with 1 Ruby object operand + Insn::Test { val } + | Insn::IsNil { val } + | Insn::IsMethodCfunc { val, .. } + | Insn::GuardShape { val, .. } + | Insn::SetGlobal { val, .. } + | Insn::SetLocal { val, .. } + | Insn::SetClassVar { val, .. } + | Insn::Return { val } + | Insn::Throw { val, .. } + | Insn::ObjToString { val, .. } + | Insn::GuardType { val, .. } + | Insn::GuardTypeNot { val, .. } + | Insn::ToArray { val, .. } + | Insn::ToNewArray { val, .. } + | Insn::Defined { v: val, .. } + | Insn::ObjectAlloc { val, .. } + | Insn::DupArrayInclude { target: val, .. } + | Insn::GetIvar { self_val: val, .. } + | Insn::CCall { recv: val, .. } + | Insn::FixnumBitCheck { val, .. } // TODO (https://github.com/Shopify/ruby/issues/859) this should check Fixnum, but then test_checkkeyword_tests_fixnum_bit fails + | Insn::DefinedIvar { self_val: val, .. } => { + self.assert_subtype(insn_id, val, types::BasicObject) + } + Insn::GuardNotFrozen { recv, .. } | Insn::GuardNotShared { recv, .. } => { + self.assert_subtype(insn_id, recv, types::HeapBasicObject) + } + // Instructions with 2 Ruby object operands + Insn::SetIvar { self_val: left, val: right, .. } + | Insn::NewRange { low: left, high: right, .. } + | Insn::AnyToString { val: left, str: right, .. } + | Insn::WriteBarrier { recv: left, val: right } => { + self.assert_subtype(insn_id, left, types::BasicObject)?; + self.assert_subtype(insn_id, right, types::BasicObject) + } + // Instructions with recv and a Vec of Ruby objects + Insn::SendWithoutBlock { recv, ref args, .. } + | Insn::SendWithoutBlockDirect { recv, ref args, .. } + | Insn::Send { recv, ref args, .. } + | Insn::SendForward { recv, ref args, .. } + | Insn::InvokeSuper { recv, ref args, .. } + | Insn::CCallWithFrame { recv, ref args, .. } + | Insn::CCallVariadic { recv, ref args, .. } + | Insn::InvokeBuiltin { recv, ref args, .. } + | Insn::InvokeProc { recv, ref args, .. } + | Insn::ArrayInclude { target: recv, elements: ref args, .. } => { + self.assert_subtype(insn_id, recv, types::BasicObject)?; + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::ArrayPackBuffer { ref elements, fmt, buffer, .. } => { + self.assert_subtype(insn_id, fmt, types::BasicObject)?; + self.assert_subtype(insn_id, buffer, types::BasicObject)?; + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + // Instructions with a Vec of Ruby objects + Insn::InvokeBlock { ref args, .. } + | Insn::NewArray { elements: ref args, .. } + | Insn::ArrayHash { elements: ref args, .. } + | Insn::ArrayMax { elements: ref args, .. } => { + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::NewHash { ref elements, .. } => { + if elements.len() % 2 != 0 { + return Err(ValidationError::MiscValidationError(insn_id, "NewHash elements length is not even".to_string())); + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + Insn::StringConcat { ref strings, .. } + | Insn::ToRegexp { values: ref strings, .. } => { + for &string in strings { + self.assert_subtype(insn_id, string, types::String)?; + } + Ok(()) + } + // Instructions with String operands + Insn::StringCopy { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringIntern { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringAppend { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::String) + } + Insn::StringAppendCodepoint { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::Fixnum) + } + // Instructions with Array operands + Insn::ArrayDup { val, .. } => self.assert_subtype(insn_id, val, types::ArrayExact), + Insn::ArrayExtend { left, right, .. } => { + // TODO(max): Do left and right need to be ArrayExact? + self.assert_subtype(insn_id, left, types::Array)?; + self.assert_subtype(insn_id, right, types::Array) + } + Insn::ArrayPush { array, .. } + | Insn::ArrayPop { array, .. } + | Insn::ArrayLength { array, .. } => { + self.assert_subtype(insn_id, array, types::Array) + } + Insn::ArrayAref { array, index } => { + self.assert_subtype(insn_id, array, types::Array)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::ArrayAset { array, index, .. } => { + self.assert_subtype(insn_id, array, types::ArrayExact)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + // Instructions with Hash operands + Insn::HashAref { hash, .. } + | Insn::HashAset { hash, .. } => self.assert_subtype(insn_id, hash, types::HashExact), + Insn::HashDup { val, .. } => self.assert_subtype(insn_id, val, types::HashExact), + // Other + Insn::ObjectAllocClass { class, .. } => { + let has_leaf_allocator = unsafe { rb_zjit_class_has_default_allocator(class) } || class_has_leaf_allocator(class); + if !has_leaf_allocator { + return Err(ValidationError::MiscValidationError(insn_id, "ObjectAllocClass must have leaf allocator".to_string())); + } + Ok(()) + } + Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + if self.is_a(left, types::CInt) && self.is_a(right, types::CInt) { + // TODO(max): Check that int sizes match + Ok(()) + } else if self.is_a(left, types::CPtr) && self.is_a(right, types::CPtr) { + Ok(()) + } else if self.is_a(left, types::RubyValue) && self.is_a(right, types::RubyValue) { + Ok(()) + } else { + return Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())); + } + } + Insn::BoxBool { val } + | Insn::IfTrue { val, .. } + | Insn::IfFalse { val, .. } => { + self.assert_subtype(insn_id, val, types::CBool) + } + Insn::BoxFixnum { val, .. } => self.assert_subtype(insn_id, val, types::CInt64), + Insn::UnboxFixnum { val } => { + self.assert_subtype(insn_id, val, types::Fixnum) + } + Insn::FixnumAref { recv, index } => { + self.assert_subtype(insn_id, recv, types::Fixnum)?; + self.assert_subtype(insn_id, index, types::Fixnum) + } + Insn::FixnumAdd { left, right, .. } + | Insn::FixnumSub { left, right, .. } + | Insn::FixnumMult { left, right, .. } + | Insn::FixnumDiv { left, right, .. } + | Insn::FixnumMod { left, right, .. } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::NewRangeFixnum { low: left, high: right, .. } + => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum) + } + Insn::FixnumLShift { left, right, .. } + | Insn::FixnumRShift { left, right, .. } => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum)?; + let Some(obj) = self.type_of(right).fixnum_value() else { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<a compile-time constant>".into(), "<unknown>".into())); + }; + if obj < 0 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<positive>".into(), format!("{obj}"))); + } + if obj > 63 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<less than 64>".into(), format!("{obj}"))); + } + Ok(()) + } + Insn::GuardBitEquals { val, expected, .. } => { + match expected { + Const::Value(_) => self.assert_subtype(insn_id, val, types::RubyValue), + Const::CInt8(_) => self.assert_subtype(insn_id, val, types::CInt8), + Const::CInt16(_) => self.assert_subtype(insn_id, val, types::CInt16), + Const::CInt32(_) => self.assert_subtype(insn_id, val, types::CInt32), + Const::CInt64(_) => self.assert_subtype(insn_id, val, types::CInt64), + Const::CUInt8(_) => self.assert_subtype(insn_id, val, types::CUInt8), + Const::CUInt16(_) => self.assert_subtype(insn_id, val, types::CUInt16), + Const::CUInt32(_) => self.assert_subtype(insn_id, val, types::CUInt32), + Const::CShape(_) => self.assert_subtype(insn_id, val, types::CShape), + Const::CUInt64(_) => self.assert_subtype(insn_id, val, types::CUInt64), + Const::CBool(_) => self.assert_subtype(insn_id, val, types::CBool), + Const::CDouble(_) => self.assert_subtype(insn_id, val, types::CDouble), + Const::CPtr(_) => self.assert_subtype(insn_id, val, types::CPtr), + } + } + Insn::GuardLess { left, right, .. } + | Insn::GuardGreaterEq { left, right, .. } => { + self.assert_subtype(insn_id, left, types::CInt64)?; + self.assert_subtype(insn_id, right, types::CInt64) + }, + Insn::StringGetbyte { string, index } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::CInt64) + }, + Insn::StringSetbyteFixnum { string, index, value } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::Fixnum)?; + self.assert_subtype(insn_id, value, types::Fixnum) + } + Insn::IsA { val, class } => { + self.assert_subtype(insn_id, val, types::BasicObject)?; + self.assert_subtype(insn_id, class, types::Class) + } + Insn::RefineType { .. } => Ok(()), + } + } + + /// Check that insn types match the expected types for each instruction. + fn validate_types(&self) -> Result<(), ValidationError> { + for block_id in self.rpo() { + for &insn_id in &self.blocks[block_id.0].insns { + self.validate_insn_type(insn_id)?; + } + } + Ok(()) + } + + /// Run all validation passes we have. + pub fn validate(&self) -> Result<(), ValidationError> { + self.validate_block_terminators_and_jumps()?; + self.validate_definite_assignment()?; + self.validate_insn_uniqueness()?; + self.validate_types()?; + Ok(()) + } +} + +impl<'a> std::fmt::Display for FunctionPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let fun = &self.fun; + // In tests, there may not be an iseq to get location from. + let iseq_name = if fun.iseq.is_null() { + String::from("<manual>") + } else { + iseq_get_location(fun.iseq, 0) + }; + + // In tests, strip the line number for builtin ISEQs to make tests stable across line changes + let iseq_name = if cfg!(test) && iseq_name.contains("@<internal:") { + iseq_name[..iseq_name.rfind(':').unwrap()].to_string() + } else { + iseq_name + }; + writeln!(f, "fn {iseq_name}:")?; + for block_id in fun.rpo() { + write!(f, "{block_id}(")?; + if !fun.blocks[block_id.0].params.is_empty() { + let mut sep = ""; + for param in &fun.blocks[block_id.0].params { + write!(f, "{sep}{param}")?; + let insn_type = fun.type_of(*param); + if !insn_type.is_subtype(types::Empty) { + write!(f, ":{}", insn_type.print(&self.ptr_map))?; + } + sep = ", "; + } + } + writeln!(f, "):")?; + for insn_id in &fun.blocks[block_id.0].insns { + let insn = fun.find(*insn_id); + if !self.display_snapshot_and_tp_patchpoints && + matches!(insn, Insn::Snapshot {..} | Insn::PatchPoint { invariant: Invariant::NoTracePoint, .. }) { + continue; + } + write!(f, " ")?; + if insn.has_output() { + let insn_type = fun.type_of(*insn_id); + if insn_type.is_subtype(types::Empty) { + write!(f, "{insn_id} = ")?; + } else { + write!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; + } + } + writeln!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; + } + } + Ok(()) + } +} + +struct HtmlEncoder<'a, 'b> { + formatter: &'a mut std::fmt::Formatter<'b>, +} + +impl<'a, 'b> std::fmt::Write for HtmlEncoder<'a, 'b> { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + for ch in s.chars() { + match ch { + '<' => self.formatter.write_str("<")?, + '>' => self.formatter.write_str(">")?, + '&' => self.formatter.write_str("&")?, + '"' => self.formatter.write_str(""")?, + '\'' => self.formatter.write_str("'")?, + _ => self.formatter.write_char(ch)?, + } + } + Ok(()) + } +} + +impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + macro_rules! write_encoded { + ($f:ident, $($arg:tt)*) => { + HtmlEncoder { formatter: $f }.write_fmt(format_args!($($arg)*)) + }; + } + use std::fmt::Write; + let fun = &self.fun; + let iseq_name = iseq_get_location(fun.iseq, 0); + write!(f, "digraph G {{ # ")?; + write_encoded!(f, "{iseq_name}")?; + writeln!(f)?; + writeln!(f, "node [shape=plaintext];")?; + writeln!(f, "mode=hier; overlap=false; splines=true;")?; + for block_id in fun.rpo() { + writeln!(f, r#" {block_id} [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">"#)?; + write!(f, r#"<TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">{block_id}("#)?; + if !fun.blocks[block_id.0].params.is_empty() { + let mut sep = ""; + for param in &fun.blocks[block_id.0].params { + write_encoded!(f, "{sep}{param}")?; + let insn_type = fun.type_of(*param); + if !insn_type.is_subtype(types::Empty) { + write_encoded!(f, ":{}", insn_type.print(&self.ptr_map))?; + } + sep = ", "; + } + } + let mut edges = vec![]; + writeln!(f, ") </TD></TR>")?; + for insn_id in &fun.blocks[block_id.0].insns { + let insn_id = fun.union_find.borrow().find_const(*insn_id); + let insn = fun.find(insn_id); + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + write!(f, r#"<TR><TD ALIGN="left" PORT="{insn_id}">"#)?; + if insn.has_output() { + let insn_type = fun.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + write_encoded!(f, "{insn_id} = ")?; + } else { + write_encoded!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; + } + } + if let Insn::Jump(ref target) | Insn::IfTrue { ref target, .. } | Insn::IfFalse { ref target, .. } = insn { + edges.push((insn_id, target.target)); + } + write_encoded!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; + writeln!(f, " </TD></TR>")?; + } + writeln!(f, "</TABLE>>];")?; + for (src, dst) in edges { + writeln!(f, " {block_id}:{src} -> {dst}:params:n;")?; + } + } + writeln!(f, "}}") + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FrameState { + iseq: IseqPtr, + insn_idx: usize, + // Ruby bytecode instruction pointer + pub pc: *const VALUE, + + stack: Vec<InsnId>, + locals: Vec<InsnId>, +} + +impl FrameState { + /// Return itself without locals. Useful for side-exiting without spilling locals. + fn without_locals(&self) -> Self { + let mut state = self.clone(); + state.locals.clear(); + state + } + + /// Return itself without stack. Used by leaf calls with GC to reset SP to the base pointer. + pub fn without_stack(&self) -> Self { + let mut state = self.clone(); + state.stack.clear(); + state + } + + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. + /// `original_argc` is the number of args originally on the stack (before processing). + fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { + let mut state = self.clone(); + let args_start = state.stack.len() - original_argc; + state.stack.truncate(args_start); + state.stack.extend_from_slice(new_args); + state + } + + fn replace(&mut self, old: InsnId, new: InsnId) { + for slot in &mut self.stack { + if *slot == old { + *slot = new; + } + } + for slot in &mut self.locals { + if *slot == old { + *slot = new; + } + } + } +} + +/// Print adaptor for [`FrameState`]. See [`PtrPrintMap`]. +pub struct FrameStatePrinter<'a> { + inner: &'a FrameState, + ptr_map: &'a PtrPrintMap, +} + +/// Compute the index of a local variable from its slot index +fn ep_offset_to_local_idx(iseq: IseqPtr, ep_offset: u32) -> usize { + // Layout illustration + // This is an array of VALUE + // | VM_ENV_DATA_SIZE | + // v v + // low addr <+-------+-------+-------+-------+------------------+ + // |local 0|local 1| ... |local n| .... | + // +-------+-------+-------+-------+------------------+ + // ^ ^ ^ ^ + // +-------+---local_table_size----+ cfp->ep--+ + // | | + // +------------------ep_offset---------------+ + // + // See usages of local_var_name() from iseq.c for similar calculation. + + // Equivalent of iseq->body->local_table_size + let local_table_size: i32 = unsafe { get_iseq_body_local_table_size(iseq) } + .try_into() + .unwrap(); + let op = (ep_offset - VM_ENV_DATA_SIZE) as i32; + let local_idx = local_table_size - op - 1; + assert!(local_idx >= 0 && local_idx < local_table_size); + local_idx.try_into().unwrap() +} + +impl FrameState { + fn new(iseq: IseqPtr) -> FrameState { + FrameState { iseq, pc: std::ptr::null::<VALUE>(), insn_idx: 0, stack: vec![], locals: vec![] } + } + + /// Get the number of stack operands + pub fn stack_size(&self) -> usize { + self.stack.len() + } + + /// Iterate over all stack slots + pub fn stack(&self) -> Iter<'_, InsnId> { + self.stack.iter() + } + + /// Iterate over all local variables + pub fn locals(&self) -> Iter<'_, InsnId> { + self.locals.iter() + } + + /// Push a stack operand + fn stack_push(&mut self, opnd: InsnId) { + self.stack.push(opnd); + } + + /// Pop a stack operand + fn stack_pop(&mut self) -> Result<InsnId, ParseError> { + self.stack.pop().ok_or_else(|| ParseError::StackUnderflow(self.clone())) + } + + fn stack_pop_n(&mut self, count: usize) -> Result<Vec<InsnId>, ParseError> { + // Check if we have enough values on the stack + let stack_len = self.stack.len(); + if stack_len < count { + return Err(ParseError::StackUnderflow(self.clone())); + } + + Ok(self.stack.split_off(stack_len - count)) + } + + /// Get a stack-top operand + fn stack_top(&self) -> Result<InsnId, ParseError> { + self.stack.last().ok_or_else(|| ParseError::StackUnderflow(self.clone())).copied() + } + + /// Set a stack operand at idx + fn stack_setn(&mut self, idx: usize, opnd: InsnId) { + let idx = self.stack.len() - idx - 1; + self.stack[idx] = opnd; + } + + /// Get a stack operand at idx + fn stack_topn(&self, idx: usize) -> Result<InsnId, ParseError> { + let idx = self.stack.len() - idx - 1; + self.stack.get(idx).ok_or_else(|| ParseError::StackUnderflow(self.clone())).copied() + } + + fn setlocal(&mut self, ep_offset: u32, opnd: InsnId) { + let idx = ep_offset_to_local_idx(self.iseq, ep_offset); + self.locals[idx] = opnd; + } + + fn getlocal(&mut self, ep_offset: u32) -> InsnId { + let idx = ep_offset_to_local_idx(self.iseq, ep_offset); + self.locals[idx] + } + + fn as_args(&self, self_param: InsnId) -> Vec<InsnId> { + // We're currently passing around the self parameter as a basic block + // argument because the register allocator uses a fixed register based + // on the basic block argument index, which would cause a conflict if + // we reuse an argument from another basic block. + // TODO: Modify the register allocator to allow reusing an argument + // of another basic block. + let mut args = vec![self_param]; + args.extend(self.locals.iter().chain(self.stack.iter()).copied()); + args + } + + /// Get the opcode for the current instruction + pub fn get_opcode(&self) -> i32 { + unsafe { rb_iseq_opcode_at_pc(self.iseq, self.pc) } + } + + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> FrameStatePrinter<'a> { + FrameStatePrinter { inner: self, ptr_map } + } +} + +impl Display for FrameStatePrinter<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let inner = self.inner; + write!(f, "FrameState {{ pc: {:?}, stack: ", self.ptr_map.map_ptr(inner.pc))?; + write_vec(f, &inner.stack)?; + write!(f, ", locals: [")?; + for (idx, local) in inner.locals.iter().enumerate() { + let name: ID = unsafe { rb_zjit_local_id(inner.iseq, idx.try_into().unwrap()) }; + let name = name.contents_lossy(); + if idx > 0 { write!(f, ", ")?; } + write!(f, "{name}={local}")?; + } + write!(f, "] }}") + } +} + +/// Get YARV instruction argument +fn get_arg(pc: *const VALUE, arg_idx: isize) -> VALUE { + unsafe { *(pc.offset(arg_idx + 1)) } +} + +/// Compute YARV instruction index at relative offset +fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { + ((idx as isize) + (offset as isize)) as u32 +} + +/// List of insn_idx that starts a JIT entry block +pub fn jit_entry_insns(iseq: IseqPtr) -> Vec<u32> { + // TODO(alan): Make an iterator type for this instead of copying all of the opt_table each call + let params = unsafe { iseq.params() }; + let opt_num = params.opt_num; + if opt_num > 0 { + let mut result = vec![]; + + let opt_table = params.opt_table; // `opt_num + 1` entries + for opt_idx in 0..=opt_num as isize { + let insn_idx = unsafe { opt_table.offset(opt_idx).read().as_u32() }; + result.push(insn_idx); + } + result + } else { + vec![0] + } +} + +struct BytecodeInfo { + jump_targets: Vec<u32>, + has_blockiseq: bool, +} + +fn compute_bytecode_info(iseq: *const rb_iseq_t, opt_table: &[u32]) -> BytecodeInfo { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + let mut jump_targets: HashSet<u32> = opt_table.iter().copied().collect(); + let mut has_blockiseq = false; + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + insn_idx += insn_len(opcode as usize); + match opcode { + YARVINSN_branchunless | YARVINSN_jump | YARVINSN_branchif | YARVINSN_branchnil => { + let offset = get_arg(pc, 0).as_i64(); + jump_targets.insert(insn_idx_at_offset(insn_idx, offset)); + } + YARVINSN_opt_new => { + let offset = get_arg(pc, 1).as_i64(); + jump_targets.insert(insn_idx_at_offset(insn_idx, offset)); + } + YARVINSN_leave | YARVINSN_opt_invokebuiltin_delegate_leave => { + if insn_idx < iseq_size { + jump_targets.insert(insn_idx); + } + } + YARVINSN_send | YARVINSN_invokesuper => { + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + if !blockiseq.is_null() { + has_blockiseq = true; + } + } + _ => {} + } + } + let mut result = jump_targets.into_iter().collect::<Vec<_>>(); + result.sort(); + BytecodeInfo { jump_targets: result, has_blockiseq } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum CallType { + Splat, + Kwarg, + Tailcall, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ParseError { + StackUnderflow(FrameState), + MalformedIseq(u32), // insn_idx into iseq_encoded + Validation(ValidationError), + NotAllowed, +} + +/// Return the number of locals in the current ISEQ (includes parameters) +fn num_locals(iseq: *const rb_iseq_t) -> usize { + (unsafe { get_iseq_body_local_table_size(iseq) }).to_usize() +} + +/// If we can't handle the type of send (yet), bail out. +fn unhandled_call_type(flags: u32) -> Result<(), CallType> { + if (flags & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } + Ok(()) +} + +/// If a given call to a c func uses overly complex arguments, then we won't specialize. +fn unspecializable_c_call_type(flags: u32) -> bool { + ((flags & VM_CALL_KWARG) != 0) || + unspecializable_call_type(flags) +} + +/// If a given call uses overly complex arguments, then we won't specialize. +fn unspecializable_call_type(flags: u32) -> bool { + ((flags & VM_CALL_ARGS_SPLAT) != 0) || + ((flags & VM_CALL_KW_SPLAT) != 0) || + ((flags & VM_CALL_ARGS_BLOCKARG) != 0) || + ((flags & VM_CALL_FORWARDING) != 0) +} + +/// We have IseqPayload, which keeps track of HIR Types in the interpreter, but this is not useful +/// or correct to query from inside the optimizer. Instead, ProfileOracle provides an API to look +/// up profiled type information by HIR InsnId at a given ISEQ instruction. +#[derive(Debug)] +struct ProfileOracle { + payload: &'static IseqPayload, + /// types is a map from ISEQ instruction indices -> profiled type information at that ISEQ + /// instruction index. At a given ISEQ instruction, the interpreter has profiled the stack + /// operands to a given ISEQ instruction, and this list of pairs of (InsnId, Type) map that + /// profiling information into HIR instructions. + types: HashMap<usize, Vec<(InsnId, TypeDistributionSummary)>>, +} + +impl ProfileOracle { + fn new(payload: &'static IseqPayload) -> Self { + Self { payload, types: Default::default() } + } + + /// Map the interpreter-recorded types of the stack onto the HIR operands on our compile-time virtual stack + fn profile_stack(&mut self, state: &FrameState) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + // operand_types is always going to be <= stack size (otherwise it would have an underflow + // at run-time) so use that to drive iteration. + for (idx, insn_type_distribution) in operand_types.iter().rev().enumerate() { + let insn = state.stack_topn(idx).expect("Unexpected stack underflow in profiling"); + entry.push((insn, TypeDistributionSummary::new(insn_type_distribution))) + } + } + + /// Map the interpreter-recorded types of self onto the HIR self + fn profile_self(&mut self, state: &FrameState, self_param: InsnId) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + if operand_types.is_empty() { + return; + } + let self_type_distribution = &operand_types[0]; + entry.push((self_param, TypeDistributionSummary::new(self_type_distribution))) + } +} + +fn invalidates_locals(opcode: u32, operands: *const VALUE) -> bool { + match opcode { + // Control-flow is non-leaf in the interpreter because it can execute arbitrary code on + // interrupt. But in the JIT, we side-exit if there is a pending interrupt. + YARVINSN_jump + | YARVINSN_branchunless + | YARVINSN_branchif + | YARVINSN_branchnil + | YARVINSN_leave => false, + // TODO(max): Read the invokebuiltin target from operands and determine if it's leaf + _ => unsafe { !rb_zjit_insn_leaf(opcode as i32, operands) } + } +} + +/// The index of the self parameter in the HIR function +pub const SELF_PARAM_IDX: usize = 0; + +/// Compile ISEQ into High-level IR +pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { + if !ZJITState::can_compile_iseq(iseq) { + return Err(ParseError::NotAllowed); + } + let payload = get_or_create_iseq_payload(iseq); + let mut profiles = ProfileOracle::new(payload); + let mut fun = Function::new(iseq); + + // Compute a map of PC->Block by finding jump targets + let jit_entry_insns = jit_entry_insns(iseq); + let BytecodeInfo { jump_targets, has_blockiseq } = compute_bytecode_info(iseq, &jit_entry_insns); + + // Make all empty basic blocks. The ordering of the BBs matters for getting fallthrough jumps + // in good places, but it's not necessary for correctness. TODO: Higher quality scheduling during lowering. + let mut insn_idx_to_block = HashMap::new(); + // Make blocks for optionals first, and put them right next to their JIT entrypoint + for insn_idx in jit_entry_insns.iter().copied() { + let jit_entry_block = fun.new_block(insn_idx); + fun.jit_entry_blocks.push(jit_entry_block); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Make blocks for the rest of the jump targets + for insn_idx in jump_targets { + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Done, drop `mut`. + let insn_idx_to_block = insn_idx_to_block; + + // Compile an entry_block for the interpreter + compile_entry_block(&mut fun, jit_entry_insns.as_slice(), &insn_idx_to_block); + + // Compile all JIT-to-JIT entry blocks + for (jit_entry_idx, insn_idx) in jit_entry_insns.iter().enumerate() { + let target_block = insn_idx_to_block.get(insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + compile_jit_entry_block(&mut fun, jit_entry_idx, target_block); + } + + // Check if the EP is escaped for the ISEQ from the beginning. We give up + // optimizing locals in that case because they're shared with other frames. + let ep_escaped = iseq_escapes_ep(iseq); + + // Iteratively fill out basic blocks using a queue. + // TODO(max): Basic block arguments at edges + let mut queue = VecDeque::new(); + for &insn_idx in jit_entry_insns.iter() { + queue.push_back((FrameState::new(iseq), insn_idx_to_block[&insn_idx], /*insn_idx=*/insn_idx, /*local_inval=*/false)); + } + + // Keep compiling blocks until the queue becomes empty + let mut visited = HashSet::new(); + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + while let Some((incoming_state, block, mut insn_idx, mut local_inval)) = queue.pop_front() { + // Compile each block only once + if visited.contains(&block) { continue; } + visited.insert(block); + + // Load basic block params first + let self_param = fun.push_insn(block, Insn::Param); + let mut state = { + let mut result = FrameState::new(iseq); + let local_size = if jit_entry_insns.contains(&insn_idx) { num_locals(iseq) } else { incoming_state.locals.len() }; + for _ in 0..local_size { + result.locals.push(fun.push_insn(block, Insn::Param)); + } + for _ in incoming_state.stack { + result.stack.push(fun.push_insn(block, Insn::Param)); + } + result + }; + + // Start the block off with a Snapshot so that if we need to insert a new Guard later on + // and we don't have a Snapshot handy, we can just iterate backward (at the earliest, to + // the beginning of the block). + fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + while insn_idx < iseq_size { + state.insn_idx = insn_idx as usize; + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + state.pc = pc; + let exit_state = state.clone(); + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + + // If TracePoint has been enabled after we have collected profiles, we'll see + // trace_getinstancevariable in the ISEQ. We have to treat it like getinstancevariable + // for profiling purposes: there is no operand on the stack to look up; we have + // profiled cfp->self. + if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_setinstancevariable || opcode == YARVINSN_trace_setinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_definedivar || opcode == YARVINSN_trace_definedivar { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some(operand_types) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + if let [self_type_distribution] = &operand_types[..] { + let summary = TypeDistributionSummary::new(&self_type_distribution); + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 } { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_iseq)); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 } { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_ifunc)); + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_other)); + } + } else if summary.is_skewed_polymorphic() || summary.is_polymorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_polymorphic)); + } else if summary.is_skewed_megamorphic() || summary.is_megamorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_megamorphic)); + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles)); + } + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles)); + } + } + } + } else { + profiles.profile_stack(&exit_state); + } + + // Flag a future getlocal/setlocal to add a patch point if this instruction is not leaf. + if invalidates_locals(opcode, unsafe { pc.offset(1) }) { + local_inval = true; + } + + // We add NoTracePoint patch points before every instruction that could be affected by TracePoint. + // This ensures that if TracePoint is enabled, we can exit the generated code as fast as possible. + unsafe extern "C" { + fn rb_iseq_event_flags(iseq: IseqPtr, pos: usize) -> rb_event_flag_t; + } + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.clone() }); + if unsafe { rb_iseq_event_flags(iseq, insn_idx as usize) } != 0 { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state: exit_id }); + } + + // Increment zjit_insn_count for each YARV instruction if --zjit-stats is enabled. + if get_option!(stats) { + fun.push_insn(block, Insn::IncrCounter(Counter::zjit_insn_count)); + } + // Move to the next instruction to compile + insn_idx += insn_len(opcode as usize); + + match opcode { + YARVINSN_nop => {}, + YARVINSN_putnil => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); }, + YARVINSN_putobject => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) })); }, + YARVINSN_putspecialobject => { + let value_type = SpecialObjectType::from(get_arg(pc, 0).as_u32()); + let insn = if value_type == SpecialObjectType::VMCore { + Insn::Const { val: Const::Value(unsafe { rb_mRubyVMFrozenCore }) } + } else { + Insn::PutSpecialObject { value_type } + }; + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_putstring => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: false, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_putchilledstring => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: true, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_putself => { state.stack_push(self_param); } + YARVINSN_intern => { + let val = state.stack_pop()?; + let insn_id = fun.push_insn(block, Insn::StringIntern { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_concatstrings => { + let count = get_arg(pc, 0).as_u32(); + let strings = state.stack_pop_n(count as usize)?; + let insn_id = fun.push_insn(block, Insn::StringConcat { strings, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_toregexp => { + // First arg contains the options (multiline, extended, ignorecase) used to create the regexp + let opt = get_arg(pc, 0).as_usize(); + let count = get_arg(pc, 1).as_usize(); + let values = state.stack_pop_n(count)?; + let insn_id = fun.push_insn(block, Insn::ToRegexp { opt, values, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_newarray => { + let count = get_arg(pc, 0).as_usize(); + let elements = state.stack_pop_n(count)?; + state.stack_push(fun.push_insn(block, Insn::NewArray { elements, state: exit_id })); + } + YARVINSN_opt_newarray_send => { + let count = get_arg(pc, 0).as_usize(); + let method = get_arg(pc, 1).as_u32(); + let (bop, insn) = match method { + VM_OPT_NEWARRAY_SEND_MAX => { + let elements = state.stack_pop_n(count)?; + (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_HASH => { + let elements = state.stack_pop_n(count)?; + (BOP_HASH, Insn::ArrayHash { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_INCLUDE_P => { + let target = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_INCLUDE_P, Insn::ArrayInclude { elements, target, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => { + let buffer = state.stack_pop()?; + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 2)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer, state: exit_id }) + } + _ => { + // Unknown opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method) }); + break; // End the block + } + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) }); + break; // End the block + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_duparray => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::ArrayDup { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_opt_duparray_send => { + let ary = get_arg(pc, 0); + let method_id = get_arg(pc, 1).as_u64(); + let argc = get_arg(pc, 2).as_usize(); + if argc != 1 { + break; + } + let target = state.stack_pop()?; + let bop = match method_id { + x if x == ID!(include_p).0 => BOP_INCLUDE_P, + _ => { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id) }); + break; + }, + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) }); + break; + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + let insn_id = fun.push_insn(block, Insn::DupArrayInclude { ary, target, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_newhash => { + let count = get_arg(pc, 0).as_usize(); + assert!(count % 2 == 0, "newhash count should be even"); + let mut elements = vec![]; + for _ in 0..(count/2) { + let value = state.stack_pop()?; + let key = state.stack_pop()?; + elements.push(value); + elements.push(key); + } + elements.reverse(); + state.stack_push(fun.push_insn(block, Insn::NewHash { elements, state: exit_id })); + } + YARVINSN_duphash => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::HashDup { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_splatarray => { + let flag = get_arg(pc, 0); + let result_must_be_mutable = flag.test(); + let val = state.stack_pop()?; + let obj = if result_must_be_mutable { + fun.push_insn(block, Insn::ToNewArray { val, state: exit_id }) + } else { + fun.push_insn(block, Insn::ToArray { val, state: exit_id }) + }; + state.stack_push(obj); + } + YARVINSN_concattoarray => { + let right = state.stack_pop()?; + let left = state.stack_pop()?; + let right_array = fun.push_insn(block, Insn::ToArray { val: right, state: exit_id }); + fun.push_insn(block, Insn::ArrayExtend { left, right: right_array, state: exit_id }); + state.stack_push(left); + } + YARVINSN_pushtoarray => { + let count = get_arg(pc, 0).as_usize(); + let vals = state.stack_pop_n(count)?; + let array = state.stack_pop()?; + for val in vals.into_iter() { + fun.push_insn(block, Insn::ArrayPush { array, val, state: exit_id }); + } + state.stack_push(array); + } + YARVINSN_putobject_INT2FIX_0_ => { + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(0)) })); + } + YARVINSN_putobject_INT2FIX_1_ => { + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(1)) })); + } + YARVINSN_defined => { + // (rb_num_t op_type, VALUE obj, VALUE pushval) + let op_type = get_arg(pc, 0).as_usize(); + let obj = get_arg(pc, 1); + let pushval = get_arg(pc, 2); + let v = state.stack_pop()?; + state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v, state: exit_id })); + } + YARVINSN_definedivar => { + // (ID id, IVC ic, VALUE pushval) + let id = ID(get_arg(pc, 0).as_u64()); + let pushval = get_arg(pc, 2); + state.stack_push(fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id })); + } + YARVINSN_checkkeyword => { + // When a keyword is unspecified past index 32, a hash will be used instead. + // This can only happen in iseqs taking more than 32 keywords. + // In this case, we side exit to the interpreter. + if unsafe {(*rb_get_iseq_body_param_keyword(iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap()} { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters }); + break; + } + let ep_offset = get_arg(pc, 0).as_u32(); + let index = get_arg(pc, 1).as_u64(); + let index: u8 = index.try_into().map_err(|_| ParseError::MalformedIseq(insn_idx))?; + // Use FrameState to get kw_bits when possible, just like getlocal_WC_0. + let val = if !local_inval { + state.getlocal(ep_offset) + } else if ep_escaped || has_blockiseq { + fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }) + } else { + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + state.getlocal(ep_offset) + }; + state.stack_push(fun.push_insn(block, Insn::FixnumBitCheck { val, index })); + } + YARVINSN_opt_getconstant_path => { + let ic = get_arg(pc, 0).as_ptr(); + let snapshot = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + state.stack_push(fun.push_insn(block, Insn::GetConstantPath { ic, state: snapshot })); + } + YARVINSN_branchunless => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::Test { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let nil_false_type = types::NilClass.union(types::FalseClass); + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + let mut iffalse_state = state.clone(); + iffalse_state.replace(val, nil_false); + let _branch_id = fun.push_insn(block, Insn::IfFalse { + val: test_id, + target: BranchEdge { target, args: iffalse_state.as_args(self_param) } + }); + let not_nil_false_type = types::BasicObject.subtract(types::NilClass).subtract(types::FalseClass); + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + state.replace(val, not_nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_branchif => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::Test { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let not_nil_false_type = types::BasicObject.subtract(types::NilClass).subtract(types::FalseClass); + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, not_nil_false); + let _branch_id = fun.push_insn(block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target, args: iftrue_state.as_args(self_param) } + }); + let nil_false_type = types::NilClass.union(types::FalseClass); + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + state.replace(val, nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_branchnil => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::IsNil { val }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let nil = fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, nil); + let _branch_id = fun.push_insn(block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target, args: iftrue_state.as_args(self_param) } + }); + let new_type = types::BasicObject.subtract(types::NilClass); + let not_nil = fun.push_insn(block, Insn::RefineType { val, new_type }); + state.replace(val, not_nil); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_opt_case_dispatch => { + // TODO: Some keys are visible at compile time, so in the future we can + // compile jump targets for certain cases + // Pop the key from the stack and fallback to the === branches for now + state.stack_pop()?; + } + YARVINSN_opt_new => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let dst = get_arg(pc, 1).as_i64(); + + // Check if #new resolves to rb_class_new_instance_pass_kw. + // TODO: Guard on a profiled class and add a patch point for #new redefinition + let argc = unsafe { vm_ci_argc((*cd).ci) } as usize; + let val = state.stack_topn(argc)?; + let test_id = fun.push_insn(block, Insn::IsMethodCfunc { val, cd, cfunc: rb_class_new_instance_pass_kw as *const u8, state: exit_id }); + + // Jump to the fallback block if it's not the expected function. + // Skip CheckInterrupts since the #new call will do it very soon anyway. + let target_idx = insn_idx_at_offset(insn_idx, dst); + let target = insn_idx_to_block[&target_idx]; + let _branch_id = fun.push_insn(block, Insn::IfFalse { + val: test_id, + target: BranchEdge { target, args: state.as_args(self_param) } + }); + queue.push_back((state.clone(), target, target_idx, local_inval)); + + // Move on to the fast path + let insn_id = fun.push_insn(block, Insn::ObjectAlloc { val, state: exit_id }); + state.stack_setn(argc, insn_id); + state.stack_setn(argc + 1, insn_id); + } + YARVINSN_jump => { + let offset = get_arg(pc, 0).as_i64(); + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + let target_idx = insn_idx_at_offset(insn_idx, offset); + let target = insn_idx_to_block[&target_idx]; + let _branch_id = fun.push_insn(block, Insn::Jump( + BranchEdge { target, args: state.as_args(self_param) } + )); + queue.push_back((state.clone(), target, target_idx, local_inval)); + break; // Don't enqueue the next block as a successor + } + YARVINSN_getlocal_WC_0 => { + let ep_offset = get_arg(pc, 0).as_u32(); + if !local_inval { + // The FrameState is the source of truth for locals until invalidated. + // In case of JIT-to-JIT send locals might never end up in EP memory. + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else if ep_escaped || has_blockiseq { // TODO: figure out how to drop has_blockiseq here + // Read the local using EP + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); // remember the result to spill on side-exits + state.stack_push(val); + } else { + assert!(local_inval); // if check above + // There has been some non-leaf call since JIT entry or the last patch point, + // so add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + + // Read the local from FrameState + let val = state.getlocal(ep_offset); + state.stack_push(val); + } + } + YARVINSN_setlocal_WC_0 => { + let ep_offset = get_arg(pc, 0).as_u32(); + let val = state.stack_pop()?; + if ep_escaped || has_blockiseq { // TODO: figure out how to drop has_blockiseq here + // Write the local using EP + fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 }); + } else if local_inval { + // If there has been any non-leaf call since JIT entry or the last patch point, + // add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + } + // Write the local into FrameState + state.setlocal(ep_offset, val); + } + YARVINSN_getlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 1, use_sp: false, rest_param: false })); + } + YARVINSN_setlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level: 1 }); + } + YARVINSN_getlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level, use_sp: false, rest_param: false })); + } + YARVINSN_setlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level }); + } + YARVINSN_getblockparamproxy => { + let level = get_arg(pc, 1).as_u32(); + fun.push_insn(block, Insn::GuardBlockParamProxy { level, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) })); + } + YARVINSN_pop => { state.stack_pop()?; } + YARVINSN_dup => { state.stack_push(state.stack_top()?); } + YARVINSN_dupn => { + // Duplicate the top N element of the stack. As we push, n-1 naturally + // points higher in the original stack. + let n = get_arg(pc, 0).as_usize(); + for _ in 0..n { + state.stack_push(state.stack_topn(n-1)?); + } + } + YARVINSN_swap => { + let right = state.stack_pop()?; + let left = state.stack_pop()?; + state.stack_push(right); + state.stack_push(left); + } + YARVINSN_setn => { + let n = get_arg(pc, 0).as_usize(); + let top = state.stack_top()?; + state.stack_setn(n, top); + } + YARVINSN_topn => { + let n = get_arg(pc, 0).as_usize(); + let top = state.stack_topn(n)?; + state.stack_push(top); + } + YARVINSN_adjuststack => { + let mut n = get_arg(pc, 0).as_usize(); + while n > 0 { + state.stack_pop()?; + n -= 1; + } + } + YARVINSN_opt_neq => { + // NB: opt_neq has two cd; get_arg(0) is for eq and get_arg(1) is for neq + let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::SendWithoutBlock { recv, cd, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + } + YARVINSN_opt_hash_freeze => { + let klass = HASH_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_ary_freeze => { + let klass = ARRAY_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_str_freeze => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_str_uminus => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_UMINUS; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_leave => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + fun.push_insn(block, Insn::Return { val: state.stack_pop()? }); + break; // Don't enqueue the next block as a successor + } + YARVINSN_throw => { + fun.push_insn(block, Insn::Throw { throw_state: get_arg(pc, 0).as_u32(), val: state.stack_pop()?, state: exit_id }); + break; // Don't enqueue the next block as a successor + } + + // These are opt_send_without_block and all the opt_* instructions + // specialized to a certain method that could also be serviced + // using the general send implementation. The optimizer start from + // a general send for all of these later in the pipeline. + YARVINSN_opt_nil_p | + YARVINSN_opt_plus | + YARVINSN_opt_minus | + YARVINSN_opt_mult | + YARVINSN_opt_div | + YARVINSN_opt_mod | + YARVINSN_opt_eq | + YARVINSN_opt_lt | + YARVINSN_opt_le | + YARVINSN_opt_gt | + YARVINSN_opt_ge | + YARVINSN_opt_ltlt | + YARVINSN_opt_aset | + YARVINSN_opt_length | + YARVINSN_opt_size | + YARVINSN_opt_aref | + YARVINSN_opt_empty_p | + YARVINSN_opt_succ | + YARVINSN_opt_and | + YARVINSN_opt_or | + YARVINSN_opt_not | + YARVINSN_opt_regexpmatch2 | + YARVINSN_opt_send_without_block => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::SendWithoutBlock { recv, cd, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + } + YARVINSN_send => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::Send { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } + YARVINSN_sendforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send_forward); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } + YARVINSN_invokesuper => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; + let recv = state.stack_pop()?; + let blockiseq: IseqPtr = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::InvokeSuper { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } + YARVINSN_invokeblock => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; + let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + } + YARVINSN_getglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let result = fun.push_insn(block, Insn::GetGlobal { id, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetGlobal { id, val, state: exit_id }); + } + YARVINSN_getinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // ic is in arg 1 + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_getivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_getivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_setivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_setivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id }); + } + YARVINSN_getclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::GetClassVar { id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetClassVar { id, val, ic, state: exit_id }); + } + YARVINSN_opt_reverse => { + // Reverse the order of the top N stack items. + let n = get_arg(pc, 0).as_usize(); + for i in 0..n/2 { + let bottom = state.stack_topn(n - 1 - i)?; + let top = state.stack_topn(i)?; + state.stack_setn(i, bottom); + state.stack_setn(n - 1 - i, top); + } + } + YARVINSN_newrange => { + let flag = RangeType::from(get_arg(pc, 0).as_u32()); + let high = state.stack_pop()?; + let low = state.stack_pop()?; + let insn_id = fun.push_insn(block, Insn::NewRange { low, high, flag, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_invokebuiltin => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + + let mut args = vec![]; + for _ in 0..bf.argc { + args.push(state.stack_pop()?); + } + args.push(self_param); + args.reverse(); + + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_opt_invokebuiltin_delegate | + YARVINSN_opt_invokebuiltin_delegate_leave => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let index = get_arg(pc, 1).as_usize(); + let argc = bf.argc as usize; + + let mut args = vec![self_param]; + for &local in state.locals().skip(index).take(argc) { + args.push(local); + } + + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_objtostring => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + assert_eq!(0, argc, "objtostring should not have args"); + + let recv = state.stack_pop()?; + let objtostring = fun.push_insn(block, Insn::ObjToString { val: recv, cd, state: exit_id }); + state.stack_push(objtostring) + } + YARVINSN_anytostring => { + let str = state.stack_pop()?; + let val = state.stack_pop()?; + + let anytostring = fun.push_insn(block, Insn::AnyToString { val, str, state: exit_id }); + state.stack_push(anytostring); + } + YARVINSN_getspecial => { + let key = get_arg(pc, 0).as_u64(); + let svar = get_arg(pc, 1).as_u64(); + + if svar == 0 { + // TODO: Handle non-backref + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key) }); + // End the block + break; + } else if svar & 0x01 != 0 { + // Handle symbol backrefs like $&, $`, $', $+ + let shifted_svar: u8 = (svar >> 1).try_into().unwrap(); + let symbol_type = SpecialBackrefSymbol::try_from(shifted_svar).expect("invalid backref symbol"); + let result = fun.push_insn(block, Insn::GetSpecialSymbol { symbol_type, state: exit_id }); + state.stack_push(result); + } else { + // Handle number backrefs like $1, $2, $3 + let result = fun.push_insn(block, Insn::GetSpecialNumber { nth: svar, state: exit_id }); + state.stack_push(result); + } + } + YARVINSN_expandarray => { + let num = get_arg(pc, 0).as_u64(); + let flag = get_arg(pc, 1).as_u64(); + if flag != 0 { + // We don't (yet) handle 0x01 (rest args), 0x02 (post args), or 0x04 + // (reverse?) + // + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let val = state.stack_pop()?; + let array = fun.push_insn(block, Insn::GuardType { val, guard_type: types::ArrayExact, state: exit_id, }); + let length = fun.push_insn(block, Insn::ArrayLength { array }); + fun.push_insn(block, Insn::GuardBitEquals { val: length, expected: Const::CInt64(num as i64), reason: SideExitReason::ExpandArray, state: exit_id }); + for i in (0..num).rev() { + // We do not emit a length guard here because in-bounds is already + // ensured by the expandarray length check above. + let index = fun.push_insn(block, Insn::Const { val: Const::CInt64(i.try_into().unwrap()) }); + let element = fun.push_insn(block, Insn::ArrayAref { array, index }); + state.stack_push(element); + } + } + _ => { + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + } + + if insn_idx_to_block.contains_key(&insn_idx) { + let target = insn_idx_to_block[&insn_idx]; + fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args(self_param) })); + queue.push_back((state, target, insn_idx, local_inval)); + break; // End the block + } + } + } + + fun.set_param_types(); + fun.infer_types(); + + match get_option!(dump_hir_init) { + Some(DumpHIR::WithoutSnapshot) => println!("Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)), + Some(DumpHIR::All) => println!("Initial HIR:\n{}", FunctionPrinter::with_snapshot(&fun)), + Some(DumpHIR::Debug) => println!("Initial HIR:\n{:#?}", &fun), + None => {}, + } + + fun.profiles = Some(profiles); + if let Err(err) = fun.validate() { + debug!("ZJIT: {err:?}: Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)); + return Err(ParseError::Validation(err)); + } + Ok(fun) +} + +/// Compile an entry_block for the interpreter +fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_block: &HashMap<u32, BlockId>) { + let entry_block = fun.entry_block; + let (self_param, entry_state) = compile_entry_state(fun); + let mut pc: Option<InsnId> = None; + let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + + // Check-and-jump for each missing optional PC + for &jit_entry_insn in jit_entry_insns.iter() { + if jit_entry_insn == all_opts_passed_insn_idx { + continue; + } + let target_block = insn_idx_to_block.get(&jit_entry_insn) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + // Load PC once at the start of the block, shared among all cases + let pc = *pc.get_or_insert_with(|| fun.push_insn(entry_block, Insn::LoadPC)); + let expected_pc = fun.push_insn(entry_block, Insn::Const { + val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), + }); + let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); + fun.push_insn(entry_block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, + }); + } + + // Terminate the block with a jump to the block with all optionals passed + let target_block = insn_idx_to_block.get(&all_opts_passed_insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile initial locals for an entry_block for the interpreter +fn compile_entry_state(fun: &mut Function) -> (InsnId, FrameState) { + let entry_block = fun.entry_block; + fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); + + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + let self_param = fun.push_insn(entry_block, Insn::LoadSelf); + let mut entry_state = FrameState::new(iseq); + for local_idx in 0..num_locals(iseq) { + if local_idx < param_size { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + let use_sp = !iseq_escapes_ep(iseq); // If the ISEQ does not escape EP, we can assume EP + 1 == SP + let rest_param = Some(local_idx as i32) == rest_param_idx; + entry_state.locals.push(fun.push_insn(entry_block, Insn::GetLocal { level: 0, ep_offset, use_sp, rest_param })); + } else { + entry_state.locals.push(fun.push_insn(entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +/// Compile a jit_entry_block +fn compile_jit_entry_block(fun: &mut Function, jit_entry_idx: usize, target_block: BlockId) { + let jit_entry_block = fun.jit_entry_blocks[jit_entry_idx]; + fun.push_insn(jit_entry_block, Insn::EntryPoint { jit_entry_idx: Some(jit_entry_idx) }); + + // Prepare entry_state with basic block params + let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block, jit_entry_idx); + + // Jump to target_block + fun.push_insn(jit_entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile params and initial locals for a jit_entry_block +fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_entry_idx: usize) -> (InsnId, FrameState) { + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let opt_num: usize = params.opt_num.try_into().expect("iseq param opt_num >= 0"); + let lead_num: usize = params.lead_num.try_into().expect("iseq param lead_num >= 0"); + let passed_opt_num = jit_entry_idx; + + // If the iseq has keyword parameters, the keyword bits local will be appended to the local table. + let kw_bits_idx: Option<usize> = if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if !keyword.is_null() { + Some(unsafe { (*keyword).bits_start } as usize) + } else { + None + } + } else { + None + }; + + let self_param = fun.push_insn(jit_entry_block, Insn::Param); + let mut entry_state = FrameState::new(iseq); + for local_idx in 0..num_locals(iseq) { + if (lead_num + passed_opt_num..lead_num + opt_num).contains(&local_idx) { + // Omitted optionals are locals, so they start as nils before their code run + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } else if Some(local_idx) == kw_bits_idx { + // Read the kw_bits value written by the caller to the callee frame. + // This tells us which optional keywords were NOT provided and need their defaults evaluated. + // Note: The caller writes kw_bits to memory via gen_send_iseq_direct but does NOT pass it + // as a C argument, so we must read it from memory using GetLocal rather than Param. + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::GetLocal { level: 0, ep_offset, use_sp: false, rest_param: false })); + } else if local_idx < param_size { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Param)); + } else { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +pub struct Dominators<'a> { + f: &'a Function, + dominators: Vec<Vec<BlockId>>, +} + +impl<'a> Dominators<'a> { + pub fn new(f: &'a Function) -> Self { + let mut cfi = ControlFlowInfo::new(f); + Self::with_cfi(f, &mut cfi) + } + + pub fn with_cfi(f: &'a Function, cfi: &mut ControlFlowInfo) -> Self { + let block_ids = f.rpo(); + let mut dominators = vec![vec![]; f.blocks.len()]; + + // Compute dominators for each node using fixed point iteration. + // Approach can be found in Figure 1 of: + // https://www.cs.tufts.edu/~nr/cs257/archive/keith-cooper/dom14.pdf + // + // Initially we set: + // + // dom(entry) = {entry} for each entry block + // dom(b != entry) = {all nodes} + // + // Iteratively, apply: + // + // dom(b) = {b} union intersect(dom(p) for p in predecessors(b)) + // + // When we've run the algorithm and the dominator set no longer changes + // between iterations, then we have found the dominator sets. + + // Set up entry blocks. + // Entry blocks are only dominated by themselves. + for entry_block in &f.entry_blocks() { + dominators[entry_block.0] = vec![*entry_block]; + } + + // Setup the initial dominator sets. + for block_id in &block_ids { + if !f.entry_blocks().contains(block_id) { + // Non entry blocks are initially dominated by all other blocks. + dominators[block_id.0] = block_ids.clone(); + } + } + + let mut changed = true; + while changed { + changed = false; + + for block_id in &block_ids { + if *block_id == f.entry_block { + continue; + } + + // Get all predecessors for a given block. + let block_preds: Vec<BlockId> = cfi.predecessors(*block_id).collect(); + if block_preds.is_empty() { + continue; + } + + let mut new_doms = dominators[block_preds[0].0].clone(); + + // Compute the intersection of predecessor dominator sets into `new_doms`. + for pred_id in &block_preds[1..] { + let pred_doms = &dominators[pred_id.0]; + // Only keep a dominator in `new_doms` if it is also found in pred_doms + new_doms.retain(|d| pred_doms.contains(d)); + } + + // Insert sorted into `new_doms`. + match new_doms.binary_search(block_id) { + Ok(_) => {} + Err(pos) => new_doms.insert(pos, *block_id) + } + + // If we have computed a new dominator set, then we can update + // the dominators and mark that we need another iteration. + if dominators[block_id.0] != new_doms { + dominators[block_id.0] = new_doms; + changed = true; + } + } + } + + Self { f, dominators } + } + + + pub fn is_dominated_by(&self, left: BlockId, right: BlockId) -> bool { + self.dominators(left).any(|&b| b == right) + } + + pub fn dominators(&self, block: BlockId) -> Iter<'_, BlockId> { + self.dominators[block.0].iter() + } +} + +pub struct ControlFlowInfo<'a> { + function: &'a Function, + successor_map: HashMap<BlockId, Vec<BlockId>>, + predecessor_map: HashMap<BlockId, Vec<BlockId>>, +} + +impl<'a> ControlFlowInfo<'a> { + pub fn new(function: &'a Function) -> Self { + let mut successor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let mut predecessor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let uf = function.union_find.borrow(); + + for block_id in function.rpo() { + let block = &function.blocks[block_id.0]; + + // Since ZJIT uses extended basic blocks, one must check all instructions + // for their ability to jump to another basic block, rather than just + // the instructions at the end of a given basic block. + // + // Use BTreeSet to avoid duplicates and maintain an ordering. Also + // `BTreeSet<BlockId>` provides conversion trivially back to an `Vec<BlockId>`. + // Ordering is important so that the expect tests that serialize the predecessors + // and successors don't fail intermittently. + // todo(aidenfoxivey): Use `BlockSet` in lieu of `BTreeSet<BlockId>` + let successors: BTreeSet<BlockId> = block + .insns + .iter() + .map(|&insn_id| uf.find_const(insn_id)) + .filter_map(|insn_id| { + Self::extract_jump_target(&function.insns[insn_id.0]) + }) + .collect(); + + // Update predecessors for successor blocks. + for &succ_id in &successors { + predecessor_map + .entry(succ_id) + .or_default() + .push(block_id); + } + + // Store successors for this block. + // Convert successors from a `BTreeSet<BlockId>` to a `Vec<BlockId>`. + successor_map.insert(block_id, successors.iter().copied().collect()); + } + + Self { + function, + successor_map, + predecessor_map, + } + } + + pub fn is_succeeded_by(&self, left: BlockId, right: BlockId) -> bool { + self.successor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn is_preceded_by(&self, left: BlockId, right: BlockId) -> bool { + self.predecessor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn predecessors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.predecessor_map.get(&block).into_iter().flatten().copied() + } + + pub fn successors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.successor_map.get(&block).into_iter().flatten().copied() + } + + /// Helper function to extract the target of a jump instruction. + fn extract_jump_target(insn: &Insn) -> Option<BlockId> { + match insn { + Insn::Jump(target) + | Insn::IfTrue { target, .. } + | Insn::IfFalse { target, .. } => Some(target.target), + _ => None, + } + } +} + +pub struct LoopInfo<'a> { + cfi: &'a ControlFlowInfo<'a>, + dominators: &'a Dominators<'a>, + loop_depths: HashMap<BlockId, u32>, + loop_headers: BlockSet, + back_edge_sources: BlockSet, +} + +impl<'a> LoopInfo<'a> { + pub fn new(cfi: &'a ControlFlowInfo<'a>, dominators: &'a Dominators<'a>) -> Self { + let mut loop_headers: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let mut loop_depths: HashMap<BlockId, u32> = HashMap::new(); + let mut back_edge_sources: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let rpo = cfi.function.rpo(); + + for &block in &rpo { + loop_depths.insert(block, 0); + } + + // Collect loop headers. + for &block in &rpo { + // Initialize the loop depths. + for predecessor in cfi.predecessors(block) { + if dominators.is_dominated_by(predecessor, block) { + // Found a loop header, so then identify the natural loop. + loop_headers.insert(block); + back_edge_sources.insert(predecessor); + let loop_blocks = Self::find_natural_loop(cfi, block, predecessor); + // Increment the loop depth. + for loop_block in &loop_blocks { + *loop_depths.get_mut(loop_block).expect("Loop block should be populated.") += 1; + } + } + } + } + + Self { + cfi, + dominators, + loop_depths, + loop_headers, + back_edge_sources, + } + } + + fn find_natural_loop( + cfi: &ControlFlowInfo, + header: BlockId, + back_edge_source: BlockId, + ) -> HashSet<BlockId> { + // todo(aidenfoxivey): Reimplement using BlockSet + let mut loop_blocks = HashSet::new(); + let mut stack = vec![back_edge_source]; + + loop_blocks.insert(header); + loop_blocks.insert(back_edge_source); + + while let Some(block) = stack.pop() { + for pred in cfi.predecessors(block) { + // Pushes to stack only if `pred` wasn't already in `loop_blocks`. + if loop_blocks.insert(pred) { + stack.push(pred) + } + } + } + + loop_blocks + } + + pub fn loop_depth(&self, block: BlockId) -> u32 { + self.loop_depths.get(&block).copied().unwrap_or(0) + } + + pub fn is_back_edge_source(&self, block: BlockId) -> bool { + self.back_edge_sources.get(block) + } + + pub fn is_loop_header(&self, block: BlockId) -> bool { + self.loop_headers.get(block) + } +} + +#[cfg(test)] +mod union_find_tests { + use super::UnionFind; + + #[test] + fn test_find_returns_self() { + let mut uf = UnionFind::new(); + assert_eq!(uf.find(3usize), 3); + } + + #[test] + fn test_find_returns_target() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + assert_eq!(uf.find(3usize), 4); + } + + #[test] + fn test_find_returns_transitive_target() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + uf.make_equal_to(4, 5); + assert_eq!(uf.find(3usize), 5); + assert_eq!(uf.find(4usize), 5); + } + + #[test] + fn test_find_compresses_path() { + let mut uf = UnionFind::new(); + uf.make_equal_to(3, 4); + uf.make_equal_to(4, 5); + assert_eq!(uf.at(3usize), Some(4)); + assert_eq!(uf.find(3usize), 5); + assert_eq!(uf.at(3usize), Some(5)); + } +} + +#[cfg(test)] +mod rpo_tests { + use super::*; + + #[test] + fn one_block() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Return { val }); + assert_eq!(function.rpo(), vec![entry]); + } + + #[test] + fn jump() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let exit = function.new_block(0); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Return { val }); + assert_eq!(function.rpo(), vec![entry, exit]); + } + + #[test] + fn diamond_iftrue() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Return { val }); + assert_eq!(function.rpo(), vec![entry, side, exit]); + } + + #[test] + fn diamond_iffalse() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Return { val }); + assert_eq!(function.rpo(), vec![entry, side, exit]); + } + + #[test] + fn a_loop() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + function.push_insn(entry, Insn::Jump(BranchEdge { target: entry, args: vec![] })); + assert_eq!(function.rpo(), vec![entry]); + } +} + +#[cfg(test)] +mod validation_tests { + use super::*; + + #[track_caller] + fn assert_matches_err(res: Result<(), ValidationError>, expected: ValidationError) { + match res { + Err(validation_err) => { + assert_eq!(validation_err, expected); + } + Ok(_) => panic!("Expected validation error"), + } + } + + #[test] + fn one_block_no_terminator() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + assert_matches_err(function.validate(), ValidationError::BlockHasNoTerminator(entry)); + } + + #[test] + fn one_block_terminator_not_at_end() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let insn_id = function.push_insn(entry, Insn::Return { val }); + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + assert_matches_err(function.validate(), ValidationError::TerminatorNotAtEnd(entry, insn_id, 1)); + } + + #[test] + fn iftrue_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn iffalse_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn jump_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Jump ( BranchEdge { target: side, args: vec![val, val, val] } )); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn not_defined_within_bb() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + // Create an instruction without making it belong to anything. + let dangling = function.new_insn(Insn::Const{val: Const::CBool(true)}); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: dangling, state: InsnId(0usize) }); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, dangling)); + } + + #[test] + fn using_non_output_insn() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let const_ = function.push_insn(function.entry_block, Insn::Const{val: Const::CBool(true)}); + // Ret is a non-output instruction. + let ret = function.push_insn(function.entry_block, Insn::Return { val: const_ }); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: ret, state: InsnId(0usize) }); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, ret)); + } + + #[test] + fn not_dominated_by_diamond() { + // This tests that one branch is missing a definition which fails. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val1 = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val: val1, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val2 = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(exit, val2, v0)); + }); + } + + #[test] + fn dominated_by_diamond() { + // This tests that both branches with a definition succeeds. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let _val = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + // Just checking that we don't panic. + assert!(function.validate_definite_assignment().is_ok()); + }); + } + + #[test] + fn instruction_appears_twice_in_same_block() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn_id(block, val); + function.push_insn(block, Insn::Return { val }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val)); + } + + #[test] + fn instruction_appears_twice_with_different_ids() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val0 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let val1 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.make_equal_to(val1, val0); + function.push_insn(block, Insn::Return { val: val0 }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val0)); + } + + #[test] + fn instruction_appears_twice_in_different_blocks() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let exit = function.new_block(0); + function.push_insn(block, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn_id(exit, val); + function.push_insn(exit, Insn::Return { val }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(exit, val)); + } +} + +#[cfg(test)] +mod infer_tests { + use super::*; + + #[track_caller] + fn assert_subtype(left: Type, right: Type) { + assert!(left.is_subtype(right), "{left} is not a subtype of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Type, right: Type) { + assert!(left.bit_equal(right), "{left} != {right}"); + } + + #[test] + fn test_const() { + let mut function = Function::new(std::ptr::null()); + let val = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); + assert_bit_equal(function.infer_type(val), types::NilClass); + } + + #[test] + fn test_nil() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let nil = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: nil }); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(false)); + }); + } + + #[test] + fn test_false() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let false_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qfalse) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: false_ }); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(false)); + }); + } + + #[test] + fn test_truthy() { + crate::cruby::with_rubyvm(|| { + let mut function = Function::new(std::ptr::null()); + let true_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qtrue) }); + let val = function.push_insn(function.entry_block, Insn::Test { val: true_ }); + function.infer_types(); + assert_bit_equal(function.type_of(val), Type::from_cbool(true)); + }); + } + + #[test] + fn newarray() { + let mut function = Function::new(std::ptr::null()); + // Fake FrameState index of 0usize + let val = function.push_insn(function.entry_block, Insn::NewArray { elements: vec![], state: InsnId(0usize) }); + assert_bit_equal(function.infer_type(val), types::ArrayExact); + } + + #[test] + fn arraydup() { + let mut function = Function::new(std::ptr::null()); + // Fake FrameState index of 0usize + let arr = function.push_insn(function.entry_block, Insn::NewArray { elements: vec![], state: InsnId(0usize) }); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: arr, state: InsnId(0usize) }); + assert_bit_equal(function.infer_type(val), types::ArrayExact); + } + + #[test] + fn diamond_iffalse_merge_fixnum() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); + let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(4)) }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); + let param = function.push_insn(exit, Insn::Param); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + }); + assert_bit_equal(function.type_of(param), types::Fixnum); + } + + #[test] + fn diamond_iffalse_merge_bool() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(Qtrue) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); + let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); + let param = function.push_insn(exit, Insn::Param); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + assert_bit_equal(function.type_of(param), types::TrueClass.union(types::FalseClass)); + }); + } +} + +#[cfg(test)] +mod graphviz_tests { + use super::*; + use insta::assert_snapshot; + + #[track_caller] + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + format!("{}", FunctionGraphvizPrinter::new(&function)) + } + + #[test] + fn test_guard_fixnum_or_fixnum() { + eval(r#" + def test(x, y) = x | y + + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @r#" + digraph G { # test@<compiled>:2 + node [shape=plaintext]; + mode=hier; overlap=false; splines=true; + bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0() </TD></TR> + <TR><TD ALIGN="left" PORT="v0">EntryPoint interpreter </TD></TR> + <TR><TD ALIGN="left" PORT="v1">v1:BasicObject = LoadSelf </TD></TR> + <TR><TD ALIGN="left" PORT="v2">v2:BasicObject = GetLocal :x, l0, SP@5 </TD></TR> + <TR><TD ALIGN="left" PORT="v3">v3:BasicObject = GetLocal :y, l0, SP@4 </TD></TR> + <TR><TD ALIGN="left" PORT="v4">Jump bb2(v1, v2, v3) </TD></TR> + </TABLE>>]; + bb0:v4 -> bb2:params:n; + bb1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v5">EntryPoint JIT(0) </TD></TR> + <TR><TD ALIGN="left" PORT="v9">Jump bb2(v6, v7, v8) </TD></TR> + </TABLE>>]; + bb1:v9 -> bb2:params:n; + bb2 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v15">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v18">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v24">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v25">PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) </TD></TR> + <TR><TD ALIGN="left" PORT="v26">v26:Fixnum = GuardType v11, Fixnum </TD></TR> + <TR><TD ALIGN="left" PORT="v27">v27:Fixnum = GuardType v12, Fixnum </TD></TR> + <TR><TD ALIGN="left" PORT="v28">v28:Fixnum = FixnumOr v26, v27 </TD></TR> + <TR><TD ALIGN="left" PORT="v29">IncrCounter inline_cfunc_optimized_send_count </TD></TR> + <TR><TD ALIGN="left" PORT="v21">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v22">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v23">Return v28 </TD></TR> + </TABLE>>]; + } + "#); + } + + #[test] + fn test_multiple_blocks() { + eval(r#" + def test(c) + if c + 3 + else + 4 + end + end + + test(1) + test("x") + "#); + assert_snapshot!(hir_string("test"), @r#" + digraph G { # test@<compiled>:3 + node [shape=plaintext]; + mode=hier; overlap=false; splines=true; + bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0() </TD></TR> + <TR><TD ALIGN="left" PORT="v0">EntryPoint interpreter </TD></TR> + <TR><TD ALIGN="left" PORT="v1">v1:BasicObject = LoadSelf </TD></TR> + <TR><TD ALIGN="left" PORT="v2">v2:BasicObject = GetLocal :c, l0, SP@4 </TD></TR> + <TR><TD ALIGN="left" PORT="v3">Jump bb2(v1, v2) </TD></TR> + </TABLE>>]; + bb0:v3 -> bb2:params:n; + bb1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb1(v5:BasicObject, v6:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v4">EntryPoint JIT(0) </TD></TR> + <TR><TD ALIGN="left" PORT="v7">Jump bb2(v5, v6) </TD></TR> + </TABLE>>]; + bb1:v7 -> bb2:params:n; + bb2 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb2(v8:BasicObject, v9:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v12">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v14">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v15">v15:CBool = Test v9 </TD></TR> + <TR><TD ALIGN="left" PORT="v16">v16:Falsy = RefineType v9, Falsy </TD></TR> + <TR><TD ALIGN="left" PORT="v17">IfFalse v15, bb3(v8, v16) </TD></TR> + <TR><TD ALIGN="left" PORT="v18">v18:Truthy = RefineType v9, Truthy </TD></TR> + <TR><TD ALIGN="left" PORT="v20">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v21">v21:Fixnum[3] = Const Value(3) </TD></TR> + <TR><TD ALIGN="left" PORT="v23">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v24">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v25">Return v21 </TD></TR> + </TABLE>>]; + bb2:v17 -> bb3:params:n; + bb3 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb3(v26:BasicObject, v27:Falsy) </TD></TR> + <TR><TD ALIGN="left" PORT="v30">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v31">v31:Fixnum[4] = Const Value(4) </TD></TR> + <TR><TD ALIGN="left" PORT="v33">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v34">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v35">Return v31 </TD></TR> + </TABLE>>]; + } + "#); + } +} diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs new file mode 100644 index 0000000000..da24025010 --- /dev/null +++ b/zjit/src/hir/opt_tests.rs @@ -0,0 +1,11261 @@ +#[cfg(test)] +mod hir_opt_tests { + use crate::hir::*; + + use crate::{hir_strings, options::*}; + use insta::assert_snapshot; + use crate::hir::tests::hir_build_tests::assert_contains_opcode; + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[test] + fn test_fold_iftrue_away() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_iftrue_into_jump() { + eval(" + def test + cond = false + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:FalseClass = Const Value(false) + CheckInterrupts + v20:FalseClass = RefineType v13, Falsy + v35:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_fold_fixnum_add() { + eval(" + def test + 1 + 2 + 3 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v33:Fixnum[3] = Const Value(3) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v34:Fixnum[6] = Const Value(6) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_fixnum_sub() { + eval(" + def test + 5 - 3 - 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v33:Fixnum[2] = Const Value(2) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v34:Fixnum[1] = Const Value(1) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_fixnum_sub_large_negative_result() { + eval(" + def test + 0 - 1073741825 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[1073741825] = Const Value(1073741825) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v24:Fixnum[-1073741825] = Const Value(-1073741825) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult() { + eval(" + def test + 6 * 7 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[6] = Const Value(6) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v24:Fixnum[42] = Const Value(42) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult_zero() { + eval(" + def test(n) + 0 * n + n * 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :n, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v33:Fixnum = GuardType v9, Fixnum + v45:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + v20:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v38:Fixnum = GuardType v9, Fixnum + v46:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint MethodRedefined(Integer@0x1000, +@0x1038, cme:0x1040) + v47:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v47 + "); + } + + #[test] + fn test_fold_fixnum_less() { + eval(" + def test + if 1 < 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_less_equal() { + eval(" + def test + if 1 <= 2 && 2 <= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v59:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v23:Fixnum[2] = Const Value(2) + v25:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v61:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_greater() { + eval(" + def test + if 2 > 1 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_greater_equal() { + eval(" + def test + if 2 >= 1 && 2 >= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v59:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v23:Fixnum[2] = Const Value(2) + v25:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v61:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_eq_false() { + eval(" + def test + if 1 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_eq_true() { + eval(" + def test + if 2 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_true() { + eval(" + def test + if 1 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_false() { + eval(" + def test + if 2 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn neq_with_side_effect_not_elided () { + let result = eval(" + class CustomEq + attr_reader :count + + def ==(o) + @count = @count.to_i + 1 + self.equal?(o) + end + end + + def test(object) + # intentionally unused, but also can't assign to underscore + object != object + nil + end + + custom = CustomEq.new + test(custom) + test(custom) + + custom.count + "); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :object, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(CustomEq@0x1000) + PatchPoint MethodRedefined(CustomEq@0x1000, !=@0x1008, cme:0x1010) + v28:HeapObject[class_exact:CustomEq] = GuardType v9, HeapObject[class_exact:CustomEq] + v29:BoolExact = CCallWithFrame v28, :BasicObject#!=@0x1038, v9 + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_replace_guard_if_known_fixnum() { + eval(" + def test(a) + a + 1 + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_param_forms_get_bb_param() { + eval(" + def rest(*array) = array + def kw(k:) = k + def kw_rest(**k) = k + def post(*rest, post) = post + def block(&b) = nil + "); + assert_snapshot!(hir_strings!("rest", "kw", "kw_rest", "block", "post"), @r" + fn rest@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :array, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + CheckInterrupts + Return v9 + + fn kw@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k, l0, SP@5 + v3:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:BasicObject = GetLocal <empty>, l0, EP@3 + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + CheckInterrupts + Return v11 + + fn kw_rest@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + Return v9 + + fn block@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + CheckInterrupts + Return v13 + + fn post@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :rest, l0, SP@5, * + v3:BasicObject = GetLocal :post, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:ArrayExact, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:ArrayExact, v12:BasicObject): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_optimize_top_level_call_into_send_direct() { + eval(" + def foo = [] + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_iseq() { + eval(" + def foo = 1 + alias bar foo + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_cfunc() { + eval(" + alias bar itself + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_inline_nonparam_local_return() { + // Methods that return non-parameter local variables should NOT be inlined, + // because the local variable index will be out of bounds for args. + // The method must have a parameter so param_size > 0, and return a local + // that's not a parameter so local_idx >= param_size. + // Use dead code (if false) to create a local without initialization instructions, + // resulting in just getlocal + leave which enters the inlining code path. + eval(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc() { + eval(" + class C < Array + alias fun_new_map map + end + def test(o) = o.fun_new_map {|e| e } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, fun_new_map@0x1008, cme:0x1010) + v23:ArraySubclass[class_exact:C] = GuardType v9, ArraySubclass[class_exact:C] + v24:BasicObject = CCallWithFrame v23, :C#fun_new_map@0x1038, block=0x1040 + v15:BasicObject = GetLocal :o, l0, EP@3 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_nonexistent_top_level_call() { + eval(" + def foo + end + def test + foo + end + test; test + undef :foo + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: SendWithoutBlock: unsupported method type Null + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_private_top_level_call() { + eval(" + def foo = [] + private :foo + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_top_level_call_with_overloaded_cme() { + eval(" + def test + Integer(3) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :Integer (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_top_level_call_with_args_into_send_direct() { + eval(" + def foo(a, b) = [] + def test + foo 1, 2 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_top_level_sends_into_send_direct() { + eval(" + def foo = [] + def bar = [] + def test + foo + bar + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v23:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v24:BasicObject = SendWithoutBlockDirect v23, :foo (0x1038) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, bar@0x1040, cme:0x1048) + v27:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v28:BasicObject = SendWithoutBlockDirect v27, :bar (0x1038) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_direct_no_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_direct_one_optional_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_direct_all_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3, 4 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_call_with_correct_and_too_many_args_for_method() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4) = [a, b, c, d] + def test = [target(), target(10, 20, 30), begin; target(10, 20, 30, 40, 50) rescue ArgumentError; end] + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v44:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v45:BasicObject = SendWithoutBlockDirect v44, :target (0x1038) + v14:Fixnum[10] = Const Value(10) + v16:Fixnum[20] = Const Value(20) + v18:Fixnum[30] = Const Value(30) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v49:BasicObject = SendWithoutBlockDirect v48, :target (0x1038), v14, v16, v18 + v24:Fixnum[10] = Const Value(10) + v26:Fixnum[20] = Const Value(20) + v28:Fixnum[30] = Const Value(30) + v30:Fixnum[40] = Const Value(40) + v32:Fixnum[50] = Const Value(50) + v34:BasicObject = SendWithoutBlock v6, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count + v37:ArrayExact = NewArray v45, v49, v34 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_optimize_variadic_ccall() { + eval(" + def test + puts 'Hello' + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, puts@0x1010, cme:0x1018) + v22:HeapObject[class_exact*:Object@VALUE(0x1008)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1008)] + v23:BasicObject = CCallVariadic v22, :Kernel#puts@0x1040, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_fixnum_add_if_redefined() { + eval(" + class Integer + def +(other) + 100 + end + end + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + IncrCounter inline_iseq_optimized_send_count + v28:Fixnum[100] = Const Value(100) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_both_profiled() { + eval(" + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAdd v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_left_profiled() { + eval(" + def test(a) = a + 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_right_profiled() { + eval(" + def test(a) = 1 + a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v13, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn integer_aref_with_fixnum_emits_fixnum_aref() { + eval(" + def test(a, b) = a[b] + test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAref v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn elide_fixnum_aref() { + eval(" + def test + 1[2] + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_too_many_args() { + eval(" + def test = 1[2, 3] + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v14:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v23:BasicObject = CCallVariadic v10, :Integer#[]@0x1038, v12, v14 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_non_fixnum() { + eval(r#" + def test = 1["x"] + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v23:BasicObject = CCallVariadic v10, :Integer#[]@0x1040, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_both_profiled() { + eval(" + def test(a, b) = a < b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:BoolExact = FixnumLt v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_left_profiled() { + eval(" + def test(a) = a < 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_right_profiled() { + eval(" + def test(a) = 1 < a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v13, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_literals() { + eval(" + def test() + a = 2 + (1..a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v25:RangeExact = NewRangeFixnum v17 NewRangeInclusive v13 + CheckInterrupts + Return v25 + "); + } + + + #[test] + fn test_optimize_new_range_fixnum_exclusive_literals() { + eval(" + def test() + a = 2 + (1...a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v25:RangeExact = NewRangeFixnum v17 NewRangeExclusive v13 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_high_guarded() { + eval(" + def test(a) + (1..a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v13 NewRangeInclusive v21 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_high_guarded() { + eval(" + def test(a) + (1...a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v13 NewRangeExclusive v21 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_low_guarded() { + eval(" + def test(a) + (a..10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v21 NewRangeInclusive v14 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_low_guarded() { + eval(" + def test(a) + (a...10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v21 NewRangeExclusive v14 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_eliminate_new_array() { + eval(" + def test() + c = [] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_opt_aref_array() { + eval(" + arr = [1,2,3] + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []@0x1008, cme:0x1010) + v25:ArrayExact = GuardType v9, ArrayExact + v26:CInt64[0] = UnboxFixnum v14 + v27:CInt64 = ArrayLength v25 + v28:CInt64[0] = GuardLess v26, v27 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64[0] = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v25, v30 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + assert_snapshot!(inspect("test [1,2,3]"), @"1"); + } + + #[test] + fn test_opt_aref_hash() { + eval(" + arr = {0 => 4} + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []@0x1008, cme:0x1010) + v25:HashExact = GuardType v9, HashExact + v26:BasicObject = HashAref v25, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + assert_snapshot!(inspect("test({0 => 4})"), @"4"); + } + + #[test] + fn test_eliminate_new_range() { + eval(" + def test() + c = (1..2) + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:RangeExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_do_not_eliminate_new_range_non_fixnum() { + eval(" + def test() + _ = (-'a'..'b') + 0 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:StringExact = StringCopy v16 + v19:RangeExact = NewRange v14 NewRangeInclusive v17 + PatchPoint NoEPEscape(test) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_new_array_with_elements() { + eval(" + def test(a) + c = [a] + 5 + end + test(1); test(2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v17:ArrayExact = NewArray v11 + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_new_hash() { + eval(" + def test() + c = {} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_eliminate_new_hash_with_elements() { + eval(" + def test(aval, bval) + c = {a: aval, b: bval} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :aval, l0, SP@6 + v3:BasicObject = GetLocal :bval, l0, SP@5 + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:NilClass): + v19:StaticSymbol[:a] = Const Value(VALUE(0x1000)) + v22:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v25:HashExact = NewHash v19: v13, v22: v14 + PatchPoint NoEPEscape(test) + v31:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_array_dup() { + eval(" + def test + c = [1, 2] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_hash_dup() { + eval(" + def test + c = {a: 1, b: 2} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_putself() { + eval(" + def test() + c = self + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_eliminate_string_copy() { + eval(r#" + def test() + c = "abc" + 5 + end + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_fixnum_add() { + eval(" + def test(a, b) + a + b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_sub() { + eval(" + def test(a, b) + a - b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_mul() { + eval(" + def test(a, b) + a * b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_div() { + eval(" + def test(a, b) + a / b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumDiv v30, v31 + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_mod() { + eval(" + def test(a, b) + a % b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumMod v30, v31 + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_lt() { + eval(" + def test(a, b) + a < b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_le() { + eval(" + def test(a, b) + a <= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_gt() { + eval(" + def test(a, b) + a > b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_ge() { + eval(" + def test(a, b) + a >= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_eq() { + eval(" + def test(a, b) + a == b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_neq() { + eval(" + def test(a, b) + a != b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v32:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_get_constant_path() { + eval(" + def test() + C + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn kernel_itself_const() { + eval(" + def test(x) = x.itself + test(0) # profile + test(1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + v21:Fixnum = GuardType v9, Fixnum + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn kernel_itself_known_type() { + eval(" + def test = [].itself + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v10 + "); + } + + #[test] + fn eliminate_kernel_itself() { + eval(" + def test + x = [].itself + 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_module_name() { + eval(" + module M; end + def test + x = M.name + 1 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v29:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v34:StringExact|NilClass = CCall v29, :Module#name@0x1048 + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn eliminate_array_length() { + eval(" + def test + [].length + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn normal_class_type_inference() { + eval(" + class C; end + def test = C + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v18:Class[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn core_classes_type_inference() { + eval(" + def test = [String, Class, Module, BasicObject] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v29:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, Class) + v32:Class[Class@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1020, Module) + v35:Class[Module@0x1028] = Const Value(VALUE(0x1028)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1030, BasicObject) + v38:Class[BasicObject@0x1038] = Const Value(VALUE(0x1038)) + v22:ArrayExact = NewArray v29, v32, v35, v38 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn module_instances_are_module_exact() { + eval(" + def test = [Enumerable, Kernel] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Enumerable) + v23:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, Kernel) + v26:ModuleExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + v16:ArrayExact = NewArray v23, v26 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn module_subclasses_are_not_module_exact() { + eval(" + class ModuleSubclass < Module; end + MY_MODULE = ModuleSubclass.new + def test = MY_MODULE + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_MODULE) + v18:ModuleSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn eliminate_array_size() { + eval(" + def test + [].size + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn kernel_itself_argc_mismatch() { + eval(" + def test = 1.itself(0) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[0] = Const Value(0) + v14:BasicObject = SendWithoutBlock v10, :itself, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_kernel_block_given_p() { + eval(" + def test = block_given? + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v20:CPtr = GetLEP + v21:BoolExact = IsBlockGiven v20 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_kernel_block_given_p_in_block() { + eval(" + TEST = proc { block_given? } + TEST.call + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn block in <compiled>@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v20:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_elide_kernel_block_given_p() { + eval(" + def test + block_given? + 5 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v23:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_cfunc_optimized_send_count + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn const_send_direct_integer() { + eval(" + def test(x) = 1.zero? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, zero?@0x1008, cme:0x1010) + IncrCounter inline_iseq_optimized_send_count + v23:BasicObject = InvokeBuiltin leaf <inline_expr>, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn class_known_send_direct_array() { + eval(" + def test(x) + a = [1,2,3] + a.first + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:ArrayExact = ArrayDup v16 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, first@0x1010, cme:0x1018) + IncrCounter inline_iseq_optimized_send_count + v31:BasicObject = InvokeBuiltin leaf <inline_expr>, v17 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn send_direct_to_module() { + eval(" + module M; end + def test = M.class + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v20:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020) + IncrCounter inline_iseq_optimized_send_count + v26:Class[Module@0x1010] = Const Value(VALUE(0x1010)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_to_instance_method() { + eval(" + class C + def foo = [] + end + + def test(c) = c.foo + c = C.new + test c + test c + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :c, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:BasicObject = SendWithoutBlockDirect v21, :foo (0x1038) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_block() { + eval(" + def foo(&block) = 1 + def test = foo {|| } + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + CheckInterrupts + Return v11 + "); + } + + #[test] + fn reload_local_across_send() { + eval(" + def foo(&block) = 1 + def test + a = 1 + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + SetLocal :a, l0, EP@3, v13 + v19:BasicObject = Send v8, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + v20:BasicObject = GetLocal :a, l0, EP@3 + v24:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_rest() { + eval(" + def foo(*args) = 1 + def test = foo 1 + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_rest + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_specialize_call_to_post_param_iseq() { + eval(" + def foo(opt=80, post) = post + def test = foo(10) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + IncrCounter complex_arg_pass_param_post + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn specialize_call_to_iseq_with_multiple_required_kw() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_to_iseq_with_required_kw_reorder() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1038), v13, v15, v11 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_to_iseq_with_positional_and_required_kw_reorder() { + eval(" + def foo(x, a:, b:) = [x, a, b] + def test = foo(0, b: 2, a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1038), v11, v15, v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_with_positional_and_optional_kw() { + eval(" + def foo(x, a: 1) = [x, a] + def test = foo(0, a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_req_kw() { + eval(" + def foo(r, x = 2, a:, b:) = [x, a] + def test = [foo(1, a: 3, b: 4), foo(1, 2, b: 4, a: 3)] # with and without the optional, change kw order + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v15:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v38:BasicObject = SendWithoutBlockDirect v37, :foo (0x1038), v11, v13, v15 + v20:Fixnum[1] = Const Value(1) + v22:Fixnum[2] = Const Value(2) + v24:Fixnum[4] = Const Value(4) + v26:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v41:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v43:BasicObject = SendWithoutBlockDirect v41, :foo (0x1038), v20, v22, v26, v24 + v30:ArrayExact = NewArray v38, v43 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_kw_optional() { + eval(" + def foo(r, x = 2, a:, b: 4) = [r, x, a, b] + def test = [foo(1, a: 3), foo(1, 2, b: 40, a: 30)] # with and without the optionals + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v35:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v36:Fixnum[4] = Const Value(4) + v38:BasicObject = SendWithoutBlockDirect v35, :foo (0x1038), v11, v13, v36 + v18:Fixnum[1] = Const Value(1) + v20:Fixnum[2] = Const Value(2) + v22:Fixnum[40] = Const Value(40) + v24:Fixnum[30] = Const Value(30) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v41:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v43:BasicObject = SendWithoutBlockDirect v41, :foo (0x1038), v18, v20, v24, v22 + v28:ArrayExact = NewArray v38, v43 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_call_with_pos_optional_and_maybe_too_many_args() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[6] = Const Value(6) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v49:BasicObject = SendWithoutBlockDirect v48, :target (0x1038), v11 + v16:Fixnum[10] = Const Value(10) + v18:Fixnum[20] = Const Value(20) + v20:Fixnum[30] = Const Value(30) + v22:Fixnum[6] = Const Value(6) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v52:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v53:BasicObject = SendWithoutBlockDirect v52, :target (0x1038), v16, v18, v20, v22 + v27:Fixnum[10] = Const Value(10) + v29:Fixnum[20] = Const Value(20) + v31:Fixnum[30] = Const Value(30) + v33:Fixnum[40] = Const Value(40) + v35:Fixnum[50] = Const Value(50) + v37:Fixnum[60] = Const Value(60) + v39:BasicObject = SendWithoutBlock v6, :target, v27, v29, v31, v33, v35, v37 # SendFallbackReason: Too many arguments for LIR + v41:ArrayExact = NewArray v49, v53, v39 + CheckInterrupts + Return v41 + "); + } + + #[test] + fn test_send_call_to_iseq_with_optional_kw() { + eval(" + def foo(a: 1) = a + def test = foo(a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_kwrest() { + eval(" + def foo(**args) = 1 + def test = foo(a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_kwrest + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn specialize_call_to_iseq_with_optional_param_kw_using_default() { + eval(" + def foo(int: 1) = int + 1 + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:Fixnum[1] = Const Value(1) + v21:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038), v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_call_kwsplat() { + eval(" + def foo(a:) = a + def test = foo(**{a: 1}) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + IncrCounter complex_arg_pass_caller_kw_splat + v14:BasicObject = SendWithoutBlock v6, :foo, v12 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v14 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_param_kwrest() { + eval(" + def foo(**kwargs) = kwargs.keys + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_kwrest + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_optimize_ccall_with_kwarg() { + eval(" + def test = sprintf('%s', a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = SendWithoutBlock v6, :sprintf, v12, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn dont_optimize_ccall_with_block_and_kwarg() { + eval(" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + test %(a\nb\nc) + test %() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact = NewArray + SetLocal :a, l0, EP@3, v16 + v22:TrueClass = Const Value(true) + IncrCounter complex_arg_pass_caller_kwarg + v24:BasicObject = Send v11, 0x1000, :each_line, v22 # SendFallbackReason: Complex argument passing + v25:BasicObject = GetLocal :s, l0, EP@4 + v26:BasicObject = GetLocal :a, l0, EP@3 + v30:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_empty_ic() { + eval(" + def test = Kernel + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_invalidated_ic() { + eval(" + def test = Kernel + test + Kernel = 5 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn replace_get_constant_path_with_const() { + eval(" + def test = Kernel + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v18:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn replace_nested_get_constant_path_with_const() { + eval(" + module Foo + module Bar + class C + end + end + end + def test = Foo::Bar::C + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo::Bar::C) + v18:Class[Foo::Bar::C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_no_initialize() { + eval(" + class C; end + def test = C.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v43:Class[C@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v46:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_initialize() { + eval(" + class C + def initialize x + @x = x + end + end + def test = C.new 1 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v46:Class[C@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v49:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v52:BasicObject = SendWithoutBlockDirect v49, :initialize (0x1068), v16 + CheckInterrupts + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_object() { + eval(" + def test = Object.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Object) + v43:Class[Object@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Object@0x1008, new@0x1009, cme:0x1010) + v46:ObjectExact = ObjectAllocClass Object:VALUE(0x1008) + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_basic_object() { + eval(" + def test = BasicObject.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, BasicObject) + v43:Class[BasicObject@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(BasicObject@0x1008, new@0x1009, cme:0x1010) + v46:BasicObjectExact = ObjectAllocClass BasicObject:VALUE(0x1008) + PatchPoint NoSingletonClass(BasicObject@0x1008) + PatchPoint MethodRedefined(BasicObject@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_hash() { + eval(" + def test = Hash.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Hash) + v43:Class[Hash@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Hash@0x1008, new@0x1009, cme:0x1010) + v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) + IncrCounter complex_arg_pass_param_block + v20:BasicObject = SendWithoutBlock v46, :initialize # SendFallbackReason: Complex argument passing + CheckInterrupts + CheckInterrupts + Return v46 + "); + assert_snapshot!(inspect("test"), @"{}"); + } + + #[test] + fn test_opt_new_array() { + eval(" + def test = Array.new 1 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Array) + v46:Class[Array@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Array@0x1008, new@0x1009, cme:0x1010) + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v57:BasicObject = CCallVariadic v46, :Array.new@0x1040, v16 + CheckInterrupts + Return v57 + "); + } + + #[test] + fn test_opt_new_set() { + eval(" + def test = Set.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Set) + v43:Class[Set@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Set@0x1008, new@0x1009, cme:0x1010) + v18:HeapBasicObject = ObjectAlloc v43 + PatchPoint NoSingletonClass(Set@0x1008) + PatchPoint MethodRedefined(Set@0x1008, initialize@0x1038, cme:0x1040) + v49:SetExact = GuardType v18, SetExact + v50:BasicObject = CCallVariadic v49, :Set#initialize@0x1068 + CheckInterrupts + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_string() { + eval(" + def test = String.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v43:Class[String@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(String@0x1008, new@0x1009, cme:0x1010) + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v54:BasicObject = CCallVariadic v43, :String.new@0x1040 + CheckInterrupts + Return v54 + "); + } + + #[test] + fn test_opt_new_regexp() { + eval(" + def test = Regexp.new '' + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Regexp) + v47:Class[Regexp@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v17:StringExact = StringCopy v16 + PatchPoint MethodRedefined(Regexp@0x1008, new@0x1018, cme:0x1020) + v50:RegexpExact = ObjectAllocClass Regexp:VALUE(0x1008) + PatchPoint NoSingletonClass(Regexp@0x1008) + PatchPoint MethodRedefined(Regexp@0x1008, initialize@0x1048, cme:0x1050) + v54:BasicObject = CCallVariadic v50, :Regexp#initialize@0x1078, v17 + CheckInterrupts + CheckInterrupts + Return v50 + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v29:CInt64 = ArrayLength v18 + v30:Fixnum = BoxFixnum v29 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v29:CInt64 = ArrayLength v18 + v30:Fixnum = BoxFixnum v29 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + GuardBlockParamProxy l0 + v15:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + v17:BasicObject = Send v8, 0x1008, :tap, v15 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + IncrCounter getivar_fallback_not_monomorphic + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_monomorphic + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_true() { + eval(" + @foo = 4 + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v15:HeapBasicObject = GuardType v6, HeapBasicObject + v16:CShape = LoadField v15, :_shape_id@0x1000 + v17:CShape[0x1001] = GuardBitEquals v16, CShape(0x1001) + v18:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_false() { + eval(" + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v15:HeapBasicObject = GuardType v6, HeapBasicObject + v16:CShape = LoadField v15, :_shape_id@0x1000 + v17:CShape[0x1001] = GuardBitEquals v16, CShape(0x1001) + v18:NilClass = Const Value(nil) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_proc_call() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.call(1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, call@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_proc_aref() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p[2] + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, []@0x1008, cme:0x1010) + v24:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v25:BasicObject = InvokeProc v24, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_proc_yield() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.yield(3) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, yield@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_proc_eqq() { + eval(" + p = proc { |x| x > 0 } + def test(p) + p === 1 + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, ===@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_specialize_proc_call_splat() { + eval(" + p = proc { } + def test(p) + empty = [] + p.call(*empty) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact = NewArray + v22:ArrayExact = ToArray v16 + IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = SendWithoutBlock v11, :call, v22 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_specialize_proc_call_kwarg() { + eval(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_caller_kwarg + v16:BasicObject = SendWithoutBlock v9, :call, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_dont_specialize_definedivar_with_t_data() { + eval(" + class C < Range + def test = defined?(@a) + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_not_t_object + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_definedivar() { + set_call_threshold(3); + eval(" + class C + def test = defined?(@a) + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_not_monomorphic + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_definedivar() { + eval(r#" + class C + def test = defined?(@a) + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_too_complex + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_already_in_shape() { + eval(" + @foo = 4 + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v19:HeapBasicObject = GuardType v6, HeapBasicObject + v20:CShape = LoadField v19, :_shape_id@0x1000 + v21:CShape[0x1001] = GuardBitEquals v20, CShape(0x1001) + StoreField v19, :@foo@0x1002, v10 + WriteBarrier v19, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_with_shape_transition() { + eval(" + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v19:HeapBasicObject = GuardType v6, HeapBasicObject + v20:CShape = LoadField v19, :_shape_id@0x1000 + v21:CShape[0x1001] = GuardBitEquals v20, CShape(0x1001) + StoreField v19, :@foo@0x1002, v10 + WriteBarrier v19, v10 + v24:CShape[0x1003] = Const CShape(0x1003) + StoreField v19, :_shape_id@0x1000, v24 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_multiple_monomorphic_setivar_with_shape_transition() { + eval(" + def test + @foo = 1 + @bar = 2 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v25:HeapBasicObject = GuardType v6, HeapBasicObject + v26:CShape = LoadField v25, :_shape_id@0x1000 + v27:CShape[0x1001] = GuardBitEquals v26, CShape(0x1001) + StoreField v25, :@foo@0x1002, v10 + WriteBarrier v25, v10 + v30:CShape[0x1003] = Const CShape(0x1003) + StoreField v25, :_shape_id@0x1000, v30 + v16:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + v32:HeapBasicObject = GuardType v6, HeapBasicObject + v33:CShape = LoadField v32, :_shape_id@0x1000 + v34:CShape[0x1003] = GuardBitEquals v33, CShape(0x1003) + StoreField v32, :@bar@0x1004, v16 + WriteBarrier v32, v16 + v37:CShape[0x1005] = Const CShape(0x1005) + StoreField v32, :_shape_id@0x1000, v37 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_dont_specialize_setivar_with_t_data() { + eval(" + class C < Range + def test = @a = 5 + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_t_object + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_setivar() { + set_call_threshold(3); + eval(" + class C + def test = @a = 5 + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_monomorphic + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_setivar() { + eval(r#" + class C + def test = @a = 5 + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_too_complex + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_setivar_when_next_shape_is_too_complex() { + eval(r#" + class AboutToBeTooComplex + def test = @abc = 5 + end + SHAPE_MAX_VARIATIONS = 8 # see shape.h + SHAPE_MAX_VARIATIONS.times do + AboutToBeTooComplex.new.instance_variable_set(:"@a#{_1}", 1) + end + AboutToBeTooComplex.new.test + TEST = AboutToBeTooComplex.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_new_shape_too_complex + SetIvar v6, :@abc, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_hash() { + eval(" + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_hash_freeze_if_redefined() { + eval(" + class Hash + def freeze; end + end + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_hash() { + eval(" + def test = {}.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_hash() { + eval(" + def test = {}.dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, dup@0x1008, cme:0x1010) + v22:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = SendWithoutBlock v22, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_hash_with_args() { + eval(" + def test = {}.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + v12:NilClass = Const Value(nil) + v14:BasicObject = SendWithoutBlock v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_ary() { + eval(" + def test = [].freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_ary() { + eval(" + def test = [].freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_ary() { + eval(" + def test = [].dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, dup@0x1008, cme:0x1010) + v22:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = SendWithoutBlock v22, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_ary_with_args() { + eval(" + def test = [].freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + v12:NilClass = Const Value(nil) + v14:BasicObject = SendWithoutBlock v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_str() { + eval(" + def test = ''.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_str() { + eval(" + def test = ''.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_str() { + eval(" + def test = ''.dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v23:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = SendWithoutBlock v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_elide_freeze_str_with_args() { + eval(" + def test = ''.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:NilClass = Const Value(nil) + v15:BasicObject = SendWithoutBlock v11, :freeze, v13 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_elide_uminus_with_frozen_str() { + eval(" + def test = -'' + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_uminus_with_refrozen_str() { + eval(" + def test = -''.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_uminus_with_unfrozen_str() { + eval(" + def test = -''.dup + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v23:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = SendWithoutBlock v23, :-@ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_objtostring_anytostring_string() { + eval(r##" + def test = "#{('foo')}" + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + v21:StringExact = StringConcat v10, v14 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_objtostring_anytostring_with_non_string() { + eval(r##" + def test = "#{1}" + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled() { + eval(" + def test(a) + \"#{a}\" + end + test('foo'); test('foo') + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + v27:String = GuardType v9, String + v21:StringExact = StringConcat v13, v27 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled_string_subclass() { + eval(" + class MyString < String; end + + def test(a) + \"#{a}\" + end + foo = MyString.new('foo') + test(MyString.new(foo)); test(MyString.new(foo)) + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(MyString@0x1008) + v27:String = GuardType v9, String + v21:StringExact = StringConcat v13, v27 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_objtostring_profiled_nonstring_falls_back_to_send() { + eval(" + def test(a) + \"#{a}\" + end + test([1,2,3]); test([1,2,3]) # No fast path for array + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:ArrayExact = GuardType v9, ArrayExact + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, to_s@0x1010, cme:0x1018) + v31:BasicObject = CCallWithFrame v26, :Array#to_s@0x1040 + v19:String = AnyToString v9, str: v31 + v21:StringExact = StringConcat v13, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_branchnil_nil() { + eval(" + def test + x = nil + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + CheckInterrupts + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_branchnil_truthy() { + eval(" + def test + x = 1 + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + v23:Fixnum[1] = RefineType v13, NotNil + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_eliminate_load_from_non_frozen_array() { + eval(r##" + S = [4,5,6] + def test = S[0] + test + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v27:CInt64[0] = UnboxFixnum v13 + v28:CInt64 = ArrayLength v23 + v29:CInt64[0] = GuardLess v27, v28 + v30:CInt64[0] = Const CInt64(0) + v31:CInt64[0] = GuardGreaterEq v29, v30 + v32:BasicObject = ArrayAref v23, v31 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + // TODO(max): Check the result of `S[0] = 5; test` using `inspect` to make sure that we + // actually do the load at run-time. + } + + #[test] + fn test_eliminate_load_from_frozen_array_in_bounds() { + eval(r##" + def test = [4,5,6].freeze[1] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[1] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[1] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[1] = GuardGreaterEq v26, v27 + v31:Fixnum[5] = Const Value(5) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative() { + eval(r##" + def test = [4,5,6].freeze[-3] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-3] = Const Value(-3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[-3] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[-3] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[-3] = GuardGreaterEq v26, v27 + v31:Fixnum[4] = Const Value(4) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[-10] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[-10] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[-10] = GuardGreaterEq v26, v27 + v31:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[10] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[10] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[10] = GuardGreaterEq v26, v27 + v31:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_dont_optimize_array_aref_if_redefined() { + eval(r##" + class Array + def [](index) = [] + end + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:BasicObject = SendWithoutBlockDirect v11, :[] (0x1040), v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_array_aset_if_redefined() { + eval(r##" + class Array + def []=(*args); :redefined; end + end + + def test(arr) + arr[1] = 10 + end + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[10] = Const Value(10) + v22:BasicObject = SendWithoutBlock v9, :[]=, v16, v18 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_dont_optimize_array_max_if_redefined() { + eval(r##" + class Array + def max = [] + end + def test = [4,5,6].max + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) + v20:BasicObject = SendWithoutBlockDirect v11, :max (0x1040) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_set_type_from_constant() { + eval(" + MY_SET = Set.new + + def test = MY_SET + + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_SET) + v18:SetExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_regexp_type() { + eval(" + def test = /a/ + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_bmethod_send_direct() { + eval(" + define_method(:zero) { :b } + define_method(:one) { |arg| arg } + + def test = one(zero) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, zero@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v30:StaticSymbol[:b] = Const Value(VALUE(0x1038)) + PatchPoint SingleRactorMode + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, one@0x1040, cme:0x1048) + v27:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_symbol_block_bmethod() { + eval(" + define_method(:identity, &:itself) + def test = identity(100) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[100] = Const Value(100) + v13:BasicObject = SendWithoutBlock v6, :identity, v11 # SendFallbackReason: Bmethod: Proc object is not defined by an ISEQ + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_call_bmethod_with_block() { + eval(" + define_method(:bmethod) { :b } + def test = (bmethod {}) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :bmethod # SendFallbackReason: Send: unsupported method type Bmethod + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_call_shareable_bmethod() { + eval(" + class Foo + class << self + define_method(:identity, &(Ractor.make_shareable ->(val){val})) + end + end + def test = Foo.identity(100) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo) + v22:Class[Foo@0x1008] = Const Value(VALUE(0x1008)) + v13:Fixnum[100] = Const Value(100) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, identity@0x1018, cme:0x1020) + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nil_nil_specialized_to_ccall() { + eval(" + def test = nil.nil? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v20:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_eliminate_nil_nil_specialized_to_ccall() { + eval(" + def test + nil.nil? + 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_non_nil_nil_specialized_to_ccall() { + eval(" + def test = 1.nil? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v20:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_eliminate_non_nil_nil_specialized_to_ccall() { + eval(" + def test + 1.nil? + 2 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_guard_nil_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v22:NilClass = GuardType v9, NilClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_false_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(false) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1000, nil?@0x1008, cme:0x1010) + v22:FalseClass = GuardType v9, FalseClass + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_true_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(true) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1000, nil?@0x1008, cme:0x1010) + v22:TrueClass = GuardType v9, TrueClass + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_symbol_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(:foo) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1000, nil?@0x1008, cme:0x1010) + v22:StaticSymbol = GuardType v9, StaticSymbol + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_fixnum_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v22:Fixnum = GuardType v9, Fixnum + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_float_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1.0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Float@0x1000, nil?@0x1008, cme:0x1010) + v22:Flonum = GuardType v9, Flonum + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_string_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test('foo') + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, nil?@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + v24:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_basicobject_not_truthy() { + eval(" + def test(a) = !a + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, !@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_basicobject_not_false() { + eval(" + def test(a) = !a + + test(false) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1000, !@0x1008, cme:0x1010) + v22:FalseClass = GuardType v9, FalseClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_basicobject_not_nil() { + eval(" + def test(a) = !a + + test(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1000, !@0x1008, cme:0x1010) + v22:NilClass = GuardType v9, NilClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_basicobject_not_falsy() { + eval(" + def test(a) = !(if a then false else nil end) + + # TODO(max): Make this not GuardType NilClass and instead just reason + # statically + test(false) + test(true) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + v20:FalseClass = Const Value(false) + CheckInterrupts + Jump bb4(v8, v18, v20) + bb3(v24:BasicObject, v25:Falsy): + v28:NilClass = Const Value(nil) + Jump bb4(v24, v25, v28) + bb4(v30:BasicObject, v31:BasicObject, v32:Falsy): + PatchPoint MethodRedefined(NilClass@0x1000, !@0x1008, cme:0x1010) + v43:NilClass = GuardType v32, NilClass + v44:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_specialize_array_empty_p() { + eval(" + def test(a) = a.empty? + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, empty?@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:CInt64[0] = Const CInt64(0) + v26:CBool = IsBitEqual v24, v25 + v27:BoolExact = BoxBool v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_specialize_hash_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test({}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, empty?@0x1008, cme:0x1010) + v23:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v25:BoolExact = CCall v23, :Hash#empty?@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basic_object_eq_to_ccall() { + eval(" + class C; end + def test(a, b) = a == b + + test(C.new, C.new) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010) + v27:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v28:CBool = IsBitEqual v27, v12 + v29:BoolExact = BoxBool v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_guard_fixnum_and_fixnum() { + eval(" + def test(x, y) = x & y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAnd v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_guard_fixnum_or_fixnum() { + eval(" + def test(x, y) = x | y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumOr v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_method_redefinition_patch_point_on_top_level_method() { + eval(" + def foo; end + def test = foo + + test; test + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_embedded() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:BasicObject = LoadField v21, :@foo@0x103a + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_getivar_extended() { + eval(r#" + class C + attr_reader :foo + def initialize + 1000.times do |i| + instance_variable_set("@v#{i}", i) + end + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:CPtr = LoadField v21, :_as_heap@0x103a + v27:BasicObject = LoadField v26, :@foo@0x103b + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_getivar_on_module() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_class() { + eval(" + class C + @foo = 42 + def self.test = @foo + end + C.test + "); + assert_snapshot!(hir_string_proc("C.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data() { + eval(" + class C < Range + def test = @a + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit UnhandledYARVInsn(getinstancevariable) + "); + } + + #[test] + fn test_optimize_attr_reader_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + class << self + attr_reader :foo + end + def self.test = foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_getivar_polymorphic() { + set_call_threshold(3); + eval(" + class C + attr_reader :foo, :bar + + def foo_then_bar + @foo = 1 + @bar = 2 + end + + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + + O1 = C.new + O1.foo_then_bar + O2 = C.new + O2.bar_then_foo + def test(o) = o.foo + test O1 + test O2 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:20: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:BasicObject = SendWithoutBlock v9, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_dont_optimize_getivar_with_too_complex_shape() { + eval(r#" + class C + attr_accessor :foo + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + def test(o) = o.foo + test obj + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:12: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter getivar_fallback_too_complex + v22:BasicObject = GetIvar v21, :@foo + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_send_with_block() { + eval(r#" + def test = [1, 2, 3].map { |x| x * 2 } + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) + v21:BasicObject = CCallWithFrame v11, :Array#map@0x1040, block=0x1048 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_variadic_with_block() { + eval(r#" + A = [1, 2, 3] + B = ["a", "b", "c"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + SetLocal :result, l0, EP@3, v13 + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v36:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, B) + v39:ArrayExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(Array@0x1020) + PatchPoint MethodRedefined(Array@0x1020, zip@0x1028, cme:0x1030) + v43:BasicObject = CCallVariadic v36, :zip@0x1058, v39 + v25:BasicObject = GetLocal :result, l0, EP@3 + v29:BasicObject = GetLocal :result, l0, EP@3 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_do_not_optimize_send_with_block_forwarding() { + eval(r#" + def test(&block) = [].map(&block) + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:ArrayExact = NewArray + GuardBlockParamProxy l0 + v16:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + IncrCounter complex_arg_pass_caller_blockarg + v18:BasicObject = Send v13, 0x1008, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_do_not_optimize_send_to_iseq_method_with_block() { + eval(r#" + def foo + yield 1 + end + + def test = foo {} + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_attr_reader_constant() { + eval(" + class C + attr_reader :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_attr_accessor_constant() { + eval(" + class C + attr_accessor :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_attr_reader() { + eval(" + class C + attr_reader :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:NilClass = Const Value(nil) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_inline_attr_accessor() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:NilClass = Const Value(nil) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_inline_attr_accessor_set() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v29:CShape = LoadField v26, :_shape_id@0x1038 + v30:CShape[0x1039] = GuardBitEquals v29, CShape(0x1039) + StoreField v26, :@foo@0x103a, v16 + WriteBarrier v26, v16 + v33:CShape[0x103b] = Const CShape(0x103b) + StoreField v26, :_shape_id@0x1038, v33 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_inline_attr_writer_set() { + eval(" + class C + attr_writer :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v29:CShape = LoadField v26, :_shape_id@0x1038 + v30:CShape[0x1039] = GuardBitEquals v29, CShape(0x1039) + StoreField v26, :@foo@0x103a, v16 + WriteBarrier v26, v16 + v33:CShape[0x103b] = Const CShape(0x103b) + StoreField v26, :_shape_id@0x1038, v33 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_inline_struct_aref_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:BasicObject = LoadField v21, :foo@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_struct_aref_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:CPtr = LoadField v21, :_as_heap@0x1038 + v23:BasicObject = LoadField v22, :foo@0x1039 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_elide_struct_aref() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) + o.foo + 5 + end + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v25:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_struct_aset_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@5 + v3:BasicObject = GetLocal :v, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v29:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v30:HeapObject[class_exact:C] = GuardNotFrozen v29 + StoreField v30, :foo=@0x1038, v12 + WriteBarrier v30, v12 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_inline_struct_aset_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@5 + v3:BasicObject = GetLocal :v, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v29:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v30:HeapObject[class_exact:C] = GuardNotFrozen v29 + v31:CPtr = LoadField v30, :_as_heap@0x1038 + StoreField v31, :foo=@0x1039, v12 + WriteBarrier v30, v12 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_array_reverse_returns_array() { + eval(r#" + def test = [].reverse + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v20:ArrayExact = CCallWithFrame v10, :Array#reverse@0x1038 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_array_reverse_is_elidable() { + eval(r#" + def test + [].reverse + 5 + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_array_join_returns_string() { + eval(r#" + def test = [].join "," + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, join@0x1010, cme:0x1018) + v23:StringExact = CCallVariadic v10, :Array#join@0x1040, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_string_to_s_returns_string() { + eval(r#" + def test = "".to_s + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_string_literal_to_s() { + eval(r#" + def test = "foo".to_s + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_profiled_string_to_s() { + eval(r#" + def test(o) = o.to_s + test "foo" + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, to_s@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fixnum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test 5 + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_s@0x1008, cme:0x1010) + v21:Fixnum = GuardType v9, Fixnum + v22:StringExact = CCallVariadic v21, :Integer#to_s@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_bignum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test (2**65) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_s@0x1008, cme:0x1010) + v21:Integer = GuardType v9, Integer + v22:StringExact = CCallVariadic v21, :Integer#to_s@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_any_to_string_with_known_string_exact() { + eval(r##" + def test(x) = "#{x}" + test 123 + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:Fixnum = GuardType v9, Fixnum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v30:StringExact = CCallVariadic v26, :Integer#to_s@0x1040 + v21:StringExact = StringConcat v13, v30 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_array_aref_fixnum_literal() { + eval(" + def test + arr = [1, 2, 3] + arr[0] + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v19:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v30:CInt64[0] = UnboxFixnum v19 + v31:CInt64 = ArrayLength v14 + v32:CInt64[0] = GuardLess v30, v31 + v33:CInt64[0] = Const CInt64(0) + v34:CInt64[0] = GuardGreaterEq v32, v33 + v35:BasicObject = ArrayAref v14, v34 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_array_aref_fixnum_profiled() { + eval(" + def test(arr, idx) + arr[idx] + end + test([1, 2, 3], 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@5 + v3:BasicObject = GetLocal :idx, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []@0x1008, cme:0x1010) + v27:ArrayExact = GuardType v11, ArrayExact + v28:Fixnum = GuardType v12, Fixnum + v29:CInt64 = UnboxFixnum v28 + v30:CInt64 = ArrayLength v27 + v31:CInt64 = GuardLess v29, v30 + v32:CInt64[0] = Const CInt64(0) + v33:CInt64 = GuardGreaterEq v31, v32 + v34:BasicObject = ArrayAref v27, v33 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_array_aref_fixnum_array_subclass() { + eval(" + class C < Array; end + def test(arr, idx) + arr[idx] + end + test(C.new([1, 2, 3]), 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@5 + v3:BasicObject = GetLocal :idx, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []@0x1008, cme:0x1010) + v27:ArraySubclass[class_exact:C] = GuardType v11, ArraySubclass[class_exact:C] + v28:Fixnum = GuardType v12, Fixnum + v29:CInt64 = UnboxFixnum v28 + v30:CInt64 = ArrayLength v27 + v31:CInt64 = GuardLess v29, v30 + v32:CInt64[0] = Const CInt64(0) + v33:CInt64 = GuardGreaterEq v31, v32 + v34:BasicObject = ArrayAref v27, v33 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_hash_aref_literal() { + eval(" + def test + arr = {1 => 3} + arr[1] + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v19:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v30:BasicObject = HashAref v14, v19 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_hash_aref_profiled() { + eval(" + def test(hash, key) + hash[key] + end + test({1 => 3}, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@5 + v3:BasicObject = GetLocal :key, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []@0x1008, cme:0x1010) + v27:HashExact = GuardType v11, HashExact + v28:BasicObject = HashAref v27, v12 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_no_optimize_hash_aref_subclass() { + eval(" + class C < Hash; end + def test(hash, key) + hash[key] + end + test(C.new({0 => 3}), 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@5 + v3:BasicObject = GetLocal :key, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []@0x1008, cme:0x1010) + v27:HashSubclass[class_exact:C] = GuardType v11, HashSubclass[class_exact:C] + v28:BasicObject = CCallWithFrame v27, :Hash#[]@0x1038, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_does_not_fold_hash_aref_with_frozen_hash() { + eval(" + H = {a: 0}.freeze + def test = H[:a] + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, H) + v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:StaticSymbol[:a] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(Hash@0x1018) + PatchPoint MethodRedefined(Hash@0x1018, []@0x1020, cme:0x1028) + v27:BasicObject = HashAref v23, v13 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_hash_aset_literal() { + eval(" + def test + h = {} + h[1] = 3 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + v24:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + HashAset v13, v22, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_hash_aset_profiled() { + eval(" + def test(hash, key, val) + hash[key] = val + end + test({}, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@6 + v3:BasicObject = GetLocal :key, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + v35:HashExact = GuardType v13, HashExact + HashAset v35, v14, v15 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_optimize_hash_aset_subclass() { + eval(" + class C < Hash; end + def test(hash, key, val) + hash[key] = val + end + test(C.new, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@6 + v3:BasicObject = GetLocal :key, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []=@0x1008, cme:0x1010) + v35:HashSubclass[class_exact:C] = GuardType v13, HashSubclass[class_exact:C] + v36:BasicObject = CCallWithFrame v35, :Hash#[]=@0x1038, v14, v15 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_thread_current() { + eval(" + def test = Thread.current + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Thread) + v20:Class[Thread@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, current@0x1018, cme:0x1020) + v24:CPtr = LoadEC + v25:CPtr = LoadField v24, :thread_ptr@0x1048 + v26:BasicObject = LoadField v25, :self@0x1049 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_array_aset_literal() { + eval(" + def test(arr) + arr[1] = 10 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []=@0x1008, cme:0x1010) + v31:ArrayExact = GuardType v9, ArrayExact + v32:ArrayExact = GuardNotFrozen v31 + v33:ArrayExact = GuardNotShared v32 + v34:CInt64[1] = UnboxFixnum v16 + v35:CInt64 = ArrayLength v33 + v36:CInt64[1] = GuardLess v34, v35 + v37:CInt64[0] = Const CInt64(0) + v38:CInt64[1] = GuardGreaterEq v36, v37 + ArrayAset v33, v38, v18 + WriteBarrier v33, v18 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_array_aset_profiled() { + eval(" + def test(arr, index, val) + arr[index] = val + end + test([], 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@6 + v3:BasicObject = GetLocal :index, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []=@0x1008, cme:0x1010) + v35:ArrayExact = GuardType v13, ArrayExact + v36:Fixnum = GuardType v14, Fixnum + v37:ArrayExact = GuardNotFrozen v35 + v38:ArrayExact = GuardNotShared v37 + v39:CInt64 = UnboxFixnum v36 + v40:CInt64 = ArrayLength v38 + v41:CInt64 = GuardLess v39, v40 + v42:CInt64[0] = Const CInt64(0) + v43:CInt64 = GuardGreaterEq v41, v42 + ArrayAset v38, v43, v15 + WriteBarrier v38, v15 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_array_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, index, val) + arr[index] = val + end + a = MyArray.new + test(a, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@6 + v3:BasicObject = GetLocal :index, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(MyArray@0x1000) + PatchPoint MethodRedefined(MyArray@0x1000, []=@0x1008, cme:0x1010) + v35:ArraySubclass[class_exact:MyArray] = GuardType v13, ArraySubclass[class_exact:MyArray] + v36:BasicObject = CCallVariadic v35, :Array#[]=@0x1038, v14, v15 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_array_ltlt() { + eval(" + def test(arr) + arr << 1 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, <<@0x1008, cme:0x1010) + v25:ArrayExact = GuardType v9, ArrayExact + ArrayPush v25, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_array_push_single_arg() { + eval(" + def test(arr) + arr.push(1) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, push@0x1008, cme:0x1010) + v24:ArrayExact = GuardType v9, ArrayExact + ArrayPush v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_optimize_array_push_multi_arg() { + eval(" + def test(arr) + arr.push(1,2,3) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:Fixnum[2] = Const Value(2) + v18:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, push@0x1008, cme:0x1010) + v28:ArrayExact = GuardType v9, ArrayExact + v29:BasicObject = CCallVariadic v28, :Array#push@0x1038, v14, v16, v18 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_array_length() { + eval(" + def test(arr) = arr.length + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_array_size() { + eval(" + def test(arr) = arr.size + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_regexpmatch2() { + eval(r#" + def test(s) = s =~ /a/ + test("foo") + "#); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, =~@0x1010, cme:0x1018) + v25:StringExact = GuardType v9, StringExact + v26:BasicObject = CCallWithFrame v25, :String#=~@0x1040, v14 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_string_getbyte_fixnum() { + eval(r#" + def test(s, i) = s.getbyte(i) + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:BasicObject = GetLocal :i, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, getbyte@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:Fixnum = GuardType v12, Fixnum + v28:CInt64 = UnboxFixnum v27 + v29:CInt64 = LoadField v26, :len@0x1038 + v30:CInt64 = GuardLess v28, v29 + v31:CInt64[0] = Const CInt64(0) + v32:CInt64 = GuardGreaterEq v30, v31 + v33:Fixnum = StringGetbyte v26, v30 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_elide_string_getbyte_fixnum() { + eval(r#" + def test(s, i) + s.getbyte(i) + 5 + end + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:BasicObject = GetLocal :i, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, getbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v11, StringExact + v31:Fixnum = GuardType v12, Fixnum + v32:CInt64 = UnboxFixnum v31 + v33:CInt64 = LoadField v30, :len@0x1038 + v34:CInt64 = GuardLess v32, v33 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + IncrCounter inline_cfunc_optimized_send_count + v22:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_string_setbyte_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 127) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v13, StringExact + v31:Fixnum = GuardType v14, Fixnum + v32:Fixnum = GuardType v15, Fixnum + v33:CInt64 = UnboxFixnum v31 + v34:CInt64 = LoadField v30, :len@0x1038 + v35:CInt64 = GuardLess v33, v34 + v36:CInt64[0] = Const CInt64(0) + v37:CInt64 = GuardGreaterEq v35, v36 + v38:StringExact = GuardNotFrozen v30 + v39:Fixnum = StringSetbyteFixnum v38, v31, v32 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_string_subclass_setbyte_fixnum() { + eval(r#" + class MyString < String + end + def test(s, idx, val) + s.setbyte(idx, val) + end + test(MyString.new('foo'), 0, 127) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1000) + PatchPoint MethodRedefined(MyString@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringSubclass[class_exact:MyString] = GuardType v13, StringSubclass[class_exact:MyString] + v31:Fixnum = GuardType v14, Fixnum + v32:Fixnum = GuardType v15, Fixnum + v33:CInt64 = UnboxFixnum v31 + v34:CInt64 = LoadField v30, :len@0x1038 + v35:CInt64 = GuardLess v33, v34 + v36:CInt64[0] = Const CInt64(0) + v37:CInt64 = GuardGreaterEq v35, v36 + v38:StringSubclass[class_exact:MyString] = GuardNotFrozen v30 + v39:Fixnum = StringSetbyteFixnum v38, v31, v32 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_do_not_optimize_string_setbyte_non_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 3.14) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v13, StringExact + v31:BasicObject = CCallWithFrame v30, :String#setbyte@0x1038, v14, v15 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_string_empty() { + eval(r#" + def test(s) + s.empty? + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, empty?@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + v24:CInt64 = LoadField v23, :len@0x1038 + v25:CInt64[0] = Const CInt64(0) + v26:CBool = IsBitEqual v24, v25 + v27:BoolExact = BoxBool v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_eliminate_string_empty() { + eval(r#" + def test(s) + s.empty? + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, empty?@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_integer_succ_with_fixnum() { + eval(" + def test(x) = x.succ + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, succ@0x1008, cme:0x1010) + v22:Fixnum = GuardType v9, Fixnum + v23:Fixnum[1] = Const Value(1) + v24:Fixnum = FixnumAdd v22, v23 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_bignum() { + eval(" + def test(x) = x.succ + test(4 << 70) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, succ@0x1008, cme:0x1010) + v22:Integer = GuardType v9, Integer + v23:BasicObject = CCallWithFrame v22, :Integer#succ@0x1038 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_inline_integer_ltlt_with_known_fixnum() { + eval(" + def test(x) = x << 5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumLShift v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_negative() { + eval(" + def test(x) = x << -5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BasicObject = CCallWithFrame v24, :Integer#<<@0x1038, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_out_of_range() { + eval(" + def test(x) = x << 64 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BasicObject = CCallWithFrame v24, :Integer#<<@0x1038, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_unknown_fixnum() { + eval(" + def test(x, y) = x << y + test(4, 5) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1038, v12 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_integer_gtgt_with_known_fixnum() { + eval(" + def test(x) = x >> 5 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:Fixnum = FixnumRShift v23, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_negative() { + eval(" + def test(x) = x >> -5 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:BasicObject = CCallWithFrame v23, :Integer#>>@0x1038, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_out_of_range() { + eval(" + def test(x) = x >> 64 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:BasicObject = CCallWithFrame v23, :Integer#>>@0x1038, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_unknown_fixnum() { + eval(" + def test(x, y) = x >> y + test(4, 5) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1038, v12 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_string_append() { + eval(r#" + def test(x, y) = x << y + test("iron", "fish") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:StringExact = StringAppend v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_string_append_codepoint() { + eval(r#" + def test(x, y) = x << y + test("iron", 4) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:Fixnum = GuardType v12, Fixnum + v29:StringExact = StringAppendCodepoint v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_string_append_string_subclass() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test("iron", MyString.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:StringExact = StringAppend v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_do_not_optimize_string_subclass_append_string() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test(MyString.new, "iron") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1000) + PatchPoint MethodRedefined(MyString@0x1000, <<@0x1008, cme:0x1010) + v27:StringSubclass[class_exact:MyString] = GuardType v11, StringSubclass[class_exact:MyString] + v28:BasicObject = CCallWithFrame v27, :String#<<@0x1038, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_optimize_string_append_non_string() { + eval(r#" + def test = "iron" << :a + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, <<@0x1018, cme:0x1020) + v24:BasicObject = CCallWithFrame v11, :String#<<@0x1048, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_many_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo(3, 3, 3) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[3] = Const Value(3) + v14:Fixnum[3] = Const Value(3) + v16:Fixnum[3] = Const Value(3) + v18:BasicObject = SendWithoutBlock v10, :foo, v12, v14, v16 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_string_ascii_only_p() { + eval(r#" + def test(x) = x.ascii_only? + test("iron") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ascii_only?@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v24:BoolExact = CCall v22, :String#ascii_only?@0x1038 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_few_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:BasicObject = SendWithoutBlock v10, :foo # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_args() { + eval(" + def test = 4.succ 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = SendWithoutBlock v10, :succ, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_integer_xor_with_fixnum() { + eval(" + def test(x, y) = x ^ y + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:Fixnum = GuardType v12, Fixnum + v27:Fixnum = FixnumXor v25, v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_eliminate_integer_xor() { + eval(r#" + def test(x, y) + x ^ y + 42 + end + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v29:Fixnum = GuardType v11, Fixnum + v30:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v22:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_or_boolean() { + eval(" + def test(x, y) = x ^ y + test(4 << 70, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Integer = GuardType v11, Integer + v26:BasicObject = CCallWithFrame v25, :Integer#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + + eval(" + def test(x, y) = x ^ y + test(1, 4 << 70) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + + eval(" + def test(x, y) = x ^ y + test(true, 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1000, ^@0x1008, cme:0x1010) + v25:TrueClass = GuardType v11, TrueClass + v26:BasicObject = CCallWithFrame v25, :TrueClass#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_args() { + eval(" + def test(x, y) = x.^() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v17:BasicObject = SendWithoutBlock v11, :^ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_specialize_hash_size() { + eval(" + def test(hash) = hash.size + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v23:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :Hash#size@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_hash_size() { + eval(" + def test(hash) + hash.size + 5 + end + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v27:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_respond_to_p_true() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v28:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_no_method() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint MethodRedefined(C@0x1008, respond_to_missing?@0x1040, cme:0x1048) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1070, cme:0x1078) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_default_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v28:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, false) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:FalseClass = Const Value(false) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_true_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, true) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_truthy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, 4) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_missing() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint MethodRedefined(C@0x1008, respond_to_missing?@0x1040, cme:0x1048) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1070, cme:0x1078) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_do_not_optimize_redefined_respond_to_missing() { + eval(r#" + class C + def respond_to_missing?(method, include_private = false) + true + end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v25:BasicObject = CCallVariadic v24, :Kernel#respond_to?@0x1040, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putself() { + eval(r#" + def callee = self + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_string() { + eval(r#" + # frozen_string_literal: true + def callee = "abc" + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:StringExact[VALUE(0x1038)] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putnil() { + eval(r#" + def callee = nil + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_true() { + eval(r#" + def callee = true + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:TrueClass = Const Value(true) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_false() { + eval(r#" + def callee = false + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:FalseClass = Const Value(false) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_zero() { + eval(r#" + def callee = 0 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_one() { + eval(r#" + def callee = 1 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_parameter() { + eval(r#" + def callee(x) = x + def test = callee 3 + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_without_block_direct_last_parameter() { + eval(r#" + def callee(x, y, z) = z + def test = callee 1, 2, 3 + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_splat() { + eval(" + def foo = itself + + def test + # Use a local to inhibit compile.c peephole optimization to ensure callsites have VM_CALL_ARGS_SPLAT + empty = [] + foo(*empty) + ''.display(*empty) + itself(*empty) + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v19:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v21:BasicObject = SendWithoutBlock v8, :foo, v19 # SendFallbackReason: Complex argument passing + v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:StringExact = StringCopy v25 + PatchPoint NoEPEscape(test) + v31:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v33:BasicObject = SendWithoutBlock v26, :display, v31 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v41:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v43:BasicObject = SendWithoutBlock v8, :itself, v41 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_inline_symbol_to_sym() { + eval(r#" + def test(o) = o.to_sym + test :foo + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1000, to_sym@0x1008, cme:0x1010) + v20:StaticSymbol = GuardType v9, StaticSymbol + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_integer_to_i() { + eval(r#" + def test(o) = o.to_i + test 5 + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_i@0x1008, cme:0x1010) + v20:Fixnum = GuardType v9, Fixnum + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_stringexact_eq_stringexact() { + eval(r#" + def test(l, r) = l == r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ==@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010) + v27:StringSubclass[class_exact:C] = GuardType v11, StringSubclass[class_exact:C] + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ==@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_stringexact() { + eval(r#" + def test(l, r) = l === r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ===@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ===@0x1008, cme:0x1010) + v26:StringSubclass[class_exact:C] = GuardType v11, StringSubclass[class_exact:C] + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ===@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_specialize_string_size() { + eval(r#" + def test(s) + s.size + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, size@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :String#size@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_elide_string_size() { + eval(r#" + def test(s) + s.size + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, size@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, bytesize@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + v23:CInt64 = LoadField v22, :len@0x1038 + v24:Fixnum = BoxFixnum v23 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_elide_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, bytesize@0x1008, cme:0x1010) + v26:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_string_length() { + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, length@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :String#length@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_class_eqq() { + eval(r#" + def test(o) = String === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v26:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, ===@0x1018, cme:0x1020) + v30:BoolExact = IsA v9, v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_module_eqq() { + eval(r#" + def test(o) = Kernel === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v26:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, ===@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v31:BoolExact = CCall v26, :Module#===@0x1048, v9 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_is_a_class() { + eval(r#" + def test(o) = o.is_a?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v24:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, is_a?@0x1009, cme:0x1010) + v28:StringExact = GuardType v9, StringExact + v29:BoolExact = IsA v28, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_specialize_is_a_module() { + eval(r#" + def test(o) = o.is_a?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v24:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1018, cme:0x1020) + v28:StringExact = GuardType v9, StringExact + v29:BasicObject = CCallWithFrame v28, :Kernel#is_a?@0x1048, v24 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_elide_is_a() { + eval(r#" + def test(o) + o.is_a?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v28:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1018, cme:0x1020) + v32:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_elide_class_eqq() { + eval(r#" + def test(o) + Integer === o + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v30:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, ===@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_kind_of_class() { + eval(r#" + def test(o) = o.kind_of?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v24:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, kind_of?@0x1009, cme:0x1010) + v28:StringExact = GuardType v9, StringExact + v29:BoolExact = IsA v28, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_specialize_kind_of_module() { + eval(r#" + def test(o) = o.kind_of?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v24:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1018, cme:0x1020) + v28:StringExact = GuardType v9, StringExact + v29:BasicObject = CCallWithFrame v28, :Kernel#kind_of?@0x1048, v24 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_elide_kind_of() { + eval(r#" + def test(o) + o.kind_of?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v28:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1018, cme:0x1020) + v32:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn counting_complex_feature_use_for_fallback() { + eval(" + define_method(:fancy) { |_a, *_b, kw: 100, **kw_rest, &block| } + def test = fancy(1) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_rest + IncrCounter complex_arg_pass_param_block + IncrCounter complex_arg_pass_param_kwrest + v13:BasicObject = SendWithoutBlock v6, :fancy, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn call_method_forwardable_param() { + eval(" + def forwardable(...) = itself(...) + def call_forwardable = forwardable + call_forwardable + "); + assert_snapshot!(hir_string("call_forwardable"), @r" + fn call_forwardable@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_forwardable + v11:BasicObject = SendWithoutBlock v6, :forwardable # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_string_length() { + eval(r#" + def test(s) + s.length + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, length@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_fold_self_class_respond_to_true() { + eval(r#" + class C + class << self + attr_accessor :_lex_actions + private :_lex_actions, :_lex_actions= + end + self._lex_actions = [1, 2, 3] + def initialize + if self.class.respond_to?(:_lex_actions, true) + :CORRECT + else + :oh_no_wrong + end + end + end + C.new # warm up + TEST = C.instance_method(:initialize) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn initialize@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v42:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v46:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + v13:StaticSymbol[:_lex_actions] = Const Value(VALUE(0x1038)) + v15:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(Class@0x1040) + PatchPoint MethodRedefined(Class@0x1040, respond_to?@0x1048, cme:0x1050) + PatchPoint NoSingletonClass(Class@0x1040) + PatchPoint MethodRedefined(Class@0x1040, _lex_actions@0x1078, cme:0x1080) + v54:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v26:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_self_class_name() { + eval(r#" + class C; end + def test(o) = o.class.name + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v23:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v27:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, name@0x1040, cme:0x1048) + IncrCounter inline_cfunc_optimized_send_count + v33:StringExact|NilClass = CCall v27, :Module#name@0x1070 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_kernel_class() { + eval(r#" + class C; end + def test(o) = o.class + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v25:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_class() { + eval(r#" + def test = 5.class + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, class@0x1008, cme:0x1010) + IncrCounter inline_iseq_optimized_send_count + v21:Class[Integer@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_fold_singleton_class() { + eval(r#" + def test = self.class + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, class@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v22:Class[Object@0x1038] = Const Value(VALUE(0x1038)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v22 + "); + } + + #[test] + fn no_load_from_ep_right_after_entrypoint() { + let formatted = eval(" + def read_nil_local(a, _b, _c) + formatted ||= a + @formatted = formatted + -> { formatted } # the environment escapes + end + + def call + puts [], [], [], [] # fill VM stack with junk + read_nil_local(true, 1, 1) # expected direct send + end + + call # profile + call # compile + @formatted + "); + assert_eq!(Qtrue, formatted, "{}", formatted.obj_info()); + assert_snapshot!(hir_string("read_nil_local"), @r" + fn read_nil_local@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :_b, l0, SP@6 + v4:BasicObject = GetLocal :_c, l0, SP@5 + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject): + EntryPoint JIT(0) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass): + CheckInterrupts + SetLocal :formatted, l0, EP@3, v15 + PatchPoint SingleRactorMode + v56:HeapBasicObject = GuardType v14, HeapBasicObject + v57:CShape = LoadField v56, :_shape_id@0x1000 + v58:CShape[0x1001] = GuardBitEquals v57, CShape(0x1001) + StoreField v56, :@formatted@0x1002, v15 + WriteBarrier v56, v15 + v61:CShape[0x1003] = Const CShape(0x1003) + StoreField v56, :_shape_id@0x1000, v61 + v45:Class[VMFrozenCore] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, lambda@0x1018, cme:0x1020) + v66:BasicObject = CCallWithFrame v45, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 + v48:BasicObject = GetLocal :a, l0, EP@6 + v49:BasicObject = GetLocal :_b, l0, EP@5 + v50:BasicObject = GetLocal :_c, l0, EP@4 + v51:BasicObject = GetLocal :formatted, l0, EP@3 + CheckInterrupts + Return v66 + "); + } + + #[test] + fn test_fold_load_field_frozen_constant_object() { + // Basic case: frozen constant object with attr_accessor + eval(" + class TestFrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + FROZEN_OBJ = TestFrozen.new.freeze + + def test = FROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_OBJ) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozen@0x1010) + PatchPoint MethodRedefined(TestFrozen@0x1010, a@0x1018, cme:0x1020) + v29:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_multiple_ivars() { + // Frozen object with multiple instance variables + eval(" + class TestMultiIvars + attr_accessor :a, :b, :c + def initialize + @a = 10 + @b = 20 + @c = 30 + end + end + + MULTI_FROZEN = TestMultiIvars.new.freeze + + def test = MULTI_FROZEN.b + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MULTI_FROZEN) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestMultiIvars@0x1010) + PatchPoint MethodRedefined(TestMultiIvars@0x1010, b@0x1018, cme:0x1020) + v29:Fixnum[20] = Const Value(20) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_string_value() { + // Frozen object with a string ivar + eval(r#" + class TestFrozenStr + attr_accessor :name + def initialize + @name = "hello" + end + end + + FROZEN_STR = TestFrozenStr.new.freeze + + def test = FROZEN_STR.name + test + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_STR) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenStr@0x1010) + PatchPoint MethodRedefined(TestFrozenStr@0x1010, name@0x1018, cme:0x1020) + v29:StringExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_nil_value() { + // Frozen object with nil ivar + eval(" + class TestFrozenNil + attr_accessor :value + def initialize + @value = nil + end + end + + FROZEN_NIL = TestFrozenNil.new.freeze + + def test = FROZEN_NIL.value + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_NIL) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenNil@0x1010) + PatchPoint MethodRedefined(TestFrozenNil@0x1010, value@0x1018, cme:0x1020) + v29:NilClass = Const Value(nil) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_no_fold_load_field_unfrozen_object() { + // Non-frozen object should NOT be folded + eval(" + class TestUnfrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + UNFROZEN_OBJ = TestUnfrozen.new + + def test = UNFROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, UNFROZEN_OBJ) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestUnfrozen@0x1010) + PatchPoint MethodRedefined(TestUnfrozen@0x1010, a@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:BasicObject = LoadField v20, :@a@0x104a + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_load_field_frozen_with_attr_reader() { + // Using attr_reader instead of attr_accessor + eval(" + class TestAttrReader + attr_reader :value + def initialize(v) + @value = v + end + end + + FROZEN_READER = TestAttrReader.new(42).freeze + + def test = FROZEN_READER.value + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_READER) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestAttrReader@0x1010) + PatchPoint MethodRedefined(TestAttrReader@0x1010, value@0x1018, cme:0x1020) + v29:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_symbol_value() { + // Frozen object with a symbol ivar + eval(" + class TestFrozenSym + attr_accessor :sym + def initialize + @sym = :hello + end + end + + FROZEN_SYM = TestFrozenSym.new.freeze + + def test = FROZEN_SYM.sym + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_SYM) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenSym@0x1010) + PatchPoint MethodRedefined(TestFrozenSym@0x1010, sym@0x1018, cme:0x1020) + v29:StaticSymbol[:hello] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_true_false() { + // Frozen object with boolean ivars + eval(" + class TestFrozenBool + attr_accessor :flag + def initialize + @flag = true + end + end + + FROZEN_TRUE = TestFrozenBool.new.freeze + + def test = FROZEN_TRUE.flag + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_TRUE) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenBool@0x1010) + PatchPoint MethodRedefined(TestFrozenBool@0x1010, flag@0x1018, cme:0x1020) + v29:TrueClass = Const Value(true) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_no_fold_load_field_dynamic_receiver() { + // Dynamic receiver (not a constant) should NOT be folded even if object is frozen + eval(" + class TestDynamic + attr_accessor :val + def initialize + @val = 99 + end + end + + def test(obj) = obj.val + o = TestDynamic.new.freeze + test o + test o + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :obj, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(TestDynamic@0x1000) + PatchPoint MethodRedefined(TestDynamic@0x1000, val@0x1008, cme:0x1010) + v21:HeapObject[class_exact:TestDynamic] = GuardType v9, HeapObject[class_exact:TestDynamic] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:BasicObject = LoadField v21, :@val@0x103a + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_load_field_frozen_nested_access() { + // Accessing multiple fields from frozen constant in sequence + eval(" + class TestNestedAccess + attr_accessor :x, :y + def initialize + @x = 100 + @y = 200 + end + end + + NESTED_FROZEN = TestNestedAccess.new.freeze + + def test = NESTED_FROZEN.x + NESTED_FROZEN.y + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:12: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, NESTED_FROZEN) + v28:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, x@0x1018, cme:0x1020) + v53:Fixnum[100] = Const Value(100) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1048, NESTED_FROZEN) + v34:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, y@0x1050, cme:0x1058) + v55:Fixnum[200] = Const Value(200) + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) + v56:Fixnum[300] = Const Value(300) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v56 + "); + } + + #[test] + fn test_dont_fold_load_field_with_primitive_return_type() { + eval(r#" + S = "abc".freeze + def test = S.bytesize + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v20:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, bytesize@0x1018, cme:0x1020) + v24:CInt64 = LoadField v20, :len@0x1048 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn optimize_call_to_private_method_iseq_with_fcall() { + eval(r#" + class C + def callprivate = secret + private def secret = 42 + end + C.new.callprivate + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprivate)"), @r" + fn callprivate@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v18:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_iseq() { + eval(r#" + class C + private def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn optimize_call_to_private_method_cfunc_with_fcall() { + eval(r#" + class BasicObject + def callprivate = initialize rescue $! + end + Obj = BasicObject.new.callprivate + "#); + assert_snapshot!(hir_string_proc("BasicObject.instance_method(:callprivate)"), @r" + fn callprivate@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(BasicObject@0x1000) + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v20:BasicObjectExact = GuardType v6, BasicObjectExact + v21:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_cfunc() { + eval(r#" + Obj = BasicObject.new + def test = Obj.initialize rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:BasicObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :initialize # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_optimize_call_to_private_top_level_method() { + eval(r#" + def toplevel_method = :OK + Obj = Object.new + def test = Obj.toplevel_method rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :toplevel_method # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn optimize_call_to_protected_method_iseq_with_fcall() { + eval(r#" + class C + def callprotected = secret + protected def secret = 42 + end + C.new.callprotected + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprotected)"), @r" + fn callprotected@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v18:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_protected_method_iseq() { + eval(r#" + class C + protected def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + // Test that when a singleton class has been seen for a class, we skip the + // NoSingletonClass optimization to avoid an invalidation loop. + #[test] + fn test_skip_optimization_after_singleton_class_seen() { + // First, trigger the singleton class callback for String by creating a singleton class. + // This should mark String as having had a singleton class seen. + eval(r#" + "hello".singleton_class + "#); + + // Now define and compile a method that would normally be optimized with NoSingletonClass. + // Since String has had a singleton class, the optimization should be skipped and we + // should fall back to SendWithoutBlock. + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + + // The output should NOT have NoSingletonClass patchpoint for String, and should + // fall back to SendWithoutBlock instead of the optimized CCall path. + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :length # SendFallbackReason: Singleton class previously created for receiver class + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_invokesuper_to_iseq_optimizes_to_direct() { + eval(" + class A + def foo + 'A' + end + end + + class B < A + def foo + super + end + end + + B.new.foo; B.new.foo + "); + + // A Ruby method as the target of `super` should optimize provided no block is given. + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendWithoutBlockDirect but got:\n{hir}"); + assert!(hir.contains("SendWithoutBlockDirect"), "Should optimize to SendWithoutBlockDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + v17:CPtr = GetLEP + GuardSuperMethodEntry v17, 0x1038 + v19:RubyValue = GetBlockHandler v17 + v20:FalseClass = GuardBitEquals v19, Value(false) + v21:BasicObject = SendWithoutBlockDirect v6, :foo (0x1040) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_invokesuper_with_positional_args_optimizes_to_direct() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + super(x) + 1 + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendWithoutBlockDirect but got:\n{hir}"); + assert!(hir.contains("SendWithoutBlockDirect"), "Should optimize to SendWithoutBlockDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + v26:CPtr = GetLEP + GuardSuperMethodEntry v26, 0x1038 + v28:RubyValue = GetBlockHandler v26 + v29:FalseClass = GuardBitEquals v28, Value(false) + v30:BasicObject = SendWithoutBlockDirect v8, :foo (0x1040), v9 + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) + v33:Fixnum = GuardType v30, Fixnum + v34:Fixnum = FixnumAdd v33, v17 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_invokesuper_with_forwarded_splat_args_remains_invokesuper() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(*x) + super + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :x, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + v15:ArrayExact = ToArray v9 + v17:BasicObject = InvokeSuper v8, 0x1000, v15 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_invokesuper_with_block_literal_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo + super { 'from subclass' } + end + end + + B.new.foo; B.new.foo + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for block literal:\n{hir}"); + + // With a block, we don't optimize to SendWithoutBlockDirect + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call made with a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_to_cfunc_remains_invokesuper() { + eval(" + class MyArray < Array + def length + super + end + end + + MyArray.new.length; MyArray.new.length + "); + + let hir = hir_string_proc("MyArray.new.method(:length)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for CFUNC:\n{hir}"); + + assert_snapshot!(hir, @r" + fn length@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: unsupported target method type Cfunc + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_with_blockarg_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo(&blk) + other_block = proc { 'different block' } + super(&other_block) + end + end + + B.new.foo { 'passed' }; B.new.foo { 'passed' } + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :blk, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + PatchPoint NoSingletonClass(B@0x1000) + PatchPoint MethodRedefined(B@0x1000, proc@0x1008, cme:0x1010) + v35:HeapObject[class_exact:B] = GuardType v10, HeapObject[class_exact:B] + v36:BasicObject = CCallWithFrame v35, :Kernel#proc@0x1038, block=0x1040 + v18:BasicObject = GetLocal :blk, l0, EP@4 + SetLocal :other_block, l0, EP@3, v36 + v25:BasicObject = GetLocal :other_block, l0, EP@3 + v27:BasicObject = InvokeSuper v10, 0x1048, v25 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invokesuper_with_symbol_to_proc_remains_invokesuper() { + eval(" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + super(items, &:succ) + end + end + + B.new.foo([1, 2, 3]); B.new.foo([1, 2, 3]) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for symbol-to-proc:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :items, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:StaticSymbol[:succ] = Const Value(VALUE(0x1000)) + v17:BasicObject = InvokeSuper v8, 0x1008, v9, v15 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_invokesuper_with_keyword_args_remains_invokesuper() { + eval(" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + B.new.foo('image data'); B.new.foo('image data') + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :content, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + Jump bb4(v16, v21) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v24:BasicObject, v25:BasicObject): + v31:BasicObject = InvokeSuper v24, 0x1018, v25 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v31 + "); + } +} diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs new file mode 100644 index 0000000000..3e28178273 --- /dev/null +++ b/zjit/src/hir/tests.rs @@ -0,0 +1,4629 @@ +#[cfg(test)] +use super::*; + +#[cfg(test)] +mod snapshot_tests { + use super::*; + use insta::assert_snapshot; + + #[track_caller] + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[track_caller] + fn optimized_hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(&format!("{}.method(:{})", "self", method))); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v13:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [a=v11, b=v12] } + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [a=v11, b=v12] } + PatchPoint NoTracePoint + v16:Any = Snapshot FrameState { pc: 0x1010, stack: [v11], locals: [a=v11, b=v12] } + v17:Any = Snapshot FrameState { pc: 0x1018, stack: [v11, v12], locals: [a=v11, b=v12] } + v18:ArrayExact = NewArray v11, v12 + v19:Any = Snapshot FrameState { pc: 0x1020, stack: [v18], locals: [a=v11, b=v12] } + PatchPoint NoTracePoint + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_send_direct_with_reordered_kwargs_has_snapshot() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + v16:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15], locals: [] } + PatchPoint NoSingletonClass(Object@0x1010) + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v24:HeapObject[class_exact*:Object@VALUE(0x1010)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1010)] + v25:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1048), v13, v15, v11 + v18:Any = Snapshot FrameState { pc: 0x1050, stack: [v26], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_with_kwargs_in_order_has_snapshot() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } + PatchPoint NoSingletonClass(Object@0x1010) + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v22:HeapObject[class_exact*:Object@VALUE(0x1010)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1010)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1048), v11, v13 + v16:Any = Snapshot FrameState { pc: 0x1050, stack: [v23], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_send_direct_with_many_kwargs_no_reorder_snapshot() { + eval(" + def foo(five, six, a:, b:, c:, d:, e:, f:) = [a, b, c, d, five, six, e, f] + def test = foo(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[5] = Const Value(5) + v13:Fixnum[6] = Const Value(6) + v15:Fixnum[4] = Const Value(4) + v17:Fixnum[3] = Const Value(3) + v19:Fixnum[1] = Const Value(1) + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[7] = Const Value(7) + v25:Fixnum[8] = Const Value(8) + v26:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15, v17, v19, v21, v23, v25], locals: [] } + v27:BasicObject = SendWithoutBlock v6, :foo, v11, v13, v15, v17, v19, v21, v23, v25 # SendFallbackReason: Too many arguments for LIR + v28:Any = Snapshot FrameState { pc: 0x1010, stack: [v27], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v27 + "); + } +} + +#[cfg(test)] +pub mod hir_build_tests { + use super::*; + use insta::assert_snapshot; + + fn iseq_contains_opcode(iseq: IseqPtr, expected_opcode: u32) -> bool { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + if opcode == expected_opcode { + return true; + } + insn_idx += insn_len(opcode as usize); + } + false + } + + #[track_caller] + pub fn assert_contains_opcode(method: &str, opcode: u32) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + + #[track_caller] + fn assert_contains_opcodes(method: &str, opcodes: &[u32]) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + for &opcode in opcodes { + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + } + + /// Combine multiple hir_string() results to match all of them at once, which allows + /// us to avoid running the set of zjit-test -> zjit-test-update multiple times. + #[macro_export] + macro_rules! hir_strings { + ($( $s:expr ),+ $(,)?) => {{ + vec![$( hir_string($s) ),+].join("\n") + }}; + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn assert_compile_fails(method: &str, reason: ParseError) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let result = iseq_to_hir(iseq); + assert!(result.is_err(), "Expected an error but successfully compiled to HIR: {}", FunctionPrinter::without_snapshot(&result.unwrap())); + assert_eq!(result.unwrap_err(), reason); + } + + #[test] + fn test_compile_optional() { + eval("def test(x=1) = 123"); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + v20:Fixnum[1] = Const Value(1) + Jump bb4(v16, v20) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v23:BasicObject, v24:BasicObject): + v28:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_putobject() { + eval("def test = 123"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array() { + eval("def test = []"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array_with_element() { + eval("def test(a) = [a]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = NewArray v9 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_new_range_inclusive_with_one_element() { + eval("def test(a) = (a..10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v16:RangeExact = NewRange v9 NewRangeInclusive v14 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_new_range_inclusive_with_two_elements() { + eval("def test(a, b) = (a..b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:RangeExact = NewRange v11 NewRangeInclusive v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_new_range_exclusive_with_one_element() { + eval("def test(a) = (a...10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v16:RangeExact = NewRange v9 NewRangeExclusive v14 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_new_range_exclusive_with_two_elements() { + eval("def test(a, b) = (a...b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:RangeExact = NewRange v11 NewRangeExclusive v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_array_dup() { + eval("def test = [1, 2, 3]"); + assert_contains_opcode("test", YARVINSN_duparray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_hash_dup() { + eval("def test = {a: 1, b: 2}"); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:HashExact = HashDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_new_hash_empty() { + eval("def test = {}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_hash_with_elements() { + eval("def test(aval, bval) = {a: aval, b: bval}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :aval, l0, SP@5 + v3:BasicObject = GetLocal :bval, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v16:StaticSymbol[:a] = Const Value(VALUE(0x1000)) + v19:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v22:HashExact = NewHash v16: v11, v19: v12 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_string_copy() { + eval("def test = \"hello\""); + assert_contains_opcode("test", YARVINSN_putchilledstring); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_bignum() { + eval("def test = 999999999999999999999999999999999999"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_flonum() { + eval("def test = 1.5"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_heap_float() { + eval("def test = 1.7976931348623157e+308"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_static_sym() { + eval("def test = :foo"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_opt_plus() { + eval("def test = 1+2"); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v15:BasicObject = SendWithoutBlock v10, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze; 5; end + end + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze; 5; end + end + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze; 5; end + end + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_uminus() { + eval(" + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@; 5; end + end + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS)) + "); + } + + #[test] + fn test_setlocal_getlocal() { + eval(" + def test + a = 1 + a + end + "); + assert_contains_opcodes("test", &[YARVINSN_getlocal_WC_0, YARVINSN_setlocal_WC_0]); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nested_setlocal_getlocal() { + eval(" + l3 = 3 + _unused = _unused1 = nil + 1.times do |l2| + _ = nil + l2 = 2 + 1.times do |l1| + l1 = 1 + define_method(:test) do + l1 = l2 + l2 = l1 + l2 + l3 = l2 + l3 + end + end + end + "); + assert_contains_opcodes( + "test", + &[YARVINSN_getlocal_WC_1, YARVINSN_setlocal_WC_1, + YARVINSN_getlocal, YARVINSN_setlocal]); + assert_snapshot!(hir_string("test"), @r" + fn block (3 levels) in <compiled>@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetLocal :l2, l2, EP@4 + SetLocal :l1, l1, EP@3, v10 + v15:BasicObject = GetLocal :l1, l1, EP@3 + v17:BasicObject = GetLocal :l2, l2, EP@4 + v20:BasicObject = SendWithoutBlock v15, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l2, l2, EP@4, v20 + v25:BasicObject = GetLocal :l2, l2, EP@4 + v27:BasicObject = GetLocal :l3, l3, EP@5 + v30:BasicObject = SendWithoutBlock v25, :+, v27 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l3, l3, EP@5, v30 + CheckInterrupts + Return v30 + " + ); + } + + #[test] + fn test_setlocal_in_default_args() { + eval(" + def test(a = (b = 1)) = [a, b] + "); + assert_contains_opcode("test", YARVINSN_setlocal_WC_0); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v6:CBool = IsBitEqual v4, v5 + IfTrue v6, bb2(v1, v2, v3) + Jump bb4(v1, v2, v3) + bb1(v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v10, v11, v12) + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + v25:Fixnum[1] = Const Value(1) + Jump bb4(v19, v25, v25) + bb3(v15:BasicObject, v16:BasicObject): + EntryPoint JIT(1) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) + bb4(v30:BasicObject, v31:BasicObject, v32:NilClass|Fixnum): + v38:ArrayExact = NewArray v31, v32 + CheckInterrupts + Return v38 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_tracepoint() { + eval(" + def test(a = (b = 1)) = [a, b] + TracePoint.new(:line) {}.enable + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v6:CBool = IsBitEqual v4, v5 + IfTrue v6, bb2(v1, v2, v3) + Jump bb4(v1, v2, v3) + bb1(v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v10, v11, v12) + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + SideExit UnhandledYARVInsn(trace_putobject_INT2FIX_1_) + bb3(v15:BasicObject, v16:BasicObject): + EntryPoint JIT(1) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) + bb4(v26:BasicObject, v27:BasicObject, v28:NilClass): + v34:ArrayExact = NewArray v27, v28 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_side_exit() { + eval(" + def test(a = (def foo = nil)) = a + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + SideExit UnhandledYARVInsn(definemethod) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v22:BasicObject, v23:BasicObject): + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_setlocal_cyclic_default_args() { + eval(" + def test = proc { |a=a| a } + "); + assert_snapshot!(hir_string_proc("test"), @r" + fn block in test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb3(v9:BasicObject, v10:BasicObject): + EntryPoint JIT(1) + Jump bb2(v9, v10) + bb2(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn defined_ivar() { + eval(" + def test = defined?(@foo) + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn if_defined_ivar() { + eval(" + def test + if defined?(@foo) + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:TrueClass|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + v13:CBool = Test v10 + v14:NilClass = RefineType v10, Falsy + IfFalse v13, bb3(v6) + v16:TrueClass = RefineType v10, Truthy + v19:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v19 + bb3(v24:BasicObject): + v28:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn defined() { + eval(" + def test = return defined?(SeaChange), defined?(favourite), defined?($ruby) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined constant, v10 + v15:StringExact|NilClass = Defined func, v6 + v17:NilClass = Const Value(nil) + v19:StringExact|NilClass = Defined global-variable, v17 + v21:ArrayExact = NewArray v12, v15, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_return_const() { + eval(" + def test(cond) + if cond + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_leave); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :cond, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + v21:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v21 + bb3(v26:BasicObject, v27:Falsy): + v31:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_merge_const() { + eval(" + def test(cond) + if cond + result = 3 + else + result = 4 + end + result + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :cond, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + CheckInterrupts + v18:CBool = Test v11 + v19:Falsy = RefineType v11, Falsy + IfFalse v18, bb3(v10, v19, v12) + v21:Truthy = RefineType v11, Truthy + v24:Fixnum[3] = Const Value(3) + CheckInterrupts + Jump bb4(v10, v21, v24) + bb3(v29:BasicObject, v30:Falsy, v31:NilClass): + v35:Fixnum[4] = Const Value(4) + Jump bb4(v29, v30, v35) + bb4(v38:BasicObject, v39:BasicObject, v40:Fixnum): + CheckInterrupts + Return v40 + "); + } + + #[test] + fn test_opt_plus_fixnum() { + eval(" + def test(a, b) = a + b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_minus_fixnum() { + eval(" + def test(a, b) = a - b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_minus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :-, v12 # SendFallbackReason: Uncategorized(opt_minus) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_mult_fixnum() { + eval(" + def test(a, b) = a * b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :*, v12 # SendFallbackReason: Uncategorized(opt_mult) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_div_fixnum() { + eval(" + def test(a, b) = a / b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :/, v12 # SendFallbackReason: Uncategorized(opt_div) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_mod_fixnum() { + eval(" + def test(a, b) = a % b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :%, v12 # SendFallbackReason: Uncategorized(opt_mod) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_eq_fixnum() { + eval(" + def test(a, b) = a == b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :==, v12 # SendFallbackReason: Uncategorized(opt_eq) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_neq_fixnum() { + eval(" + def test(a, b) = a != b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :!=, v12 # SendFallbackReason: Uncategorized(opt_neq) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_lt_fixnum() { + eval(" + def test(a, b) = a < b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :<, v12 # SendFallbackReason: Uncategorized(opt_lt) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_le_fixnum() { + eval(" + def test(a, b) = a <= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :<=, v12 # SendFallbackReason: Uncategorized(opt_le) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_gt_fixnum() { + eval(" + def test(a, b) = a > b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :>, v12 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_loop() { + eval(" + def test + result = 0 + times = 10 + while times > 0 + result = result + 1 + times = times - 1 + end + result + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject): + EntryPoint JIT(0) + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[0] = Const Value(0) + v20:Fixnum[10] = Const Value(10) + CheckInterrupts + Jump bb4(v10, v16, v20) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + v32:Fixnum[0] = Const Value(0) + v35:BasicObject = SendWithoutBlock v28, :>, v32 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + v38:CBool = Test v35 + v39:Truthy = RefineType v35, Truthy + IfTrue v38, bb3(v26, v27, v28) + v41:Falsy = RefineType v35, Falsy + v43:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + bb3(v51:BasicObject, v52:BasicObject, v53:BasicObject): + v58:Fixnum[1] = Const Value(1) + v61:BasicObject = SendWithoutBlock v52, :+, v58 # SendFallbackReason: Uncategorized(opt_plus) + v66:Fixnum[1] = Const Value(1) + v69:BasicObject = SendWithoutBlock v53, :-, v66 # SendFallbackReason: Uncategorized(opt_minus) + Jump bb4(v51, v61, v69) + "); + } + + #[test] + fn test_opt_ge_fixnum() { + eval(" + def test(a, b) = a >= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :>=, v12 # SendFallbackReason: Uncategorized(opt_ge) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_display_types() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v19:CBool[true] = Test v13 + v20 = RefineType v13, Falsy + IfFalse v19, bb3(v8, v20) + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v25 + bb3(v30, v31): + v35 = Const Value(4) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_send_without_block() { + eval(" + def bar(a, b) + a+b + end + def test + bar(2, 3) + end + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + v13:Fixnum[3] = Const Value(3) + v15:BasicObject = SendWithoutBlock v6, :bar, v11, v13 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_send_with_block() { + eval(" + def test(a) + a.each {|item| + item + } + end + test([1,2,3]) + "); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:BasicObject = Send v9, 0x1000, :each # SendFallbackReason: Uncategorized(send) + v15:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_intern_interpolated_symbol() { + eval(r#" + def test + :"foo#{123}" + end + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[123] = Const Value(123) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + v21:Symbol = StringIntern v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn different_objects_get_addresses() { + eval("def test = unknown_method([0], [1], '2', '2')"); + + // The 2 string literals have the same address because they're deduped. + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:ArrayExact = ArrayDup v11 + v14:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:ArrayExact = ArrayDup v14 + v17:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v18:StringExact = StringCopy v17 + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + v23:BasicObject = SendWithoutBlock v6, :unknown_method, v12, v15, v18, v21 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_cant_compile_splat() { + eval(" + def test(a) = foo(*a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:ArrayExact = ToArray v9 + v17:BasicObject = SendWithoutBlock v8, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_compile_block_arg() { + eval(" + def test(a) = foo(&a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = Send v8, 0x1000, :foo, v9 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_cant_compile_kwarg() { + eval(" + def test(a) = foo(a: 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = SendWithoutBlock v8, :foo, v14 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_cant_compile_kw_splat() { + eval(" + def test(a) = foo(**a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v8, :foo, v9 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + // TODO(max): Figure out how to generate a call with TAILCALL flag + + #[test] + fn test_compile_super() { + eval(" + def test = super() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_compile_zsuper() { + eval(" + def test = super + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_cant_compile_super_nil_blockarg() { + eval(" + def test = super(&nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:NilClass = Const Value(nil) + v13:BasicObject = InvokeSuper v6, 0x1000, v11 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_cant_compile_super_forward() { + eval(" + def test(...) = super(...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + SideExit UnhandledYARVInsn(invokesuperforward) + "); + } + + #[test] + fn test_compile_forwardable() { + eval("def forwardable(...) = nil"); + assert_snapshot!(hir_string("forwardable"), @r" + fn forwardable@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + CheckInterrupts + Return v13 + "); + } + + // TODO(max): Figure out how to generate a call with OPT_SEND flag + + #[test] + fn test_cant_compile_kw_splat_mut() { + eval(" + def test(a) = foo **a, b: 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v16:HashExact = NewHash + PatchPoint NoEPEscape(test) + v21:BasicObject = SendWithoutBlock v14, :core#hash_merge_kwd, v16, v9 # SendFallbackReason: Uncategorized(opt_send_without_block) + v23:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v26:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v28:Fixnum[1] = Const Value(1) + v30:BasicObject = SendWithoutBlock v23, :core#hash_merge_ptr, v21, v26, v28 # SendFallbackReason: Uncategorized(opt_send_without_block) + v32:BasicObject = SendWithoutBlock v8, :foo, v30 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_cant_compile_splat_mut() { + eval(" + def test(*) = foo *, 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :*, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + v15:ArrayExact = ToNewArray v9 + v17:Fixnum[1] = Const Value(1) + ArrayPush v15, v17 + v21:BasicObject = SendWithoutBlock v8, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_compile_forwarding() { + eval(" + def test(...) = foo(...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendForward v8, 0x1000, :foo, v9 # SendFallbackReason: Uncategorized(sendforward) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_compile_triple_dots_with_positional_args() { + eval(" + def test(a, ...) = foo(a, ...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@8 + v3:ArrayExact = GetLocal :*, l0, SP@7, * + v4:BasicObject = GetLocal :**, l0, SP@6 + v5:BasicObject = GetLocal :&, l0, SP@5 + v6:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5, v6) + bb1(v9:BasicObject, v10:BasicObject, v11:ArrayExact, v12:BasicObject, v13:BasicObject): + EntryPoint JIT(0) + v14:NilClass = Const Value(nil) + Jump bb2(v9, v10, v11, v12, v13, v14) + bb2(v16:BasicObject, v17:BasicObject, v18:ArrayExact, v19:BasicObject, v20:BasicObject, v21:NilClass): + v28:ArrayExact = ToArray v18 + PatchPoint NoEPEscape(test) + GuardBlockParamProxy l0 + v34:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + SideExit UnhandledYARVInsn(splatkw) + "); + } + + #[test] + fn test_opt_new() { + eval(" + class C; end + def test = C.new + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + v13:NilClass = Const Value(nil) + v16:CBool = IsMethodCFunc v11, :new + IfFalse v16, bb3(v6, v13, v11) + v18:HeapBasicObject = ObjectAlloc v11 + v20:BasicObject = SendWithoutBlock v18, :initialize # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Jump bb4(v6, v18, v20) + bb3(v24:BasicObject, v25:NilClass, v26:BasicObject): + v29:BasicObject = SendWithoutBlock v26, :new # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb4(v24, v29, v25) + bb4(v32:BasicObject, v33:BasicObject, v34:BasicObject): + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_max_no_elements() { + eval(" + def test = [].max + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v11:BasicObject = ArrayMax + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v19:BasicObject = ArrayMax v11, v12 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX)) + "); + } + + #[test] + fn test_opt_newarray_send_min() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].min + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit UnhandledNewarraySend(MIN) + "); + } + + #[test] + fn test_opt_newarray_send_hash() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH) + v32:Fixnum = ArrayHash v15, v16 + PatchPoint NoEPEscape(test) + v39:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v40:ArrayExact = ArrayDup v39 + v42:BasicObject = SendWithoutBlock v14, :puts, v40 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH)) + "); + } + + #[test] + fn test_opt_newarray_send_pack() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v31:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v32:StringExact = StringCopy v31 + SideExit UnhandledNewarraySend(PACK) + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v29:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v30:StringExact = StringCopy v29 + v36:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v37:StringExact = StringCopy v36 + v39:BasicObject = GetLocal :buf, l0, EP@3 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v42:String = ArrayPackBuffer v15, v16, fmt: v37, buf: v39 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v29:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v30:StringExact = StringCopy v29 + v36:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v37:StringExact = StringCopy v36 + v39:BasicObject = GetLocal :buf, l0, EP@3 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_include_p() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v33:BoolExact = ArrayInclude v15, v16 | v16 + PatchPoint NoEPEscape(test) + v40:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v41:ArrayExact = ArrayDup v40 + v43:BasicObject = SendWithoutBlock v14, :puts, v41 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v15:BoolExact = DupArrayInclude VALUE(0x1000) | v9 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + v21:BasicObject = SendWithoutBlock v18, :length # SendFallbackReason: Uncategorized(opt_length) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + v21:BasicObject = SendWithoutBlock v18, :size # SendFallbackReason: Uncategorized(opt_size) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + test + "); + assert_contains_opcode("test", YARVINSN_getinstancevariable); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setinstancevariable); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_set_ivar_rescue_frozen() { + let result = eval(" + class Foo + attr_accessor :bar + def initialize + @bar = 1 + freeze + end + end + + def test(foo) + begin + foo.bar = 2 + rescue FrozenError + end + end + + foo = Foo.new + test(foo) + test(foo) + + foo.bar + "); + assert_eq!(VALUE::fixnum_from_usize(1), result); + } + + #[test] + fn test_getclassvariable() { + eval(" + class Foo + def self.test = @@foo + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_getclassvariable), "iseq Foo.test does not contain getclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetClassVar :@@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setclassvariable() { + eval(" + class Foo + def self.test = @@foo = 42 + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_setclassvariable), "iseq Foo.test does not contain setclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[42] = Const Value(42) + SetClassVar :@@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setglobal() { + eval(" + def test = $foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + SetGlobal :$foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getglobal() { + eval(" + def test = $foo + test + "); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetGlobal :$foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_splatarray_mut() { + eval(" + def test(a) = [*a] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_concattoarray() { + eval(" + def test(a) = [1, *a] + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v15:ArrayExact = NewArray v13 + v18:ArrayExact = ToArray v9 + ArrayExtend v15, v18 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_pushtoarray_one_element() { + eval(" + def test(a) = [*a, 1] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + v16:Fixnum[1] = Const Value(1) + ArrayPush v14, v16 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_pushtoarray_multiple_elements() { + eval(" + def test(a) = [*a, 1, 2, 3] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[2] = Const Value(2) + v20:Fixnum[3] = Const Value(3) + ArrayPush v14, v16 + ArrayPush v14, v18 + ArrayPush v14, v20 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_aset() { + eval(" + def test(a, b) = a[b] = 1 + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v16:NilClass = Const Value(nil) + v20:Fixnum[1] = Const Value(1) + v24:BasicObject = SendWithoutBlock v11, :[]=, v12, v20 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_aref() { + eval(" + def test(a, b) = a[b] + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :[], v12 # SendFallbackReason: Uncategorized(opt_aref) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :empty? # SendFallbackReason: Uncategorized(opt_empty_p) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_succ() { + eval(" + def test(x) = x.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :succ # SendFallbackReason: Uncategorized(opt_succ) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :&, v12 # SendFallbackReason: Uncategorized(opt_and) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :|, v12 # SendFallbackReason: Uncategorized(opt_or) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_not() { + eval(" + def test(x) = !x + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :! # SendFallbackReason: Uncategorized(opt_not) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_regexpmatch2() { + eval(" + def test(regexp, matchee) = regexp =~ matchee + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :regexp, l0, SP@5 + v3:BasicObject = GetLocal :matchee, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :=~, v12 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + CheckInterrupts + Return v19 + "); + } + + #[test] + // Tests for ConstBase requires either constant or class definition, both + // of which can't be performed inside a method. + fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias aliased __callee__ + end + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v12:BasicObject = PutSpecialObject CBase + v14:StaticSymbol[:aliased] = Const Value(VALUE(0x1008)) + v16:StaticSymbol[:__callee__] = Const Value(VALUE(0x1010)) + v18:BasicObject = SendWithoutBlock v10, :core#set_method_alias, v12, v14, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn opt_reverse() { + eval(" + def reverse_odd + a, b, c = @a, @b, @c + [a, b, c] + end + + def reverse_even + a, b, c, d = @a, @b, @c, @d + [a, b, c, d] + end + "); + assert_contains_opcode("reverse_odd", YARVINSN_opt_reverse); + assert_contains_opcode("reverse_even", YARVINSN_opt_reverse); + assert_snapshot!(hir_strings!("reverse_odd", "reverse_even"), @r" + fn reverse_odd@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:NilClass, v14:NilClass, v15:NilClass): + PatchPoint SingleRactorMode + v20:BasicObject = GetIvar v12, :@a + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v12, :@b + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v12, :@c + PatchPoint NoEPEscape(reverse_odd) + v38:ArrayExact = NewArray v20, v23, v26 + CheckInterrupts + Return v38 + + fn reverse_even@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:NilClass, v16:NilClass, v17:NilClass, v18:NilClass): + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v14, :@a + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v14, :@b + PatchPoint SingleRactorMode + v29:BasicObject = GetIvar v14, :@c + PatchPoint SingleRactorMode + v32:BasicObject = GetIvar v14, :@d + PatchPoint NoEPEscape(reverse_even) + v46:ArrayExact = NewArray v23, v26, v29, v32 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_branchnil() { + eval(" + def test(x) = x&.itself + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v16:CBool = IsNil v9 + v17:NilClass = Const Value(nil) + IfTrue v16, bb3(v8, v17, v17) + v19:NotNil = RefineType v9, NotNil + v21:BasicObject = SendWithoutBlock v19, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb3(v8, v19, v21) + bb3(v23:BasicObject, v24:BasicObject, v25:BasicObject): + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_infer_nilability_from_branchif() { + eval(" + def test(x) + if x + x&.itself + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_branchnil); + // Note that IsNil has as its operand a value that we know statically *cannot* be nil + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + CheckInterrupts + v24:CBool[false] = IsNil v18 + v25:NilClass = Const Value(nil) + IfTrue v24, bb4(v8, v25, v25) + v27:Truthy = RefineType v18, NotNil + v29:BasicObject = SendWithoutBlock v27, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v29 + bb3(v34:BasicObject, v35:Falsy): + v39:Fixnum[4] = Const Value(4) + Jump bb4(v34, v35, v39) + bb4(v41:BasicObject, v42:Falsy, v43:Fixnum[4]): + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_invokebuiltin_delegate_annotated() { + assert_contains_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("Float"), @r" + fn Float@<internal:kernel>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arg, l0, SP@6 + v3:BasicObject = GetLocal :exception, l0, SP@5 + v4:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:BasicObject = GetLocal <empty>, l0, EP@3 + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + v19:Float = InvokeBuiltin rb_f_float, v12, v13, v14 + Jump bb3(v12, v13, v14, v15, v19) + bb3(v21:BasicObject, v22:BasicObject, v23:BasicObject, v24:BasicObject, v25:Float): + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_invokebuiltin_cexpr_annotated() { + assert_contains_opcode("class", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("class"), @r" + fn class@<internal:kernel>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HeapObject = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:HeapObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_delegate_with_args() { + // Using an unannotated builtin to test InvokeBuiltin generation + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn open@<internal:dir>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :name, l0, SP@8 + v3:BasicObject = GetLocal :encoding, l0, SP@7 + v4:BasicObject = GetLocal <empty>, l0, SP@6 + v5:BasicObject = GetLocal :block, l0, SP@5 + v6:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5, v6) + bb1(v9:BasicObject, v10:BasicObject, v11:BasicObject, v13:BasicObject): + EntryPoint JIT(0) + v12:BasicObject = GetLocal <empty>, l0, EP@5 + v14:NilClass = Const Value(nil) + Jump bb2(v9, v10, v11, v12, v13, v14) + bb2(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:NilClass): + v25:BasicObject = InvokeBuiltin dir_s_open, v16, v17, v18 + PatchPoint NoEPEscape(open) + GuardBlockParamProxy l0 + v32:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + CheckInterrupts + v35:CBool[true] = Test v32 + v36 = RefineType v32, Falsy + IfFalse v35, bb3(v16, v17, v18, v19, v20, v25) + v38:HeapObject[BlockParamProxy] = RefineType v32, Truthy + v42:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) + v45:BasicObject = InvokeBuiltin dir_s_close, v16, v25 + CheckInterrupts + Return v42 + bb3(v51, v52, v53, v54, v55, v56): + CheckInterrupts + Return v56 + "); + } + + #[test] + fn test_invokebuiltin_delegate_without_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "enable")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate_leave), "iseq GC.enable does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn enable@<internal:gc>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = InvokeBuiltin gc_enable, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_with_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "start")); + assert!(iseq_contains_opcode(iseq, YARVINSN_invokebuiltin), "iseq GC.start does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn start@<internal:gc>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :full_mark, l0, SP@7 + v3:BasicObject = GetLocal :immediate_mark, l0, SP@6 + v4:BasicObject = GetLocal :immediate_sweep, l0, SP@5 + v5:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject): + EntryPoint JIT(0) + v12:BasicObject = GetLocal <empty>, l0, EP@3 + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:BasicObject): + v25:FalseClass = Const Value(false) + v27:BasicObject = InvokeBuiltin gc_start_internal, v14, v15, v16, v17, v25 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_name() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "name")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn name@<internal:symbol>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_to_s() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "to_s")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn to_s@<internal:symbol>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dupn() { + eval(" + def test(x) = (x[0, 1] ||= 2) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + v16:Fixnum[0] = Const Value(0) + v18:Fixnum[1] = Const Value(1) + v21:BasicObject = SendWithoutBlock v9, :[], v16, v18 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + v25:CBool = Test v21 + v26:Truthy = RefineType v21, Truthy + IfTrue v25, bb3(v8, v9, v13, v9, v16, v18, v26) + v28:Falsy = RefineType v21, Falsy + v31:Fixnum[2] = Const Value(2) + v34:BasicObject = SendWithoutBlock v9, :[]=, v16, v18, v31 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v31 + bb3(v40:BasicObject, v41:BasicObject, v42:NilClass, v43:BasicObject, v44:Fixnum[0], v45:Fixnum[1], v46:Truthy): + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_objtostring_anytostring() { + eval(" + def test = \"#{1}\" + "); + assert_contains_opcode("test", YARVINSN_objtostring); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:StringExact = StringConcat v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:NilClass = Const Value(nil) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_toregexp() { + eval(r##" + def test = /#{1}#{2}#{3}/ + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:RegexpExact = ToRegexp v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_toregexp_with_options() { + eval(r##" + def test = /#{1}#{2}/mixn + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:RegexpExact = ToRegexp v15, v22, MULTILINE|IGNORECASE|EXTENDED|NOENCODING + CheckInterrupts + Return v24 + "); + } + + #[test] + fn throw() { + eval(" + define_method(:throw_return) { return 1 } + define_method(:throw_break) { break 2 } + "); + assert_contains_opcode("throw_return", YARVINSN_throw); + assert_contains_opcode("throw_break", YARVINSN_throw); + assert_snapshot!(hir_strings!("throw_return", "throw_break"), @r" + fn block in <compiled>@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v12:Fixnum[1] = Const Value(1) + Throw TAG_RETURN, v12 + + fn block in <compiled>@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v12:Fixnum[2] = Const Value(2) + Throw TAG_BREAK, v12 + "); + } + + #[test] + fn test_invokeblock() { + eval(r#" + def test + yield + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = InvokeBlock # SendFallbackReason: Uncategorized(invokeblock) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_invokeblock_with_args() { + eval(r#" + def test(x, y) + yield x, y + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:BasicObject = InvokeBlock, v11, v12 # SendFallbackReason: Uncategorized(invokeblock) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_expandarray_no_splat() { + eval(r#" + def test(o) + a, b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@6 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:NilClass, v15:NilClass): + v21:ArrayExact = GuardType v13, ArrayExact + v22:CInt64 = ArrayLength v21 + v23:CInt64[2] = GuardBitEquals v22, CInt64(2) + v24:CInt64[1] = Const CInt64(1) + v25:BasicObject = ArrayAref v21, v24 + v26:CInt64[0] = Const CInt64(0) + v27:BasicObject = ArrayAref v21, v26 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_expandarray_splat() { + eval(r#" + def test(o) + a, *b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@6 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:NilClass, v15:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_expandarray_splat_post() { + eval(r#" + def test(o) + a, *b, c = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@7 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:NilClass, v17:NilClass, v18:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_checkkeyword_tests_fixnum_bit() { + eval(r#" + def test(kw: 1 + 1) = kw + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :kw, l0, SP@5 + v3:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:BasicObject = GetLocal <empty>, l0, EP@3 + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v15:BoolExact = FixnumBitCheck v12, 0 + CheckInterrupts + v18:CBool = Test v15 + v19:TrueClass = RefineType v15, Truthy + IfTrue v18, bb3(v10, v11, v12) + v21:FalseClass = RefineType v15, Falsy + v23:Fixnum[1] = Const Value(1) + v25:Fixnum[1] = Const Value(1) + v28:BasicObject = SendWithoutBlock v23, :+, v25 # SendFallbackReason: Uncategorized(opt_plus) + Jump bb3(v10, v28, v12) + bb3(v31:BasicObject, v32:BasicObject, v33:BasicObject): + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_checkkeyword_too_many_keywords_causes_side_exit() { + eval(r#" + def test(k1: k1, k2: k2, k3: k3, k4: k4, k5: k5, + k6: k6, k7: k7, k8: k8, k9: k9, k10: k10, k11: k11, + k12: k12, k13: k13, k14: k14, k15: k15, k16: k16, + k17: k17, k18: k18, k19: k19, k20: k20, k21: k21, + k22: k22, k23: k23, k24: k24, k25: k25, k26: k26, + k27: k27, k28: k28, k29: k29, k30: k30, k31: k31, + k32: k32, k33: k33) = k1 + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k1, l0, SP@37 + v3:BasicObject = GetLocal :k2, l0, SP@36 + v4:BasicObject = GetLocal :k3, l0, SP@35 + v5:BasicObject = GetLocal :k4, l0, SP@34 + v6:BasicObject = GetLocal :k5, l0, SP@33 + v7:BasicObject = GetLocal :k6, l0, SP@32 + v8:BasicObject = GetLocal :k7, l0, SP@31 + v9:BasicObject = GetLocal :k8, l0, SP@30 + v10:BasicObject = GetLocal :k9, l0, SP@29 + v11:BasicObject = GetLocal :k10, l0, SP@28 + v12:BasicObject = GetLocal :k11, l0, SP@27 + v13:BasicObject = GetLocal :k12, l0, SP@26 + v14:BasicObject = GetLocal :k13, l0, SP@25 + v15:BasicObject = GetLocal :k14, l0, SP@24 + v16:BasicObject = GetLocal :k15, l0, SP@23 + v17:BasicObject = GetLocal :k16, l0, SP@22 + v18:BasicObject = GetLocal :k17, l0, SP@21 + v19:BasicObject = GetLocal :k18, l0, SP@20 + v20:BasicObject = GetLocal :k19, l0, SP@19 + v21:BasicObject = GetLocal :k20, l0, SP@18 + v22:BasicObject = GetLocal :k21, l0, SP@17 + v23:BasicObject = GetLocal :k22, l0, SP@16 + v24:BasicObject = GetLocal :k23, l0, SP@15 + v25:BasicObject = GetLocal :k24, l0, SP@14 + v26:BasicObject = GetLocal :k25, l0, SP@13 + v27:BasicObject = GetLocal :k26, l0, SP@12 + v28:BasicObject = GetLocal :k27, l0, SP@11 + v29:BasicObject = GetLocal :k28, l0, SP@10 + v30:BasicObject = GetLocal :k29, l0, SP@9 + v31:BasicObject = GetLocal :k30, l0, SP@8 + v32:BasicObject = GetLocal :k31, l0, SP@7 + v33:BasicObject = GetLocal :k32, l0, SP@6 + v34:BasicObject = GetLocal :k33, l0, SP@5 + v35:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) + bb1(v38:BasicObject, v39:BasicObject, v40:BasicObject, v41:BasicObject, v42:BasicObject, v43:BasicObject, v44:BasicObject, v45:BasicObject, v46:BasicObject, v47:BasicObject, v48:BasicObject, v49:BasicObject, v50:BasicObject, v51:BasicObject, v52:BasicObject, v53:BasicObject, v54:BasicObject, v55:BasicObject, v56:BasicObject, v57:BasicObject, v58:BasicObject, v59:BasicObject, v60:BasicObject, v61:BasicObject, v62:BasicObject, v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject, v69:BasicObject, v70:BasicObject, v71:BasicObject): + EntryPoint JIT(0) + v72:BasicObject = GetLocal <empty>, l0, EP@3 + Jump bb2(v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63, v64, v65, v66, v67, v68, v69, v70, v71, v72) + bb2(v74:BasicObject, v75:BasicObject, v76:BasicObject, v77:BasicObject, v78:BasicObject, v79:BasicObject, v80:BasicObject, v81:BasicObject, v82:BasicObject, v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject, v91:BasicObject, v92:BasicObject, v93:BasicObject, v94:BasicObject, v95:BasicObject, v96:BasicObject, v97:BasicObject, v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject, v102:BasicObject, v103:BasicObject, v104:BasicObject, v105:BasicObject, v106:BasicObject, v107:BasicObject, v108:BasicObject): + SideExit TooManyKeywordParameters + "); + } + } + + /// Test successor and predecessor set computations. + #[cfg(test)] + mod control_flow_info_tests { + use super::*; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb1, bb2)); + assert!(cfi.is_succeeded_by(bb2, bb1)); + assert!(cfi.predecessors(bb3).eq([bb2])); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v1, target: edge(bb2)}); + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb2, bb3)); + assert!(cfi.is_preceded_by(bb1, bb3)); + assert!(!cfi.is_preceded_by(bb0, bb3)); + assert!(cfi.is_succeeded_by(bb1, bb0)); + assert!(cfi.is_succeeded_by(bb3, bb1)); + } + + #[test] + fn test_cfi_deduplicated_successors_and_predecessors() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + // Construct two separate jump instructions. + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v1, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert_eq!(cfi.predecessors(bb1).collect::<Vec<_>>().len(), 1); + assert_eq!(cfi.successors(bb0).collect::<Vec<_>>().len(), 1); + } + } + + /// Test dominator set computations. + #[cfg(test)] + mod dom_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + fn assert_dominators_contains_self(function: &Function, dominators: &Dominators) { + for (i, _) in function.blocks.iter().enumerate() { + // Ensure that each dominating set contains the block itself. + assert!(dominators.is_dominated_by(BlockId(i), BlockId(i))); + } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v3:Any = Const CBool(true) + Return v3 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb1, bb2, bb3].iter())); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb2() + bb1(): + Jump bb3() + bb2(): + Jump bb3() + bb3(): + v5:Any = Const CBool(true) + Return v5 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb3].iter())); + } + + #[test] + fn test_complex_cfg() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + let bb7 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let v0 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::IfTrue { val: v0, target: edge(bb2)}); + function.push_insn(bb1, Insn::Jump(edge(bb4))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let v1 = function.push_insn(bb3, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb3, Insn::IfTrue { val: v1, target: edge(bb5)}); + function.push_insn(bb3, Insn::Jump(edge(bb7))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + function.push_insn(bb6, Insn::Jump(edge(bb7))); + + let retval = function.push_insn(bb7, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb7, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + v1:Any = Const Value(false) + IfTrue v1, bb2() + Jump bb4() + bb2(): + Jump bb3() + bb3(): + v5:Any = Const Value(false) + IfTrue v5, bb5() + Jump bb7() + bb4(): + Jump bb5() + bb5(): + Jump bb6() + bb6(): + Jump bb7() + bb7(): + v11:Any = Const CBool(true) + Return v11 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb1, bb2, bb3].iter())); + assert!(dominators.dominators(bb4).eq([bb0, bb1, bb4].iter())); + assert!(dominators.dominators(bb5).eq([bb0, bb1, bb5].iter())); + assert!(dominators.dominators(bb6).eq([bb0, bb1, bb5, bb6].iter())); + assert!(dominators.dominators(bb7).eq([bb0, bb1, bb7].iter())); + } + + #[test] + fn test_back_edges() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let v0 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v0, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb4))); + + let v1 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::IfTrue { val: v1, target: edge(bb2)}); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let v2 = function.push_insn(bb5, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb5, Insn::IfTrue { val: v2, target: edge(bb3)}); + function.push_insn(bb5, Insn::Jump(edge(bb4))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb4() + bb1(): + v3:Any = Const Value(false) + IfTrue v3, bb2() + Jump bb3() + bb2(): + Jump bb3() + bb4(): + Jump bb5() + bb5(): + v8:Any = Const Value(false) + IfTrue v8, bb3() + Jump bb4() + bb3(): + v11:Any = Const CBool(true) + Return v11 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb3].iter())); + assert!(dominators.dominators(bb4).eq([bb0, bb4].iter())); + assert!(dominators.dominators(bb5).eq([bb0, bb4, bb5].iter())); + } + + #[test] + fn test_multiple_entry_blocks() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + function.jit_entry_blocks.push(bb1); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb2, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb2() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const CBool(true) + Return v2 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + + assert!(dominators.dominators(bb1).eq([bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb2].iter())); + + assert!(!dominators.is_dominated_by(bb1, bb2)); + } + } + + /// Test loop information computation. +#[cfg(test)] +mod loop_info_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_loop_depth() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ ┌─────┐ + // │ bb2 ◄──────┼ bb1 ◄─┐ + // └──┬──┘ └─────┘ │ + // └─────────────────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val, target: edge(bb1)}); + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb2() + v1:Any = Const Value(false) + bb2(): + IfTrue v1, bb1() + v3:Any = Const CBool(true) + Return v3 + bb1(): + Jump bb2() + "); + + assert!(loop_info.is_loop_header(bb2)); + assert!(loop_info.is_back_edge_source(bb1)); + assert_eq!(loop_info.loop_depth(bb1), 1); + } + + #[test] + fn test_nested_loops() { + // ┌─────┐ + // │ bb0 ◄─────┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄───┐ │ + // └──┬──┘ │ │ + // │ │ │ + // ┌──▼──┐ │ │ + // │ bb2 ┼───┘ │ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb3 ┼─────┘ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb4 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cond = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let cond = function.push_insn(bb3, Insn::Const { val: Const::Value(Qtrue) }); + let _ = function.push_insn(bb3, Insn::IfTrue { val: cond, target: edge(bb0) }); + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let retval = function.push_insn(bb4, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb4, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const Value(false) + IfTrue v2, bb1() + Jump bb3() + bb3(): + v5:Any = Const Value(true) + IfTrue v5, bb0() + Jump bb4() + bb4(): + v8:Any = Const CBool(true) + Return v8 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 1); + assert_eq!(loop_info.loop_depth(bb4), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + } + + #[test] + fn test_complex_loops() { + // ┌─────┐ + // ┌──────► bb0 │ + // │ └──┬──┘ + // │ ┌────┴────┐ + // │ ┌──▼──┐ ┌──▼──┐ + // │ │ bb1 ◄─┐ │ bb3 ◄─┐ + // │ └──┬──┘ │ └──┬──┘ │ + // │ │ │ │ │ + // │ ┌──▼──┐ │ ┌──▼──┐ │ + // │ │ bb2 ┼─┘ │ bb4 ┼─┘ + // │ └──┬──┘ └──┬──┘ + // │ └────┬────┘ + // │ ┌──▼──┐ + // └──────┼ bb5 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb6 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb0, Insn::Jump(edge(bb3))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let _ = function.push_insn(bb2, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb2, Insn::Jump(edge(bb5))); + + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let _ = function.push_insn(bb4, Insn::IfTrue { val: cond, target: edge(bb3) }); + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let _ = function.push_insn(bb5, Insn::IfTrue { val: cond, target: edge(bb0) }); + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + let retval = function.push_insn(bb6, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb6, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb3() + bb1(): + Jump bb2() + bb2(): + IfTrue v0, bb1() + Jump bb5() + bb3(): + Jump bb4() + bb4(): + IfTrue v0, bb3() + Jump bb5() + bb5(): + IfTrue v0, bb0() + Jump bb6() + bb6(): + v11:Any = Const CBool(true) + Return v11 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + assert!(loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb5)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb6)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 2); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + assert_eq!(loop_info.loop_depth(bb6), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + } + + #[test] + fn linked_list_non_loop() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb1 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb2 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const CBool(true) + Return v2 + "); + + assert!(!loop_info.is_loop_header(bb0)); + assert!(!loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + + assert_eq!(loop_info.loop_depth(bb0), 0); + assert_eq!(loop_info.loop_depth(bb1), 0); + assert_eq!(loop_info.loop_depth(bb2), 0); + } + + #[test] + fn triple_nested_loop() { + // ┌─────┐ + // │ bb0 ◄──┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄─┐│ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb2 ◄┐││ + // └──┬──┘│││ + // │ │││ + // ┌──▼──┐│││ + // │ bb3 ┼┘││ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb4 ┼─┘│ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb5 ┼──┘ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + let _ = function.push_insn(bb2, Insn::Jump(edge(bb3))); + let _ = function.push_insn(bb3, Insn::Jump(edge(bb4))); + let _ = function.push_insn(bb3, Insn::IfTrue {val: cond, target: edge(bb2)}); + let _ = function.push_insn(bb4, Insn::Jump(edge(bb5))); + let _ = function.push_insn(bb4, Insn::IfTrue {val: cond, target: edge(bb1)}); + let _ = function.push_insn(bb5, Insn::IfTrue {val: cond, target: edge(bb0)}); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + Jump bb1() + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + IfTrue v0, bb2() + bb4(): + Jump bb5() + IfTrue v0, bb1() + bb5(): + IfTrue v0, bb0() + "); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 3); + assert_eq!(loop_info.loop_depth(bb3), 3); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(loop_info.is_loop_header(bb2)); + assert!(!loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb5)); + } + } + +/// Test dumping to iongraph format. +#[cfg(test)] +mod iongraph_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_simple_function() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let retval = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: retval }); + + let json = function.to_iongraph_pass("simple"); + assert_snapshot!(json.to_string(), @r#"{"name":"simple", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_two_blocks() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + let json = function.to_iongraph_pass("two_blocks"); + assert_snapshot!(json.to_string(), @r#"{"name":"two_blocks", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb1()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4097, "id":1, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"Return v1", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_instructions() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let val1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: val1 }); + + let json = function.to_iongraph_pass("multiple_instructions"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_instructions", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_conditional_branch() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + + let retval1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb0, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval2 }); + + let json = function.to_iongraph_pass("conditional_branch"); + assert_snapshot!(json.to_string(), @r#"{"name":"conditional_branch", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"IfTrue v0, bb1()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}, {"ptr":4098, "id":2, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_loop_structure() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val, target: edge(bb1)}); + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let json = function.to_iongraph_pass("loop_structure"); + assert_snapshot!(json.to_string(), @r#"{"name":"loop_structure", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[2], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}, {"ptr":4097, "id":1, "opcode":"Const Value(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}]}, {"ptr":4098, "id":2, "loopDepth":1, "attributes":["loopheader"], "predecessors":[0, 1], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"IfTrue v1, bb1()", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}, {"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":1, "attributes":["backedge"], "predecessors":[2], "successors":[2], "instructions":[{"ptr":4101, "id":5, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_successors() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let retval1 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval2 }); + + let json = function.to_iongraph_pass("multiple_successors"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_successors", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1, 2], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"IfTrue v0, bb1()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}, {"ptr":4098, "id":2, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4101, "id":5, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4102, "id":6, "opcode":"Return v5", "attributes":[], "inputs":[5], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + } diff --git a/zjit/src/hir_effect/gen_hir_effect.rb b/zjit/src/hir_effect/gen_hir_effect.rb new file mode 100644 index 0000000000..51cc712feb --- /dev/null +++ b/zjit/src/hir_effect/gen_hir_effect.rb @@ -0,0 +1,119 @@ +# Generate hir_effect.inc.rs. To do this, we build up a DAG that +# represents the ZJIT effect hierarchy. + +require 'set' + +# Effect represents not just a Ruby class but a named union of other effects. +class Effect + attr_accessor :name, :subeffects + + def initialize name, subeffects=nil + @name = name + @subeffects = subeffects || [] + end + + def all_subeffects + subeffects.flat_map { |subeffect| subeffect.all_subeffects } + subeffects + end + + def subeffect name + result = Effect.new name + @subeffects << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec effect + effect.subeffects.each {|subeffect| + puts effect.name + "->" + subeffect.name + ";" + } + effect.subeffect.each {|subeffect| + to_graphviz_rec subeffect + } +end + +# Generate graphviz. +def to_graphviz effect + puts "digraph G {" + to_graphviz_rec effect + puts "}" +end + +# ===== Start generating the effect DAG ===== + +# Start at Any. All effects are subeffects of Any. +any = Effect.new 'Any' +# Build the effect universe. +allocator = any.subeffect 'Allocator' +control = any.subeffect 'Control' +memory = any.subeffect 'Memory' +other = memory.subeffect 'Other' +frame = memory.subeffect 'Frame' +pc = frame.subeffect 'PC' +locals = frame.subeffect 'Locals' +stack = frame.subeffect 'Stack' + +# Use the smallest unsigned value needed to describe all effect bits +# If it becomes an issue, this can be generated but for now we do it manually +$int_label = 'u8' + +# Assign individual bits to effect leaves and union bit patterns to nodes with subeffects +num_bits = 0 +$bits = {"Empty" => ["0#{$int_label}"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subeffects].sort_by(&:name).each {|effect| + subeffects = effect.subeffects + if subeffects.empty? + # Assign bits for leaves + $bits[effect.name] = ["1#{$int_label} << #{num_bits}"] + $numeric_bits[effect.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[effect.name] = subeffects.map(&:name).sort + end +} +[*any.all_subeffects, any].each {|effect| + subeffects = effect.subeffects + unless subeffects.empty? + $numeric_bits[effect.name] = subeffects.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir/gen_hir_effect.rb." +puts "mod bits {" +$bits.keys.sort.map {|effect_name| + subeffects = $bits[effect_name].join(" | ") + puts " pub const #{effect_name}: #{$int_label} = #{subeffects};" +} +puts " pub const AllBitPatterns: [(&str, #{$int_label}); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of an Effect. For example, Frame instead of +# PC|Stack|Locals +$numeric_bits.sort_by {|key, val| -val}.each {|effect_name, _| + puts " (\"#{effect_name}\", #{effect_name})," +} +puts " ];" +puts " pub const NumEffectBits: #{$int_label} = #{num_bits}; +}" + +puts "pub mod effect_types {" +puts " pub type EffectBits = #{$int_label};" +puts "}" + +puts "pub mod abstract_heaps { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: AbstractHeap = AbstractHeap::from_bits(bits::#{effect_name});" +} +puts "}" + +puts "pub mod effects { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: Effect = Effect::promote(abstract_heaps::#{effect_name});" +} +puts "}" diff --git a/zjit/src/hir_effect/hir_effect.inc.rs b/zjit/src/hir_effect/hir_effect.inc.rs new file mode 100644 index 0000000000..d9566b3eaa --- /dev/null +++ b/zjit/src/hir_effect/hir_effect.inc.rs @@ -0,0 +1,55 @@ +// This file is @generated by src/hir/gen_hir_effect.rb. +mod bits { + pub const Allocator: u8 = 1u8 << 0; + pub const Any: u8 = Allocator | Control | Memory; + pub const Control: u8 = 1u8 << 1; + pub const Empty: u8 = 0u8; + pub const Frame: u8 = Locals | PC | Stack; + pub const Locals: u8 = 1u8 << 2; + pub const Memory: u8 = Frame | Other; + pub const Other: u8 = 1u8 << 3; + pub const PC: u8 = 1u8 << 4; + pub const Stack: u8 = 1u8 << 5; + pub const AllBitPatterns: [(&str, u8); 10] = [ + ("Any", Any), + ("Memory", Memory), + ("Frame", Frame), + ("Stack", Stack), + ("PC", PC), + ("Other", Other), + ("Locals", Locals), + ("Control", Control), + ("Allocator", Allocator), + ("Empty", Empty), + ]; + pub const NumEffectBits: u8 = 6; +} +pub mod effect_types { + pub type EffectBits = u8; +} +pub mod abstract_heaps { + use super::*; + pub const Allocator: AbstractHeap = AbstractHeap::from_bits(bits::Allocator); + pub const Any: AbstractHeap = AbstractHeap::from_bits(bits::Any); + pub const Control: AbstractHeap = AbstractHeap::from_bits(bits::Control); + pub const Empty: AbstractHeap = AbstractHeap::from_bits(bits::Empty); + pub const Frame: AbstractHeap = AbstractHeap::from_bits(bits::Frame); + pub const Locals: AbstractHeap = AbstractHeap::from_bits(bits::Locals); + pub const Memory: AbstractHeap = AbstractHeap::from_bits(bits::Memory); + pub const Other: AbstractHeap = AbstractHeap::from_bits(bits::Other); + pub const PC: AbstractHeap = AbstractHeap::from_bits(bits::PC); + pub const Stack: AbstractHeap = AbstractHeap::from_bits(bits::Stack); +} +pub mod effects { + use super::*; + pub const Allocator: Effect = Effect::promote(abstract_heaps::Allocator); + pub const Any: Effect = Effect::promote(abstract_heaps::Any); + pub const Control: Effect = Effect::promote(abstract_heaps::Control); + pub const Empty: Effect = Effect::promote(abstract_heaps::Empty); + pub const Frame: Effect = Effect::promote(abstract_heaps::Frame); + pub const Locals: Effect = Effect::promote(abstract_heaps::Locals); + pub const Memory: Effect = Effect::promote(abstract_heaps::Memory); + pub const Other: Effect = Effect::promote(abstract_heaps::Other); + pub const PC: Effect = Effect::promote(abstract_heaps::PC); + pub const Stack: Effect = Effect::promote(abstract_heaps::Stack); +} diff --git a/zjit/src/hir_effect/mod.rs b/zjit/src/hir_effect/mod.rs new file mode 100644 index 0000000000..b1d7b27411 --- /dev/null +++ b/zjit/src/hir_effect/mod.rs @@ -0,0 +1,420 @@ +//! High-level intermediate representation effects. + +#![allow(non_upper_case_globals)] +use crate::hir::{PtrPrintMap}; +include!("hir_effect.inc.rs"); + +// NOTE: Effect very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +/// The AbstractHeap struct is the main work horse of effect inference and specialization. The main interfaces +/// will look like: +/// +/// * is AbstractHeap A a subset of AbstractHeap B +/// * union/meet AbstractHeap A and AbstractHeap B +/// +/// or +/// +/// * is Effect A a subset of Effect B +/// * union/meet Effect A and Effect B +/// +/// The AbstractHeap is the work horse because Effect is simply 2 AbstractHeaps; one for read, and one for write. +/// Currently, the abstract heap is implemented as a bitset. As we enrich our effect system, this will be updated +/// to match the name and use a heap implementation, roughly aligned with +/// <https://gist.github.com/pizlonator/cf1e72b8600b1437dda8153ea3fdb963>. +/// +/// Most questions can be rewritten in terms of these operations. +/// +/// Lattice Top corresponds to the "Any" effect. All bits are set and any effect is possible. +/// Lattice Bottom corresponds to the "None" effect. No bits are set and no effects are possible. +/// Elements between abstract_heaps have effects corresponding to the bits that are set. +/// This enables more complex analyses compared to prior ZJIT implementations such as "has_effect", +/// a function that returns a boolean value. Such functions impose an implicit single bit effect +/// system. This explicit design with a lattice allows us many bits for effects. +#[derive(Clone, Copy, Debug)] +pub struct AbstractHeap { + bits: effect_types::EffectBits +} + +#[derive(Clone, Copy, Debug)] +pub struct Effect { + /// Unlike ZJIT's type system, effects do not have a notion of subclasses. + /// Instead of specializations, the Effect struct contains two AbstractHeaps. + /// We distinguish between read effects and write effects. + /// Both use the same effects lattice, but splitting into two heaps allows + /// for finer grained optimization. + /// + /// For instance: + /// We can elide HIR instructions with no write effects, but the read effects are necessary for instruction + /// reordering optimizations. + /// + /// These fields should not be directly read or written except by internal `Effect` APIs. + read: AbstractHeap, + write: AbstractHeap +} + +/// Print adaptor for [`AbstractHeap`]. See [`PtrPrintMap`]. +pub struct AbstractHeapPrinter<'a> { + inner: AbstractHeap, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for AbstractHeapPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let effect = self.inner; + let mut bits = effect.bits; + let mut sep = ""; + // First, make sure patterns are sorted from higher order bits to lower order. + // For each match where `bits` contains the pattern, we mask off the matched bits + // and continue searching for matches until bits == 0. + // Our first match could be exact and may not require a separator, but all subsequent + // matches do. + debug_assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + for (name, pattern) in bits::AllBitPatterns { + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + // The `sep != ""` check allows us to handle the effects::None case gracefully. + if (bits == 0) & (sep != "") { break; } + } + debug_assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + Ok(()) + } +} + +impl std::fmt::Display for AbstractHeap { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +/// Print adaptor for [`Effect`]. See [`PtrPrintMap`]. +pub struct EffectPrinter<'a> { + inner: Effect, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for EffectPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}, {}", self.inner.read, self.inner.write) + } +} + +impl std::fmt::Display for Effect { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl AbstractHeap { + const fn from_bits(bits: effect_types::EffectBits) -> Self { + Self { bits } + } + + pub const fn union(&self, other: Self) -> Self { + Self::from_bits(self.bits | other.bits) + } + + pub const fn intersect(&self, other: Self) -> Self { + Self::from_bits(self.bits & other.bits) + } + + pub const fn exclude(&self, other: Self) -> Self { + Self::from_bits(self.bits - (self.bits & other.bits)) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Self) -> bool { + self.bits == other.bits + } + + pub const fn includes(&self, other: Self) -> bool { + self.bit_equal( + self.union(other) + ) + } + + pub const fn overlaps(&self, other: Self) -> bool { + !abstract_heaps::Empty.includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> AbstractHeapPrinter<'_> { + AbstractHeapPrinter { inner: self, ptr_map } + } +} + +impl Effect { + pub const fn read_write(read: AbstractHeap, write: AbstractHeap) -> Effect { + Effect { read, write } + } + + /// This function addresses the special case where the read and write heaps are the same + pub const fn promote(heap: AbstractHeap) -> Effect { + Effect {read: heap, write: heap } + } + + /// This function accepts write and heaps read to Any + pub const fn write(write: AbstractHeap) -> Effect { + Effect { read: abstract_heaps::Any, write } + } + + /// This function accepts read and heaps read to Any + pub const fn read(read: AbstractHeap) -> Effect { + Effect { read, write: abstract_heaps::Any } + } + + /// Method to access the private read field + pub const fn read_bits(&self) -> AbstractHeap { + self.read + } + + /// Method to access the private write field + pub const fn write_bits(&self) -> AbstractHeap { + self.write + } + + pub const fn union(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.union(other.read), + self.write.union(other.write) + ) + } + + pub const fn intersect(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.intersect(other.read), + self.write.intersect(other.write) + ) + } + + pub const fn exclude(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.exclude(other.read), + self.write.exclude(other.write) + ) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Effect) -> bool { + self.read.bit_equal(other.read) & self.write.bit_equal(other.write) + } + + pub const fn includes(&self, other: Effect) -> bool { + self.bit_equal(Effect::union(self, other)) + } + + pub const fn overlaps(&self, other: Effect) -> bool { + Effect::promote(abstract_heaps::Empty).includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> EffectPrinter<'_> { + EffectPrinter { inner: self, ptr_map } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[track_caller] + fn assert_heap_bit_equal(left: AbstractHeap, right: AbstractHeap) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(right.includes(left), "{left} is not a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_not_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(!right.includes(left), "{left} is a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Effect, right: Effect) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect(left: Effect, right: Effect) { + assert!(right.includes(left), "{left} is not a subeffect of {right}"); + } + + #[track_caller] + fn assert_not_subeffect(left: Effect, right: Effect) { + assert!(!right.includes(left), "{left} is a subeffect of {right}"); + } + + #[test] + fn effect_heap_none_is_subeffect_of_everything() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Control); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Stack); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Locals); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Allocator); + } + + #[test] + fn effect_heap_everything_is_subeffect_of_any() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Control, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Any); + } + + #[test] + fn effect_heap_union_never_shrinks() { + // iterate over all effect entries from bottom to top + for i in [0, 1, 4, 6, 10, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty.union(e)); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any.union(e)); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Frame.union(e)); + } + } + + #[test] + fn effect_heap_intersect_never_grows() { + // Randomly selected values from bottom to top + for i in [0, 3, 6, 8, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty.intersect(e), abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Any.intersect(e), abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame.intersect(e), abstract_heaps::Frame); + } + } + + #[test] + fn effect_heap_self_is_included() { + assert!(abstract_heaps::Stack.includes(abstract_heaps::Stack)); + assert!(abstract_heaps::Any.includes(abstract_heaps::Any)); + assert!(abstract_heaps::Empty.includes(abstract_heaps::Empty)); + } + + #[test] + fn effect_heap_frame_includes_stack_locals_and_pc() { + assert_subeffect_heap(abstract_heaps::Stack, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Frame); + } + + #[test] + fn effect_heap_frame_is_stack_locals_and_pc() { + let union = abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC)); + assert_heap_bit_equal(abstract_heaps::Frame, union); + } + + #[test] + fn effect_heap_any_includes_some_subeffects() { + assert_subeffect_heap(abstract_heaps::Allocator, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + } + + #[test] + fn effect_heap_display_exact_bits_match() { + assert_eq!(format!("{}", abstract_heaps::Empty), "Empty"); + assert_eq!(format!("{}", abstract_heaps::PC), "PC"); + assert_eq!(format!("{}", abstract_heaps::Any), "Any"); + assert_eq!(format!("{}", abstract_heaps::Frame), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + } + + #[test] + fn effect_heap_display_multiple_bits() { + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals)), "Stack|Locals"); + assert_eq!(format!("{}", abstract_heaps::PC.union(abstract_heaps::Allocator)), "PC|Allocator"); + } + + #[test] + fn effect_any_includes_everything() { + assert_subeffect(effects::Allocator, effects::Any); + assert_subeffect(effects::Frame, effects::Any); + assert_subeffect(effects::Memory, effects::Any); + // Let's do a less standard effect too + assert_subeffect( + Effect::read_write(abstract_heaps::Control, abstract_heaps::Any), + effects::Any + ); + } + + #[test] + fn effect_union_is_idempotent() { + assert_bit_equal( + Effect::read(abstract_heaps::Any) + .union(Effect::write(abstract_heaps::Any)), + effects::Any + ); + assert_bit_equal( + effects::Empty.union(effects::Empty), + effects::Empty + ); + } + + #[test] + fn effect_union_contains_and_excludes() { + assert_subeffect( + effects::Control.union(effects::Frame), + effects::Any + ); + assert_not_subeffect( + effects::Frame.union(effects::Locals), + effects::PC + ); + } + + #[test] + fn effect_intersect_is_empty() { + assert_subeffect(effects::Memory.intersect(effects::Control), effects::Empty); + assert_subeffect( + Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Other) + .intersect(Effect::read_write(abstract_heaps::Stack, abstract_heaps::PC)), + effects::Empty + ) + } + + #[test] + fn effect_intersect_exact_match() { + assert_subeffect(effects::Frame.intersect(effects::PC), effects::PC); + assert_subeffect(effects::Allocator.intersect(effects::Allocator), effects::Allocator); + } + + #[test] + fn effect_display_exact_bits_match() { + assert_eq!(format!("{}", effects::Empty), "Empty, Empty"); + assert_eq!(format!("{}", effects::PC), "PC, PC"); + assert_eq!(format!("{}", effects::Any), "Any, Any"); + assert_eq!(format!("{}", effects::Frame), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", Effect::write(abstract_heaps::Control)), "Any, Control"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Memory)), "Allocator, Memory"); + } + + #[test] + fn effect_display_multiple_bits() { + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals)), "Stack|Locals, Stack|Locals"); + assert_eq!(format!("{}", effects::PC.union(effects::Allocator)), "PC|Allocator, PC|Allocator"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Other, abstract_heaps::PC) + .union(Effect::read_write(abstract_heaps::Memory, abstract_heaps::Stack))), + "Memory, Stack|PC" + ); + } + +} diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb new file mode 100644 index 0000000000..f952a8b715 --- /dev/null +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -0,0 +1,230 @@ +# Generate hir_type.inc.rs. To do this, we build up a DAG that +# represents a slice of the Ruby type hierarchy that we care about optimizing. +# This also includes cvalue values such as C booleans, int32, and so on. + +require 'set' + +# Type represents not just a Ruby class but a named union of other types. +class Type + attr_accessor :name, :subtypes + + def initialize name, subtypes=nil + @name = name + @subtypes = subtypes || [] + end + + def all_subtypes + subtypes.flat_map { |subtype| subtype.all_subtypes } + subtypes + end + + def subtype name + result = Type.new name + @subtypes << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec type, f + type.subtypes.each {|subtype| + f.puts type.name + "->" + subtype.name + ";" + } + type.subtypes.each {|subtype| + to_graphviz_rec subtype, f + } +end + +# Generate graphviz. +def to_graphviz type, f + f.puts "digraph G {" + to_graphviz_rec type, f + f.puts "}" +end + +# ===== Start generating the type DAG ===== + +# Start at Any. All types are subtypes of Any. +any = Type.new "Any" +# Build the Ruby object universe. +value = any.subtype "RubyValue" +undef_ = value.subtype "Undef" +value.subtype "CallableMethodEntry" # rb_callable_method_entry_t* +basic_object = value.subtype "BasicObject" +basic_object_exact = basic_object.subtype "BasicObjectExact" +basic_object_subclass = basic_object.subtype "BasicObjectSubclass" +$object = basic_object.subtype "Object" +object_exact = $object.subtype "ObjectExact" +object_subclass = $object.subtype "ObjectSubclass" +$subclass = [basic_object_subclass.name, object_subclass.name] +$builtin_exact = [basic_object_exact.name, object_exact.name] + +$exact_c_names = { + "ObjectExact" => "rb_cObject", + "BasicObjectExact" => "rb_cBasicObject", +} + +$inexact_c_names = { + "Object" => "rb_cObject", + "BasicObject" => "rb_cBasicObject", +} + +# Define a new type that can be subclassed (most of them). +# If c_name is given, mark the rb_cXYZ object as equivalent to this exact type. +def base_type name, c_name: nil + type = $object.subtype name + exact = type.subtype(name+"Exact") + subclass = type.subtype(name+"Subclass") + if c_name + $exact_c_names[exact.name] = c_name + $inexact_c_names[subclass.name] = c_name + end + $builtin_exact << exact.name + $subclass << subclass.name + [type, exact] +end + +# Define a new type that cannot be subclassed. +# If c_name is given, mark the rb_cXYZ object as equivalent to this type. +def final_type name, base: $object, c_name: nil + if c_name + $exact_c_names[name] = c_name + end + type = base.subtype name + $builtin_exact << type.name + type +end + +base_type "String", c_name: "rb_cString" +base_type "Array", c_name: "rb_cArray" +base_type "Hash", c_name: "rb_cHash" +base_type "Range", c_name: "rb_cRange" +base_type "Set", c_name: "rb_cSet" +base_type "Regexp", c_name: "rb_cRegexp" +module_class, _ = base_type "Module", c_name: "rb_cModule" +class_ = final_type "Class", base: module_class, c_name: "rb_cClass" + +numeric, _ = base_type "Numeric", c_name: "rb_cNumeric" + +integer_exact = final_type "Integer", base: numeric, c_name: "rb_cInteger" +# CRuby partitions Integer into immediate and non-immediate variants. +fixnum = integer_exact.subtype "Fixnum" +integer_exact.subtype "Bignum" + +float_exact = final_type "Float", base: numeric, c_name: "rb_cFloat" +# CRuby partitions Float into immediate and non-immediate variants. +flonum = float_exact.subtype "Flonum" +float_exact.subtype "HeapFloat" + +symbol_exact = final_type "Symbol", c_name: "rb_cSymbol" +# CRuby partitions Symbol into immediate and non-immediate variants. +static_sym = symbol_exact.subtype "StaticSymbol" +symbol_exact.subtype "DynamicSymbol" + +nil_exact = final_type "NilClass", c_name: "rb_cNilClass" +true_exact = final_type "TrueClass", c_name: "rb_cTrueClass" +false_exact = final_type "FalseClass", c_name: "rb_cFalseClass" + +# Build the cvalue object universe. This is for C-level types that may be +# passed around when calling into the Ruby VM or after some strength reduction +# of HIR. +cvalue = any.subtype "CValue" +cvalue.subtype "CBool" +cvalue.subtype "CPtr" +cvalue.subtype "CDouble" +cvalue.subtype "CNull" +cvalue_int = cvalue.subtype "CInt" +signed = cvalue_int.subtype "CSigned" +unsigned = cvalue_int.subtype "CUnsigned" +[8, 16, 32, 64].each {|width| + signed.subtype "CInt#{width}" + unsigned.subtype "CUInt#{width}" +} +unsigned.subtype "CShape" + +# Assign individual bits to type leaves and union bit patterns to nodes with subtypes +num_bits = 0 +$bits = {"Empty" => ["0u64"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subtypes].sort_by(&:name).each {|type| + subtypes = type.subtypes + if subtypes.empty? + # Assign bits for leaves + $bits[type.name] = ["1u64 << #{num_bits}"] + $numeric_bits[type.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[type.name] = subtypes.map(&:name).sort + end +} +[*any.all_subtypes, any].each {|type| + subtypes = type.subtypes + unless subtypes.empty? + $numeric_bits[type.name] = subtypes.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# Unions are for names of groups of type bit patterns that don't fit neatly +# into the Ruby class hierarchy. For example, we might want to refer to a union +# of TrueClassExact|FalseClassExact by the name BoolExact even though a "bool" +# doesn't exist as a class in Ruby. +def add_union name, type_names + type_names = type_names.sort + $bits[name] = type_names + $numeric_bits[name] = type_names.map {|type_name| $numeric_bits[type_name]}.reduce(&:|) +end + +add_union "BuiltinExact", $builtin_exact +add_union "Subclass", $subclass +add_union "BoolExact", [true_exact.name, false_exact.name] +add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] +add_union "Falsy", [nil_exact.name, false_exact.name] +$bits["HeapBasicObject"] = ["BasicObject & !Immediate"] +$numeric_bits["HeapBasicObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] +$bits["HeapObject"] = ["Object & !Immediate"] +$numeric_bits["HeapObject"] = $numeric_bits["Object"] & ~$numeric_bits["Immediate"] +$bits["Truthy"] = ["BasicObject & !Falsy"] +$numeric_bits["Truthy"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Falsy"] +$bits["NotNil"] = ["BasicObject & !NilClass"] +$numeric_bits["NotNil"] = $numeric_bits["BasicObject"] & ~$numeric_bits["NilClass"] + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir_type/gen_hir_type.rb." +puts "mod bits {" +$bits.keys.sort.map {|type_name| + subtypes = $bits[type_name].join(" | ") + puts " pub const #{type_name}: u64 = #{subtypes};" +} +puts " pub const AllBitPatterns: [(&str, u64); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of a Type. For example, CSigned instead of +# CInt8|CInt16|... +$numeric_bits.sort_by {|key, val| -val}.each {|type_name, _| + puts " (\"#{type_name}\", #{type_name})," +} +puts " ];" +puts " pub const NumTypeBits: u64 = #{num_bits}; +}" + +puts "pub mod types { + use super::*;" +$bits.keys.sort.map {|type_name| + puts " pub const #{type_name}: Type = Type::from_bits(bits::#{type_name});" +} +puts " pub const ExactBitsAndClass: [(u64, *const VALUE); #{$exact_c_names.size}] = [" +$exact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$inexact_c_names = $inexact_c_names.to_a.sort_by {|name, _| $bits[name]}.to_h +puts " pub const InexactBitsAndClass: [(u64, *const VALUE); #{$inexact_c_names.size}] = [" +$inexact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +puts "}" + +File.open("zjit_types.dot", "w") do |f| + to_graphviz(any, f) +end diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs new file mode 100644 index 0000000000..886b4b54dd --- /dev/null +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -0,0 +1,262 @@ +// This file is @generated by src/hir_type/gen_hir_type.rb. +mod bits { + pub const Any: u64 = CValue | RubyValue; + pub const Array: u64 = ArrayExact | ArraySubclass; + pub const ArrayExact: u64 = 1u64 << 0; + pub const ArraySubclass: u64 = 1u64 << 1; + pub const BasicObject: u64 = BasicObjectExact | BasicObjectSubclass | Object; + pub const BasicObjectExact: u64 = 1u64 << 2; + pub const BasicObjectSubclass: u64 = 1u64 << 3; + pub const Bignum: u64 = 1u64 << 4; + pub const BoolExact: u64 = FalseClass | TrueClass; + pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | Class | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | NumericExact | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; + pub const CBool: u64 = 1u64 << 5; + pub const CDouble: u64 = 1u64 << 6; + pub const CInt: u64 = CSigned | CUnsigned; + pub const CInt16: u64 = 1u64 << 7; + pub const CInt32: u64 = 1u64 << 8; + pub const CInt64: u64 = 1u64 << 9; + pub const CInt8: u64 = 1u64 << 10; + pub const CNull: u64 = 1u64 << 11; + pub const CPtr: u64 = 1u64 << 12; + pub const CShape: u64 = 1u64 << 13; + pub const CSigned: u64 = CInt16 | CInt32 | CInt64 | CInt8; + pub const CUInt16: u64 = 1u64 << 14; + pub const CUInt32: u64 = 1u64 << 15; + pub const CUInt64: u64 = 1u64 << 16; + pub const CUInt8: u64 = 1u64 << 17; + pub const CUnsigned: u64 = CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; + pub const CValue: u64 = CBool | CDouble | CInt | CNull | CPtr; + pub const CallableMethodEntry: u64 = 1u64 << 18; + pub const Class: u64 = 1u64 << 19; + pub const DynamicSymbol: u64 = 1u64 << 20; + pub const Empty: u64 = 0u64; + pub const FalseClass: u64 = 1u64 << 21; + pub const Falsy: u64 = FalseClass | NilClass; + pub const Fixnum: u64 = 1u64 << 22; + pub const Float: u64 = Flonum | HeapFloat; + pub const Flonum: u64 = 1u64 << 23; + pub const Hash: u64 = HashExact | HashSubclass; + pub const HashExact: u64 = 1u64 << 24; + pub const HashSubclass: u64 = 1u64 << 25; + pub const HeapBasicObject: u64 = BasicObject & !Immediate; + pub const HeapFloat: u64 = 1u64 << 26; + pub const HeapObject: u64 = Object & !Immediate; + pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; + pub const Integer: u64 = Bignum | Fixnum; + pub const Module: u64 = Class | ModuleExact | ModuleSubclass; + pub const ModuleExact: u64 = 1u64 << 27; + pub const ModuleSubclass: u64 = 1u64 << 28; + pub const NilClass: u64 = 1u64 << 29; + pub const NotNil: u64 = BasicObject & !NilClass; + pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; + pub const NumericExact: u64 = 1u64 << 30; + pub const NumericSubclass: u64 = 1u64 << 31; + pub const Object: u64 = Array | FalseClass | Hash | Module | NilClass | Numeric | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; + pub const ObjectExact: u64 = 1u64 << 32; + pub const ObjectSubclass: u64 = 1u64 << 33; + pub const Range: u64 = RangeExact | RangeSubclass; + pub const RangeExact: u64 = 1u64 << 34; + pub const RangeSubclass: u64 = 1u64 << 35; + pub const Regexp: u64 = RegexpExact | RegexpSubclass; + pub const RegexpExact: u64 = 1u64 << 36; + pub const RegexpSubclass: u64 = 1u64 << 37; + pub const RubyValue: u64 = BasicObject | CallableMethodEntry | Undef; + pub const Set: u64 = SetExact | SetSubclass; + pub const SetExact: u64 = 1u64 << 38; + pub const SetSubclass: u64 = 1u64 << 39; + pub const StaticSymbol: u64 = 1u64 << 40; + pub const String: u64 = StringExact | StringSubclass; + pub const StringExact: u64 = 1u64 << 41; + pub const StringSubclass: u64 = 1u64 << 42; + pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; + pub const Symbol: u64 = DynamicSymbol | StaticSymbol; + pub const TrueClass: u64 = 1u64 << 43; + pub const Truthy: u64 = BasicObject & !Falsy; + pub const Undef: u64 = 1u64 << 44; + pub const AllBitPatterns: [(&str, u64); 74] = [ + ("Any", Any), + ("RubyValue", RubyValue), + ("Immediate", Immediate), + ("Undef", Undef), + ("BasicObject", BasicObject), + ("Object", Object), + ("NotNil", NotNil), + ("Truthy", Truthy), + ("BuiltinExact", BuiltinExact), + ("BoolExact", BoolExact), + ("TrueClass", TrueClass), + ("HeapBasicObject", HeapBasicObject), + ("HeapObject", HeapObject), + ("String", String), + ("Subclass", Subclass), + ("StringSubclass", StringSubclass), + ("StringExact", StringExact), + ("Symbol", Symbol), + ("StaticSymbol", StaticSymbol), + ("Set", Set), + ("SetSubclass", SetSubclass), + ("SetExact", SetExact), + ("Regexp", Regexp), + ("RegexpSubclass", RegexpSubclass), + ("RegexpExact", RegexpExact), + ("Range", Range), + ("RangeSubclass", RangeSubclass), + ("RangeExact", RangeExact), + ("ObjectSubclass", ObjectSubclass), + ("ObjectExact", ObjectExact), + ("Numeric", Numeric), + ("NumericSubclass", NumericSubclass), + ("NumericExact", NumericExact), + ("Falsy", Falsy), + ("NilClass", NilClass), + ("Module", Module), + ("ModuleSubclass", ModuleSubclass), + ("ModuleExact", ModuleExact), + ("Float", Float), + ("HeapFloat", HeapFloat), + ("Hash", Hash), + ("HashSubclass", HashSubclass), + ("HashExact", HashExact), + ("Flonum", Flonum), + ("Integer", Integer), + ("Fixnum", Fixnum), + ("FalseClass", FalseClass), + ("DynamicSymbol", DynamicSymbol), + ("Class", Class), + ("CallableMethodEntry", CallableMethodEntry), + ("CValue", CValue), + ("CInt", CInt), + ("CUnsigned", CUnsigned), + ("CUInt8", CUInt8), + ("CUInt64", CUInt64), + ("CUInt32", CUInt32), + ("CUInt16", CUInt16), + ("CShape", CShape), + ("CPtr", CPtr), + ("CNull", CNull), + ("CSigned", CSigned), + ("CInt8", CInt8), + ("CInt64", CInt64), + ("CInt32", CInt32), + ("CInt16", CInt16), + ("CDouble", CDouble), + ("CBool", CBool), + ("Bignum", Bignum), + ("BasicObjectSubclass", BasicObjectSubclass), + ("BasicObjectExact", BasicObjectExact), + ("Array", Array), + ("ArraySubclass", ArraySubclass), + ("ArrayExact", ArrayExact), + ("Empty", Empty), + ]; + pub const NumTypeBits: u64 = 45; +} +pub mod types { + use super::*; + pub const Any: Type = Type::from_bits(bits::Any); + pub const Array: Type = Type::from_bits(bits::Array); + pub const ArrayExact: Type = Type::from_bits(bits::ArrayExact); + pub const ArraySubclass: Type = Type::from_bits(bits::ArraySubclass); + pub const BasicObject: Type = Type::from_bits(bits::BasicObject); + pub const BasicObjectExact: Type = Type::from_bits(bits::BasicObjectExact); + pub const BasicObjectSubclass: Type = Type::from_bits(bits::BasicObjectSubclass); + pub const Bignum: Type = Type::from_bits(bits::Bignum); + pub const BoolExact: Type = Type::from_bits(bits::BoolExact); + pub const BuiltinExact: Type = Type::from_bits(bits::BuiltinExact); + pub const CBool: Type = Type::from_bits(bits::CBool); + pub const CDouble: Type = Type::from_bits(bits::CDouble); + pub const CInt: Type = Type::from_bits(bits::CInt); + pub const CInt16: Type = Type::from_bits(bits::CInt16); + pub const CInt32: Type = Type::from_bits(bits::CInt32); + pub const CInt64: Type = Type::from_bits(bits::CInt64); + pub const CInt8: Type = Type::from_bits(bits::CInt8); + pub const CNull: Type = Type::from_bits(bits::CNull); + pub const CPtr: Type = Type::from_bits(bits::CPtr); + pub const CShape: Type = Type::from_bits(bits::CShape); + pub const CSigned: Type = Type::from_bits(bits::CSigned); + pub const CUInt16: Type = Type::from_bits(bits::CUInt16); + pub const CUInt32: Type = Type::from_bits(bits::CUInt32); + pub const CUInt64: Type = Type::from_bits(bits::CUInt64); + pub const CUInt8: Type = Type::from_bits(bits::CUInt8); + pub const CUnsigned: Type = Type::from_bits(bits::CUnsigned); + pub const CValue: Type = Type::from_bits(bits::CValue); + pub const CallableMethodEntry: Type = Type::from_bits(bits::CallableMethodEntry); + pub const Class: Type = Type::from_bits(bits::Class); + pub const DynamicSymbol: Type = Type::from_bits(bits::DynamicSymbol); + pub const Empty: Type = Type::from_bits(bits::Empty); + pub const FalseClass: Type = Type::from_bits(bits::FalseClass); + pub const Falsy: Type = Type::from_bits(bits::Falsy); + pub const Fixnum: Type = Type::from_bits(bits::Fixnum); + pub const Float: Type = Type::from_bits(bits::Float); + pub const Flonum: Type = Type::from_bits(bits::Flonum); + pub const Hash: Type = Type::from_bits(bits::Hash); + pub const HashExact: Type = Type::from_bits(bits::HashExact); + pub const HashSubclass: Type = Type::from_bits(bits::HashSubclass); + pub const HeapBasicObject: Type = Type::from_bits(bits::HeapBasicObject); + pub const HeapFloat: Type = Type::from_bits(bits::HeapFloat); + pub const HeapObject: Type = Type::from_bits(bits::HeapObject); + pub const Immediate: Type = Type::from_bits(bits::Immediate); + pub const Integer: Type = Type::from_bits(bits::Integer); + pub const Module: Type = Type::from_bits(bits::Module); + pub const ModuleExact: Type = Type::from_bits(bits::ModuleExact); + pub const ModuleSubclass: Type = Type::from_bits(bits::ModuleSubclass); + pub const NilClass: Type = Type::from_bits(bits::NilClass); + pub const NotNil: Type = Type::from_bits(bits::NotNil); + pub const Numeric: Type = Type::from_bits(bits::Numeric); + pub const NumericExact: Type = Type::from_bits(bits::NumericExact); + pub const NumericSubclass: Type = Type::from_bits(bits::NumericSubclass); + pub const Object: Type = Type::from_bits(bits::Object); + pub const ObjectExact: Type = Type::from_bits(bits::ObjectExact); + pub const ObjectSubclass: Type = Type::from_bits(bits::ObjectSubclass); + pub const Range: Type = Type::from_bits(bits::Range); + pub const RangeExact: Type = Type::from_bits(bits::RangeExact); + pub const RangeSubclass: Type = Type::from_bits(bits::RangeSubclass); + pub const Regexp: Type = Type::from_bits(bits::Regexp); + pub const RegexpExact: Type = Type::from_bits(bits::RegexpExact); + pub const RegexpSubclass: Type = Type::from_bits(bits::RegexpSubclass); + pub const RubyValue: Type = Type::from_bits(bits::RubyValue); + pub const Set: Type = Type::from_bits(bits::Set); + pub const SetExact: Type = Type::from_bits(bits::SetExact); + pub const SetSubclass: Type = Type::from_bits(bits::SetSubclass); + pub const StaticSymbol: Type = Type::from_bits(bits::StaticSymbol); + pub const String: Type = Type::from_bits(bits::String); + pub const StringExact: Type = Type::from_bits(bits::StringExact); + pub const StringSubclass: Type = Type::from_bits(bits::StringSubclass); + pub const Subclass: Type = Type::from_bits(bits::Subclass); + pub const Symbol: Type = Type::from_bits(bits::Symbol); + pub const TrueClass: Type = Type::from_bits(bits::TrueClass); + pub const Truthy: Type = Type::from_bits(bits::Truthy); + pub const Undef: Type = Type::from_bits(bits::Undef); + pub const ExactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ObjectExact, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectExact, &raw const crate::cruby::rb_cBasicObject), + (bits::StringExact, &raw const crate::cruby::rb_cString), + (bits::ArrayExact, &raw const crate::cruby::rb_cArray), + (bits::HashExact, &raw const crate::cruby::rb_cHash), + (bits::RangeExact, &raw const crate::cruby::rb_cRange), + (bits::SetExact, &raw const crate::cruby::rb_cSet), + (bits::RegexpExact, &raw const crate::cruby::rb_cRegexp), + (bits::ModuleExact, &raw const crate::cruby::rb_cModule), + (bits::Class, &raw const crate::cruby::rb_cClass), + (bits::NumericExact, &raw const crate::cruby::rb_cNumeric), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + ]; + pub const InexactBitsAndClass: [(u64, *const VALUE); 10] = [ + (bits::ArraySubclass, &raw const crate::cruby::rb_cArray), + (bits::HashSubclass, &raw const crate::cruby::rb_cHash), + (bits::ModuleSubclass, &raw const crate::cruby::rb_cModule), + (bits::NumericSubclass, &raw const crate::cruby::rb_cNumeric), + (bits::RangeSubclass, &raw const crate::cruby::rb_cRange), + (bits::RegexpSubclass, &raw const crate::cruby::rb_cRegexp), + (bits::SetSubclass, &raw const crate::cruby::rb_cSet), + (bits::StringSubclass, &raw const crate::cruby::rb_cString), + (bits::Object, &raw const crate::cruby::rb_cObject), + (bits::BasicObject, &raw const crate::cruby::rb_cBasicObject), + ]; +} diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs new file mode 100644 index 0000000000..1f7526915c --- /dev/null +++ b/zjit/src/hir_type/mod.rs @@ -0,0 +1,1123 @@ +//! High-level intermediate representation types. + +#![allow(non_upper_case_globals)] +use crate::cruby::{rb_block_param_proxy, Qfalse, Qnil, Qtrue, RUBY_T_ARRAY, RUBY_T_CLASS, RUBY_T_HASH, RUBY_T_MODULE, RUBY_T_STRING, VALUE}; +use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cRange, rb_cModule, rb_zjit_singleton_class_p}; +use crate::cruby::ClassRelationship; +use crate::cruby::get_class_name; +use crate::cruby::ruby_sym_to_rust_string; +use crate::cruby::rb_mRubyVMFrozenCore; +use crate::cruby::rb_obj_class; +use crate::hir::{Const, PtrPrintMap}; +use crate::profile::ProfiledType; + +#[derive(Copy, Clone, Debug, PartialEq)] +/// Specialization of the type. If we know additional information about the object, we put it here. +/// This includes information about its value as a cvalue. For Ruby objects, type specialization +/// is split into three sub-cases: +/// +/// * Object, where we know exactly what object (pointer) the Type corresponds to +/// * Type exact, where we know exactly what class the Type represents (which could be because we +/// have an instance of it; includes Object specialization) +/// * Type, where we know that the Type could represent the given class or any of its subclasses +/// +/// It is also a lattice but a much shallower one. It is not meant to be used directly, just by +/// Type internals. +pub enum Specialization { + /// We know nothing about the specialization of this Type. + Any, + /// We know that this Type is an instance of the given Ruby class in the VALUE or any of its subclasses. + Type(VALUE), + /// We know that this Type is an instance of exactly the Ruby class in the VALUE. + TypeExact(VALUE), + /// We know that this Type is exactly the Ruby object in the VALUE. + Object(VALUE), + /// We know that this Type is exactly the given cvalue/C integer value (use the type bits to + /// inform how we should interpret the u64, e.g. as CBool or CInt32). + Int(u64), + /// We know that this Type is exactly the given cvalue/C double. + Double(f64), + /// We know that the Type is [`types::Empty`] and therefore the instruction that produces this + /// value never returns. + Empty, +} + +// NOTE: Type very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +#[derive(Copy, Clone, Debug)] +/// The main work horse of intraprocedural type inference and specialization. The main interfaces +/// will look like: +/// +/// * is type A a subset of type B +/// * union/meet type A and type B +/// +/// Most questions can be rewritten in terms of these operations. +pub struct Type { + /// A bitset representing type information about the object. Specific bits are assigned for + /// leaf types (for example, static symbols) and union-ing bitsets together represents + /// union-ing sets of types. These sets form a lattice (with Any as "could be anything" and + /// Empty as "can be nothing"). + /// + /// Capable of also representing cvalue types (bool, i32, etc). + /// + /// This field should not be directly read or written except by internal `Type` APIs. + bits: u64, + /// Specialization of the type. See [`Specialization`]. + /// + /// This field should not be directly read or written except by internal `Type` APIs. + spec: Specialization +} + +include!("hir_type.inc.rs"); + +fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::Result { + let ty = printer.inner; + match ty.spec { + Specialization::Any | Specialization::Empty => { Ok(()) }, + Specialization::Object(val) if val == unsafe { rb_mRubyVMFrozenCore } => write!(f, "[VMFrozenCore]"), + Specialization::Object(val) if val == unsafe { rb_block_param_proxy } => write!(f, "[BlockParamProxy]"), + Specialization::Object(val) if ty.is_subtype(types::Symbol) => write!(f, "[:{}]", ruby_sym_to_rust_string(val)), + Specialization::Object(val) if ty.is_subtype(types::Class) => + write!(f, "[{}@{:p}]", get_class_name(val), printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)), + Specialization::Object(val) => write!(f, "[{}]", val.print(printer.ptr_map)), + // TODO(max): Ensure singleton classes never have Type specialization + Specialization::Type(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), + Specialization::Type(val) => write!(f, "[class:{}]", get_class_name(val)), + Specialization::TypeExact(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class_exact*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), + Specialization::TypeExact(val) => + write!(f, "[class_exact:{}]", get_class_name(val)), + Specialization::Int(val) if ty.is_subtype(types::CBool) => write!(f, "[{}]", val != 0), + Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val & u8::MAX as u64) as i8), + Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val & u16::MAX as u64) as i16), + Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val & u32::MAX as u64) as i32), + Specialization::Int(val) if ty.is_subtype(types::CShape) => + write!(f, "[{:p}]", printer.ptr_map.map_shape(crate::cruby::ShapeId((val & u32::MAX as u64) as u32))), + Specialization::Int(val) if ty.is_subtype(types::CInt64) => write!(f, "[{}]", val as i64), + Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val & u8::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val & u16::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val & u32::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt64) => write!(f, "[{val}]"), + Specialization::Int(val) if ty.is_subtype(types::CPtr) => write!(f, "[{}]", Const::CPtr(val as *const u8).print(printer.ptr_map)), + Specialization::Int(val) => write!(f, "[{val}]"), + Specialization::Double(val) => write!(f, "[{val}]"), + } +} + +/// Print adaptor for [`Type`]. See [`PtrPrintMap`]. +pub struct TypePrinter<'a> { + inner: Type, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for TypePrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let ty = self.inner; + for (name, pattern) in bits::AllBitPatterns { + if ty.bits == pattern { + write!(f, "{name}")?; + return write_spec(f, self); + } + } + assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + let mut bits = ty.bits; + let mut sep = ""; + for (name, pattern) in bits::AllBitPatterns { + if bits == 0 { break; } + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + } + assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + write_spec(f, self) + } +} + +impl std::fmt::Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +fn is_array_exact(val: VALUE) -> bool { + // Prism hides array values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cArray } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_ARRAY) +} + +fn is_string_exact(val: VALUE) -> bool { + // Prism hides string values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cString } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_STRING) +} + +fn is_hash_exact(val: VALUE) -> bool { + // Prism hides hash values in the constant pool from the GC, so class_of will return 0 + val.class_of() == unsafe { rb_cHash } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_HASH) +} + +fn is_range_exact(val: VALUE) -> bool { + val.class_of() == unsafe { rb_cRange } +} + +fn is_module_exact(val: VALUE) -> bool { + if val.builtin_type() != RUBY_T_MODULE { + return false; + } + + // For Class and Module instances, `class_of` will return the singleton class of the object. + // Using `rb_obj_class` will give us the actual class of the module so we can check if the + // object is an instance of Module, or an instance of Module subclass. + let klass = unsafe { rb_obj_class(val) }; + klass == unsafe { rb_cModule } +} + +impl Type { + /// Create a `Type` from the given integer. + pub const fn fixnum(val: i64) -> Type { + Type { + bits: bits::Fixnum, + spec: Specialization::Object(VALUE::fixnum_from_usize(val as usize)), + } + } + + fn bits_from_exact_class(class: VALUE) -> Option<u64> { + types::ExactBitsAndClass + .iter() + .find(|&(_, class_object)| unsafe { **class_object } == class) + .map(|&(bits, _)| bits) + } + + fn bits_from_subclass(class: VALUE) -> Option<u64> { + types::InexactBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + // Can't be an immediate if it's a subclass. + .map(|&(bits, _)| bits & !bits::Immediate) + } + + fn from_heap_object(val: VALUE) -> Type { + assert!(!val.special_const_p(), "val should be a heap object"); + let bits = + // GC-hidden types + if is_array_exact(val) { bits::ArrayExact } + else if is_hash_exact(val) { bits::HashExact } + else if is_string_exact(val) { bits::StringExact } + // Singleton classes + else if is_module_exact(val) { bits::ModuleExact } + else if val.builtin_type() == RUBY_T_CLASS { bits::Class } + // Classes that have an immediate/heap split + else if val.class_of() == unsafe { rb_cInteger } { bits::Bignum } + else if val.class_of() == unsafe { rb_cFloat } { bits::HeapFloat } + else if val.class_of() == unsafe { rb_cSymbol } { bits::DynamicSymbol } + else if let Some(bits) = Self::bits_from_exact_class(val.class_of()) { bits } + else if let Some(bits) = Self::bits_from_subclass(val.class_of()) { bits } + else { + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(val.class_of())) + }; + let spec = Specialization::Object(val); + Type { bits, spec } + } + + /// Create a `Type` from a Ruby `VALUE`. The type is not guaranteed to have object + /// specialization in its `specialization` field (for example, `Qnil` will just be + /// `types::NilClass`), but will be available via `ruby_object()`. + pub fn from_value(val: VALUE) -> Type { + // Immediates + if val.fixnum_p() { + Type { bits: bits::Fixnum, spec: Specialization::Object(val) } + } + else if val.flonum_p() { + Type { bits: bits::Flonum, spec: Specialization::Object(val) } + } + else if val.static_sym_p() { + Type { bits: bits::StaticSymbol, spec: Specialization::Object(val) } + } + // Singleton objects; don't specialize + else if val == Qnil { types::NilClass } + else if val == Qtrue { types::TrueClass } + else if val == Qfalse { types::FalseClass } + else if val.cme_p() { + // NB: Checking for CME has to happen before looking at class_of because that's not + // valid on imemo. + Type { bits: bits::CallableMethodEntry, spec: Specialization::Object(val) } + } + else { + Self::from_heap_object(val) + } + } + + pub fn from_const(val: Const) -> Type { + match val { + Const::Value(v) => Self::from_value(v), + Const::CBool(v) => Self::from_cbool(v), + Const::CInt8(v) => Self::from_cint(types::CInt8, v as i64), + Const::CInt16(v) => Self::from_cint(types::CInt16, v as i64), + Const::CInt32(v) => Self::from_cint(types::CInt32, v as i64), + Const::CInt64(v) => Self::from_cint(types::CInt64, v), + Const::CUInt8(v) => Self::from_cint(types::CUInt8, v as i64), + Const::CUInt16(v) => Self::from_cint(types::CUInt16, v as i64), + Const::CUInt32(v) => Self::from_cint(types::CUInt32, v as i64), + Const::CShape(v) => Self::from_cint(types::CShape, v.0 as i64), + Const::CUInt64(v) => Self::from_cint(types::CUInt64, v as i64), + Const::CPtr(v) => Self::from_cptr(v), + Const::CDouble(v) => Self::from_double(v), + } + } + + pub fn from_profiled_type(val: ProfiledType) -> Type { + if val.is_fixnum() { types::Fixnum } + else if val.is_flonum() { types::Flonum } + else if val.is_static_symbol() { types::StaticSymbol } + else if val.is_nil() { types::NilClass } + else if val.is_true() { types::TrueClass } + else if val.is_false() { types::FalseClass } + else { Self::from_class(val.class()) } + } + + pub fn from_class(class: VALUE) -> Type { + if let Some(bits) = Self::bits_from_exact_class(class) { + return Type::from_bits(bits); + } + if let Some(bits) = Self::bits_from_subclass(class) { + return Type { bits, spec: Specialization::TypeExact(class) } + } + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(class)) + } + + /// Private. Only for creating type globals. + const fn from_bits(bits: u64) -> Type { + Type { + bits, + spec: if bits == bits::Empty { + Specialization::Empty + } else { + Specialization::Any + }, + } + } + + /// Create a `Type` from a cvalue integer. Use the `ty` given to specify what size the + /// `specialization` represents. For example, `Type::from_cint(types::CBool, 1)` or + /// `Type::from_cint(types::CUInt16, 12)`. + pub fn from_cint(ty: Type, val: i64) -> Type { + assert_eq!(ty.spec, Specialization::Any); + assert!((ty.is_subtype(types::CUnsigned) || ty.is_subtype(types::CSigned)) && + ty.bits != types::CUnsigned.bits && ty.bits != types::CSigned.bits, + "ty must be a specific int size"); + Type { bits: ty.bits, spec: Specialization::Int(val as u64) } + } + + pub fn from_cptr(val: *const u8) -> Type { + Type { bits: bits::CPtr, spec: Specialization::Int(val as u64) } + } + + /// Create a `Type` (a `CDouble` with double specialization) from a f64. + pub fn from_double(val: f64) -> Type { + Type { bits: bits::CDouble, spec: Specialization::Double(val) } + } + + /// Create a `Type` from a cvalue boolean. + pub fn from_cbool(val: bool) -> Type { + Type { bits: bits::CBool, spec: Specialization::Int(val as u64) } + } + + /// Return true if the value with this type is definitely truthy. + pub fn is_known_truthy(&self) -> bool { + !self.could_be(types::NilClass) && !self.could_be(types::FalseClass) + } + + /// Return true if the value with this type is definitely falsy. + pub fn is_known_falsy(&self) -> bool { + self.is_subtype(types::NilClass) || self.is_subtype(types::FalseClass) + } + + pub fn has_value(&self, val: Const) -> bool { + match (self.spec, val) { + (Specialization::Object(v1), Const::Value(v2)) => v1 == v2, + (Specialization::Int(v1), Const::CBool(v2)) if self.is_subtype(types::CBool) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt8(v2)) if self.is_subtype(types::CInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt16(v2)) if self.is_subtype(types::CInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt32(v2)) if self.is_subtype(types::CInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt64(v2)) if self.is_subtype(types::CInt64) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt8(v2)) if self.is_subtype(types::CUInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt16(v2)) if self.is_subtype(types::CUInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt32(v2)) if self.is_subtype(types::CUInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CShape(v2)) if self.is_subtype(types::CShape) => v1 == (v2.0 as u64), + (Specialization::Int(v1), Const::CUInt64(v2)) if self.is_subtype(types::CUInt64) => v1 == v2, + (Specialization::Int(v1), Const::CPtr(v2)) if self.is_subtype(types::CPtr) => v1 == (v2 as u64), + (Specialization::Double(v1), Const::CDouble(v2)) => v1.to_bits() == v2.to_bits(), + _ => false, + } + } + + /// Return the object specialization, if any. + pub fn ruby_object(&self) -> Option<VALUE> { + match self.spec { + Specialization::Object(val) => Some(val), + _ => None, + } + } + + /// Return a Ruby object that needs to be marked on GC. + /// This covers Type and TypeExact unlike ruby_object(). + pub fn gc_object(&self) -> Option<VALUE> { + match self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + + /// Mutable version of gc_object(). + pub fn gc_object_mut(&mut self) -> Option<&mut VALUE> { + match &mut self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + + pub fn unspecialized(&self) -> Self { + Type { spec: Specialization::Any, ..*self } + } + + pub fn fixnum_value(&self) -> Option<i64> { + if self.is_subtype(types::Fixnum) { + self.ruby_object().map(|val| val.as_fixnum()) + } else { + None + } + } + + pub fn cint64_value(&self) -> Option<i64> { + match (self.is_subtype(types::CInt64), &self.spec) { + (true, Specialization::Int(val)) => Some(*val as i64), + _ => None, + } + } + + /// Return true if the Type has object specialization and false otherwise. + pub fn ruby_object_known(&self) -> bool { + matches!(self.spec, Specialization::Object(_)) + } + + fn is_builtin(class: VALUE) -> bool { + types::ExactBitsAndClass + .iter() + .any(|&(_, class_object)| unsafe { *class_object } == class) + } + + /// Union both types together, preserving specialization if possible. + pub fn union(&self, other: Type) -> Type { + // Easy cases first + if self.is_subtype(other) { return other; } + if other.is_subtype(*self) { return *self; } + let bits = self.bits | other.bits; + let result = Type::from_bits(bits); + // If one type isn't type specialized, we can't return a specialized Type + if !self.type_known() || !other.type_known() { return result; } + let self_class = self.inexact_ruby_class().unwrap(); + let other_class = other.inexact_ruby_class().unwrap(); + // Pick one of self/other as the least upper bound. This is not the most specific (there + // could be intermediate classes in the inheritance hierarchy) but it is fast to compute. + let super_class = match self_class.is_subclass_of(other_class) { + ClassRelationship::Subclass => other_class, + ClassRelationship::Superclass => self_class, + ClassRelationship::NoRelation => return result, + }; + // Don't specialize built-in types; we can represent them perfectly with type bits. + if Type::is_builtin(super_class) { return result; } + // Supertype specialization can be exact only if the exact type specializations are identical + if let Some(self_class) = self.exact_ruby_class() { + if let Some(other_class) = other.exact_ruby_class() { + if self_class == other_class { + return Type { bits, spec: Specialization::TypeExact(self_class) }; + } + } + } + Type { bits, spec: Specialization::Type(super_class) } + } + + /// Intersect both types, preserving specialization if possible. + pub fn intersection(&self, other: Type) -> Type { + let bits = self.bits & other.bits; + if bits == bits::Empty { return types::Empty; } + if self.spec_is_subtype_of(other) { return Type { bits, spec: self.spec }; } + if other.spec_is_subtype_of(*self) { return Type { bits, spec: other.spec }; } + types::Empty + } + + /// Subtract `other` from `self`, preserving specialization if possible. + pub fn subtract(&self, other: Type) -> Type { + // If self is a subtype of other, the result is empty (no negative types). + if self.is_subtype(other) { return types::Empty; } + // Self is not a subtype of other. That means either: + // * Their type bits do not overlap at all (eg Int vs String) + // * Their type bits overlap but self's specialization is not a subtype of other's (eg + // Fixnum[5] vs Fixnum[4]) + // Check for the latter case, returning self unchanged if so. + if !self.spec_is_subtype_of(other) { + return *self; + } + // Now self is either a supertype of other (eg Object vs String or Fixnum vs Fixnum[5]) or + // their type bits do not overlap at all (eg Int vs String). + // Just subtract the bits and keep self's specialization. + let bits = self.bits & !other.bits; + Type { bits, spec: self.spec } + } + + pub fn could_be(&self, other: Type) -> bool { + !self.intersection(other).bit_equal(types::Empty) + } + + /// Check if the type field of `self` is a subtype of the type field of `other` and also check + /// if the specialization of `self` is a subtype of the specialization of `other`. + pub fn is_subtype(&self, other: Type) -> bool { + (self.bits & other.bits) == self.bits && self.spec_is_subtype_of(other) + } + + /// Return the type specialization, if any. Type specialization asks if we know the Ruby type + /// (including potentially its subclasses) corresponding to a `Type`, including knowing exactly + /// what object is is. + pub fn type_known(&self) -> bool { + matches!(self.spec, Specialization::TypeExact(_) | Specialization::Type(_) | Specialization::Object(_)) + } + + /// Return the exact type specialization, if any. Type specialization asks if we know the + /// *exact* Ruby type corresponding to a `Type`, including knowing exactly what object is is. + pub fn exact_class_known(&self) -> bool { + matches!(self.spec, Specialization::TypeExact(_) | Specialization::Object(_)) + } + + /// Return the exact type specialization, if any. Type specialization asks if we know the exact + /// Ruby type corresponding to a `Type` (no subclasses), including knowing exactly what object + /// it is. + pub fn exact_ruby_class(&self) -> Option<VALUE> { + match self.spec { + // If we're looking at a precise object, we can pull out its class. + Specialization::Object(val) => Some(val.class_of()), + Specialization::TypeExact(val) => Some(val), + _ => None, + } + } + + /// Return the type specialization, if any. Type specialization asks if we know the inexact + /// Ruby type corresponding to a `Type`, including knowing exactly what object is is. + pub fn inexact_ruby_class(&self) -> Option<VALUE> { + match self.spec { + // If we're looking at a precise object, we can pull out its class. + Specialization::Object(val) => Some(val.class_of()), + Specialization::TypeExact(val) | Specialization::Type(val) => Some(val), + _ => None, + } + } + + /// Return a pointer to the Ruby class that an object of this Type would have at run-time, if + /// known. This includes classes for HIR types such as ArrayExact or NilClass, which have + /// canonical Type representations that lack an explicit specialization in their `spec` fields. + pub fn runtime_exact_ruby_class(&self) -> Option<VALUE> { + if let Some(val) = self.exact_ruby_class() { + return Some(val); + } + types::ExactBitsAndClass + .iter() + .find(|&(bits, _)| self.is_subtype(Type::from_bits(*bits))) + .map(|&(_, class_object)| unsafe { *class_object }) + } + + /// Check bit equality of two `Type`s. Do not use! You are probably looking for [`Type::is_subtype`]. + pub fn bit_equal(&self, other: Type) -> bool { + self.bits == other.bits && self.spec == other.spec + } + + /// Check *only* if `self`'s specialization is a subtype of `other`'s specialization. Private. + /// You probably want [`Type::is_subtype`] instead. + fn spec_is_subtype_of(&self, other: Type) -> bool { + match (self.spec, other.spec) { + // Empty is a subtype of everything; Any is a supertype of everything + (Specialization::Empty, _) | (_, Specialization::Any) => true, + // Other is not Any from the previous case, so Any is definitely not a subtype + (Specialization::Any, _) | (_, Specialization::Empty) => false, + // Int and double specialization requires exact equality + (Specialization::Int(_), _) | (_, Specialization::Int(_)) | + (Specialization::Double(_), _) | (_, Specialization::Double(_)) => + self.bits == other.bits && self.spec == other.spec, + // Check other's specialization type in decreasing order of specificity + (_, Specialization::Object(_)) => + self.ruby_object_known() && self.ruby_object() == other.ruby_object(), + (_, Specialization::TypeExact(_)) => + self.exact_class_known() && self.inexact_ruby_class() == other.inexact_ruby_class(), + (_, Specialization::Type(other_class)) => + self.inexact_ruby_class().unwrap().is_subclass_of(other_class) == ClassRelationship::Subclass, + } + } + + pub fn is_immediate(&self) -> bool { + self.is_subtype(types::Immediate) + } + + pub fn print(self, ptr_map: &PtrPrintMap) -> TypePrinter<'_> { + TypePrinter { inner: self, ptr_map } + } + + pub fn num_bits(&self) -> u8 { + self.num_bytes() * crate::cruby::BITS_PER_BYTE as u8 + } + + pub fn num_bytes(&self) -> u8 { + if self.is_subtype(types::CUInt8) || self.is_subtype(types::CInt8) { return 1; } + if self.is_subtype(types::CUInt16) || self.is_subtype(types::CInt16) { return 2; } + if self.is_subtype(types::CUInt32) || self.is_subtype(types::CInt32) { return 4; } + if self.is_subtype(types::CShape) { + use crate::cruby::{SHAPE_ID_NUM_BITS, BITS_PER_BYTE}; + return (SHAPE_ID_NUM_BITS as usize / BITS_PER_BYTE).try_into().unwrap(); + } + // CUInt64, CInt64, CPtr, CNull, CDouble, or anything else defaults to 8 bytes + crate::cruby::SIZEOF_VALUE as u8 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cruby::rust_str_to_ruby; + use crate::cruby::rust_str_to_sym; + use crate::cruby::rb_ary_new_capa; + use crate::cruby::rb_hash_new; + use crate::cruby::rb_float_new; + use crate::cruby::define_class; + use crate::cruby::rb_cObject; + use crate::cruby::rb_cSet; + use crate::cruby::rb_cTrueClass; + use crate::cruby::rb_cFalseClass; + use crate::cruby::rb_cNilClass; + + #[track_caller] + fn assert_bit_equal(left: Type, right: Type) { + assert_eq!(left.bits, right.bits, "{left} bits are not equal to {right} bits"); + assert_eq!(left.spec, right.spec, "{left} spec is not equal to {right} spec"); + } + + #[track_caller] + fn assert_subtype(left: Type, right: Type) { + assert!(left.is_subtype(right), "{left} is not a subtype of {right}"); + } + + #[track_caller] + fn assert_not_subtype(left: Type, right: Type) { + assert!(!left.is_subtype(right), "{left} is a subtype of {right}"); + } + + #[test] + fn empty_is_subtype_of_everything() { + // Spot check a few cases + assert_subtype(types::Empty, types::NilClass); + assert_subtype(types::Empty, types::Array); + assert_subtype(types::Empty, types::Object); + assert_subtype(types::Empty, types::CUInt16); + assert_subtype(types::Empty, Type::from_cint(types::CInt32, 10)); + assert_subtype(types::Empty, types::Any); + assert_subtype(types::Empty, types::Empty); + } + + #[test] + fn everything_is_a_subtype_of_any() { + // Spot check a few cases + assert_subtype(types::NilClass, types::Any); + assert_subtype(types::Array, types::Any); + assert_subtype(types::Object, types::Any); + assert_subtype(types::CUInt16, types::Any); + assert_subtype(Type::from_cint(types::CInt32, 10), types::Any); + assert_subtype(types::Empty, types::Any); + assert_subtype(types::Any, types::Any); + } + + #[test] + fn from_const() { + let cint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int(12)); + assert_eq!(format!("{}", cint32), "CInt32[12]"); + + let cint32 = Type::from_const(Const::CInt32(-12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int((-12i64) as u64)); + assert_eq!(format!("{}", cint32), "CInt32[-12]"); + + let cuint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cuint32, types::CInt32); + assert_eq!(cuint32.spec, Specialization::Int(12)); + + let cuint32 = Type::from_const(Const::CUInt32(0xffffffff)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xffffffff)); + assert_eq!(format!("{}", cuint32), "CUInt32[4294967295]"); + + let cuint32 = Type::from_const(Const::CUInt32(0xc00087)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xc00087)); + assert_eq!(format!("{}", cuint32), "CUInt32[12583047]"); + } + + #[test] + fn integer() { + assert_subtype(Type::fixnum(123), types::Fixnum); + assert_subtype(Type::fixnum(123), Type::fixnum(123)); + assert_not_subtype(Type::fixnum(123), Type::fixnum(200)); + assert_subtype(Type::from_value(VALUE::fixnum_from_usize(123)), types::Fixnum); + assert_subtype(types::Fixnum, types::Integer); + assert_subtype(types::Bignum, types::Integer); + } + + #[test] + fn float() { + assert_subtype(types::Flonum, types::Float); + assert_subtype(types::HeapFloat, types::Float); + } + + #[test] + fn numeric() { + assert_subtype(types::Integer, types::Numeric); + assert_subtype(types::Float, types::Numeric); + assert_subtype(types::Float.union(types::Integer), types::Numeric); + assert_bit_equal(types::Float + .union(types::Integer) + .union(types::NumericExact) + .union(types::NumericSubclass), types::Numeric); + } + + #[test] + fn symbol() { + assert_subtype(types::StaticSymbol, types::Symbol); + assert_subtype(types::DynamicSymbol, types::Symbol); + } + + #[test] + fn immediate() { + assert_subtype(Type::fixnum(123), types::Immediate); + assert_subtype(types::Fixnum, types::Immediate); + assert_not_subtype(types::Bignum, types::Immediate); + assert_not_subtype(types::Integer, types::Immediate); + assert_subtype(types::NilClass, types::Immediate); + assert_subtype(types::TrueClass, types::Immediate); + assert_subtype(types::FalseClass, types::Immediate); + assert_subtype(types::StaticSymbol, types::Immediate); + assert_not_subtype(types::DynamicSymbol, types::Immediate); + assert_subtype(types::Flonum, types::Immediate); + assert_not_subtype(types::HeapFloat, types::Immediate); + } + + #[test] + fn heap_basic_object() { + assert_not_subtype(Type::fixnum(123), types::HeapBasicObject); + assert_not_subtype(types::Fixnum, types::HeapBasicObject); + assert_subtype(types::Bignum, types::HeapBasicObject); + assert_not_subtype(types::Integer, types::HeapBasicObject); + assert_not_subtype(types::NilClass, types::HeapBasicObject); + assert_not_subtype(types::TrueClass, types::HeapBasicObject); + assert_not_subtype(types::FalseClass, types::HeapBasicObject); + assert_not_subtype(types::StaticSymbol, types::HeapBasicObject); + assert_subtype(types::DynamicSymbol, types::HeapBasicObject); + assert_not_subtype(types::Flonum, types::HeapBasicObject); + assert_subtype(types::HeapFloat, types::HeapBasicObject); + assert_not_subtype(types::BasicObject, types::HeapBasicObject); + assert_not_subtype(types::Object, types::HeapBasicObject); + assert_not_subtype(types::Immediate, types::HeapBasicObject); + assert_not_subtype(types::HeapBasicObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapBasicObject); + assert_subtype(right, types::HeapBasicObject); + assert_subtype(left.union(right), types::HeapBasicObject); + }); + } + + #[test] + fn heap_object() { + assert_not_subtype(Type::fixnum(123), types::HeapObject); + assert_not_subtype(types::Fixnum, types::HeapObject); + assert_subtype(types::Bignum, types::HeapObject); + assert_not_subtype(types::Integer, types::HeapObject); + assert_not_subtype(types::NilClass, types::HeapObject); + assert_not_subtype(types::TrueClass, types::HeapObject); + assert_not_subtype(types::FalseClass, types::HeapObject); + assert_not_subtype(types::StaticSymbol, types::HeapObject); + assert_subtype(types::DynamicSymbol, types::HeapObject); + assert_not_subtype(types::Flonum, types::HeapObject); + assert_subtype(types::HeapFloat, types::HeapObject); + assert_not_subtype(types::BasicObject, types::HeapObject); + assert_not_subtype(types::Object, types::HeapObject); + assert_not_subtype(types::Immediate, types::HeapObject); + assert_not_subtype(types::HeapObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapObject); + assert_subtype(right, types::HeapObject); + assert_subtype(left.union(right), types::HeapObject); + }); + } + + #[test] + fn fixnum_has_ruby_object() { + assert_eq!(Type::fixnum(3).ruby_object(), Some(VALUE::fixnum_from_usize(3))); + assert_eq!(types::Fixnum.ruby_object(), None); + assert_eq!(types::Integer.ruby_object(), None); + } + + #[test] + fn singletons_do_not_have_ruby_object() { + assert_eq!(Type::from_value(Qnil).ruby_object(), None); + assert_eq!(types::NilClass.ruby_object(), None); + assert_eq!(Type::from_value(Qtrue).ruby_object(), None); + assert_eq!(types::TrueClass.ruby_object(), None); + assert_eq!(Type::from_value(Qfalse).ruby_object(), None); + assert_eq!(types::FalseClass.ruby_object(), None); + } + + #[test] + fn integer_has_exact_ruby_class() { + assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.exact_ruby_class(), None); + assert_eq!(types::Integer.exact_ruby_class(), None); + } + + #[test] + fn singletons_do_not_have_exact_ruby_class() { + assert_eq!(Type::from_value(Qnil).exact_ruby_class(), None); + assert_eq!(types::NilClass.exact_ruby_class(), None); + assert_eq!(Type::from_value(Qtrue).exact_ruby_class(), None); + assert_eq!(types::TrueClass.exact_ruby_class(), None); + assert_eq!(Type::from_value(Qfalse).exact_ruby_class(), None); + assert_eq!(types::FalseClass.exact_ruby_class(), None); + } + + #[test] + fn singletons_do_not_have_ruby_class() { + assert_eq!(Type::from_value(Qnil).inexact_ruby_class(), None); + assert_eq!(types::NilClass.inexact_ruby_class(), None); + assert_eq!(Type::from_value(Qtrue).inexact_ruby_class(), None); + assert_eq!(types::TrueClass.inexact_ruby_class(), None); + assert_eq!(Type::from_value(Qfalse).inexact_ruby_class(), None); + assert_eq!(types::FalseClass.inexact_ruby_class(), None); + } + + #[test] + fn from_class() { + crate::cruby::with_rubyvm(|| { + assert_bit_equal(Type::from_class(unsafe { rb_cInteger }), types::Integer); + assert_bit_equal(Type::from_class(unsafe { rb_cString }), types::StringExact); + assert_bit_equal(Type::from_class(unsafe { rb_cArray }), types::ArrayExact); + assert_bit_equal(Type::from_class(unsafe { rb_cHash }), types::HashExact); + assert_bit_equal(Type::from_class(unsafe { rb_cNilClass }), types::NilClass); + assert_bit_equal(Type::from_class(unsafe { rb_cTrueClass }), types::TrueClass); + assert_bit_equal(Type::from_class(unsafe { rb_cFalseClass }), types::FalseClass); + let c_class = define_class("C", unsafe { rb_cObject }); + assert_bit_equal(Type::from_class(c_class), Type { bits: bits::HeapObject, spec: Specialization::TypeExact(c_class) }); + }); + } + + #[test] + fn integer_has_ruby_class() { + crate::cruby::with_rubyvm(|| { + assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.inexact_ruby_class(), None); + assert_eq!(types::Integer.inexact_ruby_class(), None); + }); + } + + #[test] + fn set() { + assert_subtype(types::SetExact, types::Set); + assert_subtype(types::SetSubclass, types::Set); + } + + #[test] + fn set_has_ruby_class() { + crate::cruby::with_rubyvm(|| { + assert_eq!(types::SetExact.runtime_exact_ruby_class(), Some(unsafe { rb_cSet })); + assert_eq!(types::Set.runtime_exact_ruby_class(), None); + assert_eq!(types::SetSubclass.runtime_exact_ruby_class(), None); + }); + } + + #[test] + fn display_exact_bits_match() { + assert_eq!(format!("{}", Type::fixnum(4)), "Fixnum[4]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt8, -1)), "CInt8[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt8, -1)), "CUInt8[255]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt16, -1)), "CInt16[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt16, -1)), "CUInt16[65535]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt32, -1)), "CInt32[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt32, -1)), "CUInt32[4294967295]"); + assert_eq!(format!("{}", Type::from_cint(types::CInt64, -1)), "CInt64[-1]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt64, -1)), "CUInt64[18446744073709551615]"); + assert_eq!(format!("{}", Type::from_cbool(true)), "CBool[true]"); + assert_eq!(format!("{}", Type::from_cbool(false)), "CBool[false]"); + assert_eq!(format!("{}", types::Fixnum), "Fixnum"); + assert_eq!(format!("{}", types::Integer), "Integer"); + } + + #[test] + fn display_multiple_bits() { + assert_eq!(format!("{}", types::CSigned), "CSigned"); + assert_eq!(format!("{}", types::CUInt8.union(types::CInt32)), "CUInt8|CInt32"); + assert_eq!(format!("{}", types::HashExact.union(types::HashSubclass)), "Hash"); + } + + #[test] + fn union_equal() { + assert_bit_equal(types::Fixnum.union(types::Fixnum), types::Fixnum); + assert_bit_equal(Type::fixnum(3).union(Type::fixnum(3)), Type::fixnum(3)); + } + + #[test] + fn union_bits_subtype() { + assert_bit_equal(types::Fixnum.union(types::Integer), types::Integer); + assert_bit_equal(types::Fixnum.union(types::Object), types::Object); + assert_bit_equal(Type::fixnum(3).union(types::Fixnum), types::Fixnum); + + assert_bit_equal(types::Integer.union(types::Fixnum), types::Integer); + assert_bit_equal(types::Object.union(types::Fixnum), types::Object); + assert_bit_equal(types::Fixnum.union(Type::fixnum(3)), types::Fixnum); + } + + #[test] + fn union_bits_unions_bits() { + assert_bit_equal(types::Fixnum.union(types::StaticSymbol), Type { bits: bits::Fixnum | bits::StaticSymbol, spec: Specialization::Any }); + } + + #[test] + fn union_int_specialized() { + assert_bit_equal(Type::from_cbool(true).union(Type::from_cbool(true)), Type::from_cbool(true)); + assert_bit_equal(Type::from_cbool(true).union(Type::from_cbool(false)), types::CBool); + assert_bit_equal(Type::from_cbool(true).union(types::CBool), types::CBool); + + assert_bit_equal(Type::from_cbool(false).union(Type::from_cbool(true)), types::CBool); + assert_bit_equal(types::CBool.union(Type::from_cbool(true)), types::CBool); + } + + #[test] + fn union_one_type_specialized_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let specialized = Type::from_value(unsafe { rb_ary_new_capa(0) }); + let unspecialized = types::StringExact; + assert_bit_equal(specialized.union(unspecialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + assert_bit_equal(unspecialized.union(specialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + }); + } + + #[test] + fn union_specialized_builtin_subtype_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let hello = Type::from_value(rust_str_to_ruby("hello")); + let world = Type::from_value(rust_str_to_ruby("world")); + assert_bit_equal(hello.union(world), types::StringExact); + }); + crate::cruby::with_rubyvm(|| { + let hello = Type::from_value(rust_str_to_sym("hello")); + let world = Type::from_value(rust_str_to_sym("world")); + assert_bit_equal(hello.union(world), types::StaticSymbol); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("hello")); + assert_bit_equal(left.union(right), types::StringExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_sym("hello")); + let right = Type::from_value(rust_str_to_sym("hello")); + assert_bit_equal(left.union(right), left); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_ary_new_capa(0) }); + let right = Type::from_value(unsafe { rb_ary_new_capa(0) }); + assert_bit_equal(left.union(right), types::ArrayExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_hash_new() }); + let right = Type::from_value(unsafe { rb_hash_new() }); + assert_bit_equal(left.union(right), types::HashExact); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_float_new(1.0) }); + let right = Type::from_value(unsafe { rb_float_new(2.0) }); + assert_bit_equal(left.union(right), types::Flonum); + }); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(unsafe { rb_float_new(1.7976931348623157e+308) }); + let right = Type::from_value(unsafe { rb_float_new(1.7976931348623157e+308) }); + assert_bit_equal(left.union(right), types::HeapFloat); + }); + } + + #[test] + fn cme() { + use crate::cruby::{rb_callable_method_entry, ID}; + crate::cruby::with_rubyvm(|| { + let cme = unsafe { rb_callable_method_entry(rb_cInteger, ID!(to_s)) }; + assert!(!cme.is_null()); + let cme_value: VALUE = cme.into(); + let ty = Type::from_value(cme_value); + assert_subtype(ty, types::CallableMethodEntry); + assert!(ty.ruby_object_known()); + }); + } + + #[test] + fn string_subclass_is_string_subtype() { + crate::cruby::with_rubyvm(|| { + assert_subtype(types::StringExact, types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::StringExact); + let c_class = define_class("C", unsafe { rb_cString }); + assert_subtype(Type::from_class(c_class), types::String); + }); + } + + #[test] + fn union_specialized_with_no_relation_returns_unspecialized() { + crate::cruby::with_rubyvm(|| { + let string = Type::from_value(rust_str_to_ruby("hello")); + let array = Type::from_value(unsafe { rb_ary_new_capa(0) }); + assert_bit_equal(string.union(array), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + }); + } + + #[test] + fn union_specialized_with_subclass_relationship_returns_superclass() { + crate::cruby::with_rubyvm(|| { + let c_class = define_class("C", unsafe { rb_cObject }); + let d_class = define_class("D", c_class); + let c_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }; + let d_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(d_class) }; + assert_bit_equal(c_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class)}); + assert_bit_equal(c_instance.union(d_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); + assert_bit_equal(d_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); + }); + } + + #[test] + fn has_value() { + // With known values + crate::cruby::with_rubyvm(|| { + let a = rust_str_to_sym("a"); + let b = rust_str_to_sym("b"); + let ty = Type::from_value(a); + assert!(ty.has_value(Const::Value(a))); + assert!(!ty.has_value(Const::Value(b))); + }); + + let true_ty = Type::from_cbool(true); + assert!(true_ty.has_value(Const::CBool(true))); + assert!(!true_ty.has_value(Const::CBool(false))); + + let int8_ty = Type::from_cint(types::CInt8, 42); + assert!(int8_ty.has_value(Const::CInt8(42))); + assert!(!int8_ty.has_value(Const::CInt8(-1))); + let neg_int8_ty = Type::from_cint(types::CInt8, -1); + assert!(neg_int8_ty.has_value(Const::CInt8(-1))); + + let int16_ty = Type::from_cint(types::CInt16, 1000); + assert!(int16_ty.has_value(Const::CInt16(1000))); + assert!(!int16_ty.has_value(Const::CInt16(2000))); + + let int32_ty = Type::from_cint(types::CInt32, 100000); + assert!(int32_ty.has_value(Const::CInt32(100000))); + assert!(!int32_ty.has_value(Const::CInt32(-100000))); + + let int64_ty = Type::from_cint(types::CInt64, i64::MAX); + assert!(int64_ty.has_value(Const::CInt64(i64::MAX))); + assert!(!int64_ty.has_value(Const::CInt64(0))); + + let uint8_ty = Type::from_cint(types::CUInt8, u8::MAX as i64); + assert!(uint8_ty.has_value(Const::CUInt8(u8::MAX))); + assert!(!uint8_ty.has_value(Const::CUInt8(0))); + + let uint16_ty = Type::from_cint(types::CUInt16, u16::MAX as i64); + assert!(uint16_ty.has_value(Const::CUInt16(u16::MAX))); + assert!(!uint16_ty.has_value(Const::CUInt16(1))); + + let uint32_ty = Type::from_cint(types::CUInt32, u32::MAX as i64); + assert!(uint32_ty.has_value(Const::CUInt32(u32::MAX))); + assert!(!uint32_ty.has_value(Const::CUInt32(42))); + + let uint64_ty = Type::from_cint(types::CUInt64, i64::MAX); + assert!(uint64_ty.has_value(Const::CUInt64(i64::MAX as u64))); + assert!(!uint64_ty.has_value(Const::CUInt64(123))); + + let shape_ty = Type::from_cint(types::CShape, 0x1234); + assert!(shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + assert!(!shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x5678)))); + + let ptr = 0x1000 as *const u8; + let ptr_ty = Type::from_cptr(ptr); + assert!(ptr_ty.has_value(Const::CPtr(ptr))); + assert!(!ptr_ty.has_value(Const::CPtr(0x2000 as *const u8))); + + let double_ty = Type::from_double(std::f64::consts::PI); + assert!(double_ty.has_value(Const::CDouble(std::f64::consts::PI))); + assert!(!double_ty.has_value(Const::CDouble(3.123))); + + let nan_ty = Type::from_double(f64::NAN); + assert!(nan_ty.has_value(Const::CDouble(f64::NAN))); + + // Mismatched types + assert!(!int8_ty.has_value(Const::CInt16(42))); + assert!(!int16_ty.has_value(Const::CInt32(1000))); + assert!(!uint8_ty.has_value(Const::CInt8(-1i8))); + + // Wrong specialization (unknown value) + assert!(!types::CInt8.has_value(Const::CInt8(42))); + assert!(!types::CBool.has_value(Const::CBool(true))); + assert!(!types::CShape.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + } + + #[test] + fn test_subtract_with_superset_returns_empty() { + let left = types::NilClass; + let right = types::BasicObject; + let result = left.subtract(right); + assert_bit_equal(result, types::Empty); + } + + #[test] + fn test_subtract_with_subset_removes_bits() { + let left = types::BasicObject; + let right = types::NilClass; + let result = left.subtract(right); + assert_subtype(result, types::BasicObject); + assert_not_subtype(types::NilClass, result); + } + + #[test] + fn test_subtract_with_no_overlap_returns_self() { + let left = types::Fixnum; + let right = types::StringExact; + let result = left.subtract(right); + assert_bit_equal(result, left); + } + + #[test] + fn test_subtract_with_no_specialization_overlap_returns_self() { + let left = Type::fixnum(4); + let right = Type::fixnum(5); + let result = left.subtract(right); + assert_bit_equal(result, left); + } + + #[test] + fn test_subtract_with_specialization_subset_removes_specialization() { + let left = types::Fixnum; + let right = Type::fixnum(42); + let result = left.subtract(right); + assert_bit_equal(result, types::Fixnum); + } +} diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs new file mode 100644 index 0000000000..f1180acf2a --- /dev/null +++ b/zjit/src/invariants.rs @@ -0,0 +1,476 @@ +//! Code invalidation and patching for speculative optimizations. + +use std::{collections::{HashMap, HashSet}, mem}; + +use crate::{backend::lir::{Assembler, asm_comment}, cruby::{ID, IseqPtr, RedefinitionFlag, VALUE, iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock}, hir::Invariant, options::debug, state::{ZJITState, zjit_enabled_p}, virtualmem::CodePtr}; +use crate::payload::{IseqVersionRef, IseqStatus, get_or_create_iseq_payload}; +use crate::codegen::{MAX_ISEQ_VERSIONS, gen_iseq_call}; +use crate::cruby::{rb_iseq_reset_jit_func, iseq_get_location}; +use crate::stats::with_time_stat; +use crate::stats::Counter::invalidation_time_ns; +use crate::gc::remove_gc_offsets; + +macro_rules! compile_patch_points { + ($cb:expr, $patch_points:expr, $($comment_args:tt)*) => { + with_time_stat(invalidation_time_ns, || { + for patch_point in $patch_points { + let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { + let mut asm = Assembler::new(); + asm.new_block_without_id(); + asm_comment!(asm, $($comment_args)*); + asm.jmp(patch_point.side_exit_ptr.into()); + asm.compile(cb).expect("can write existing code"); + }); + // Stop marking GC offsets corrupted by the jump instruction + remove_gc_offsets(patch_point.version, &written_range); + + // If the ISEQ doesn't have max versions, invalidate this version. + let mut version = patch_point.version; + let iseq = unsafe { version.as_ref() }.iseq; + if !iseq.is_null() { + let payload = get_or_create_iseq_payload(iseq); + if unsafe { version.as_ref() }.status != IseqStatus::Invalidated && payload.versions.len() < MAX_ISEQ_VERSIONS { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(version.as_ref().iseq) }; + + // Recompile JIT-to-JIT calls into the invalidated ISEQ + for incoming in unsafe { version.as_ref() }.incoming.iter() { + if let Err(err) = gen_iseq_call($cb, incoming) { + debug!("{err:?}: gen_iseq_call failed on PatchPoint: {}", iseq_get_location(incoming.iseq.get(), 0)); + } + } + } + } + } + }); + }; +} + +/// When a PatchPoint is invalidated, it generates a jump instruction from `from` to `to`. +#[derive(Debug, Eq, Hash, PartialEq)] +struct PatchPoint { + /// Code pointer to be invalidated + patch_point_ptr: CodePtr, + /// Code pointer to a side exit + side_exit_ptr: CodePtr, + /// ISEQ version to be invalidated + version: IseqVersionRef, +} + +impl PatchPoint { + /// PatchPointer constructor + fn new(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, version: IseqVersionRef) -> PatchPoint { + Self { + patch_point_ptr, + side_exit_ptr, + version, + } + } +} + +/// Used to track all of the various block references that contain assumptions +/// about the state of the virtual machine. +#[derive(Default)] +pub struct Invariants { + /// Set of ISEQs that are known to escape EP + ep_escape_iseqs: HashSet<IseqPtr>, + + /// Map from ISEQ that's assumed to not escape EP to a set of patch points + no_ep_escape_iseq_patch_points: HashMap<IseqPtr, HashSet<PatchPoint>>, + + /// Map from a class and its associated basic operator to a set of patch points + bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet<PatchPoint>>, + + /// Map from CME to patch points that assume the method hasn't been redefined + cme_patch_points: HashMap<*const rb_callable_method_entry_t, HashSet<PatchPoint>>, + + /// Map from constant ID to patch points that assume the constant hasn't been redefined + constant_state_patch_points: HashMap<ID, HashSet<PatchPoint>>, + + /// Set of patch points that assume that the TracePoint is not enabled + no_trace_point_patch_points: HashSet<PatchPoint>, + + /// Set of patch points that assume that the interpreter is running with only one ractor + single_ractor_patch_points: HashSet<PatchPoint>, + + /// Map from a class to a set of patch points that assume objects of the class + /// will have no singleton class. + no_singleton_class_patch_points: HashMap<VALUE, HashSet<PatchPoint>>, +} + +impl Invariants { + /// Update object references in Invariants + pub fn update_references(&mut self) { + self.update_ep_escape_iseqs(); + self.update_no_ep_escape_iseq_patch_points(); + self.update_cme_patch_points(); + self.update_no_singleton_class_patch_points(); + } + + /// Forget an ISEQ when freeing it. We need to because a) if the address is reused, we'd be + /// tracking the wrong object b) dead VALUEs in the table can means we risk passing invalid + /// VALUEs to `rb_gc_location()`. + pub fn forget_iseq(&mut self, iseq: IseqPtr) { + // Why not patch the patch points? If the ISEQ is dead then the GC also proved that all + // generated code referencing the ISEQ are unreachable. We mark the ISEQs baked into + // generated code. + self.ep_escape_iseqs.remove(&iseq); + self.no_ep_escape_iseq_patch_points.remove(&iseq); + } + + /// Forget a CME when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_cme(&mut self, cme: *const rb_callable_method_entry_t) { + self.cme_patch_points.remove(&cme); + } + + /// Forget a class when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_klass(&mut self, klass: VALUE) { + self.no_singleton_class_patch_points.remove(&klass); + } + + /// Update ISEQ references in Invariants::ep_escape_iseqs + fn update_ep_escape_iseqs(&mut self) { + let updated = std::mem::take(&mut self.ep_escape_iseqs) + .into_iter() + .map(|iseq| unsafe { rb_gc_location(iseq.into()) }.as_iseq()) + .collect(); + self.ep_escape_iseqs = updated; + } + + /// Update ISEQ references in Invariants::no_ep_escape_iseq_patch_points + fn update_no_ep_escape_iseq_patch_points(&mut self) { + let updated = std::mem::take(&mut self.no_ep_escape_iseq_patch_points) + .into_iter() + .map(|(iseq, patch_points)| { + let new_iseq = unsafe { rb_gc_location(iseq.into()) }; + (new_iseq.as_iseq(), patch_points) + }) + .collect(); + self.no_ep_escape_iseq_patch_points = updated; + } + + fn update_cme_patch_points(&mut self) { + let updated_cme_patch_points = std::mem::take(&mut self.cme_patch_points) + .into_iter() + .map(|(cme, patch_points)| { + let new_cme = unsafe { rb_gc_location(cme.into()) }; + (new_cme.as_cme(), patch_points) + }) + .collect(); + self.cme_patch_points = updated_cme_patch_points; + } + + fn update_no_singleton_class_patch_points(&mut self) { + let updated_no_singleton_class_patch_points = std::mem::take(&mut self.no_singleton_class_patch_points) + .into_iter() + .map(|(klass, patch_points)| { + let new_klass = unsafe { rb_gc_location(klass) }; + (new_klass, patch_points) + }) + .collect(); + self.no_singleton_class_patch_points = updated_no_singleton_class_patch_points; + } +} + +/// Called when a basic operator is redefined. Note that all the blocks assuming +/// the stability of different operators are invalidated together and we don't +/// do fine-grained tracking. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic_operators) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.bop_patch_points.get(&(klass, bop)) { + let cb = ZJITState::get_code_block(); + let bop = Invariant::BOPRedefined { klass, bop }; + debug!("BOP is redefined: {}", bop); + + // Invalidate all patch points for this BOP + compile_patch_points!(cb, patch_points, "BOP is redefined: {}", bop); + + cb.mark_all_executable(); + } + }); +} + +/// Invalidate blocks for a given ISEQ that assumes environment pointer is +/// equal to base pointer. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_ep_escape(iseq: IseqPtr) { + // Skip tracking EP escapes on boot. We don't need to invalidate anything during boot. + if !ZJITState::has_instance() { + return; + } + + // Remember that this ISEQ may escape EP + let invariants = ZJITState::get_invariants(); + invariants.ep_escape_iseqs.insert(iseq); + + // If the ISEQ has been compiled assuming it doesn't escape EP, invalidate the JIT code. + if let Some(patch_points) = invariants.no_ep_escape_iseq_patch_points.get(&iseq) { + debug!("EP is escaped: {}", iseq_name(iseq)); + + // Invalidate the patch points for this ISEQ + let cb = ZJITState::get_code_block(); + compile_patch_points!(cb, patch_points, "EP is escaped: {}", iseq_name(iseq)); + + cb.mark_all_executable(); + } +} + +/// Track that JIT code for a ISEQ will assume that base pointer is equal to environment pointer. +pub fn track_no_ep_escape_assumption( + iseq: IseqPtr, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_ep_escape_iseq_patch_points.entry(iseq).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Returns true if a given ISEQ has previously escaped environment pointer. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + ZJITState::get_invariants().ep_escape_iseqs.contains(&iseq) +} + +/// Track a patch point for a basic operator in a given class. +pub fn track_bop_assumption( + klass: RedefinitionFlag, + bop: ruby_basic_operators, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for a callable method entry (CME). +pub fn track_cme_assumption( + cme: *const rb_callable_method_entry_t, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for each constant name in a constant path assumption. +pub fn track_stable_constant_names_assumption( + idlist: *const ID, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + + let mut idx = 0; + loop { + let id = unsafe { *idlist.wrapping_add(idx) }; + if id.0 == 0 { + break; + } + + invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); + + idx += 1; + } +} + +/// Track a patch point for objects of a given class will have no singleton class. +pub fn track_no_singleton_class_assumption( + klass: VALUE, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_singleton_class_patch_points.entry(klass).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Called when a method is redefined. Invalidates all JIT code that depends on the CME. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + // Get the CMD's jumps and remove the entry from the map as it has been invalidated + if let Some(patch_points) = invariants.cme_patch_points.remove(&cme) { + let cb = ZJITState::get_code_block(); + debug!("CME is invalidated: {:?}", cme); + + // Invalidate all patch points for this CME + compile_patch_points!(cb, patch_points, "CME is invalidated: {:?}", cme); + + cb.mark_all_executable(); + } + }); +} + +/// Called when a constant is redefined. Invalidates all JIT code that depends on the constant. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.constant_state_patch_points.get(&id) { + let cb = ZJITState::get_code_block(); + debug!("Constant state changed: {:?}", id); + + // Invalidate all patch points for this constant ID + compile_patch_points!(cb, patch_points, "Constant state changed: {:?}", id); + + cb.mark_all_executable(); + } + }); +} + +/// Track the JIT code that assumes that the interpreter is running with only one ractor +pub fn track_single_ractor_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.single_ractor_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Callback for when Ruby is about to spawn a ractor. In that case we need to +/// invalidate every block that is assuming single ractor mode. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_before_ractor_spawn() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); + + // Invalidate all patch points for single ractor mode + compile_patch_points!(cb, patch_points, "Another ractor spawned, invalidating single ractor mode assumption"); + + cb.mark_all_executable(); + }); +} + +pub fn track_no_trace_point_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_trace_point_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_tracing_invalidate_all() { + use crate::payload::{get_or_create_iseq_payload, IseqStatus}; + use crate::cruby::for_each_iseq; + + if !zjit_enabled_p() { + return; + } + + // Stop other ractors since we are going to patch machine code. + with_vm_lock(src_loc!(), || { + debug!("Invalidating all ZJIT compiled code due to TracePoint"); + + for_each_iseq(|iseq| { + let payload = get_or_create_iseq_payload(iseq); + + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + } + unsafe { rb_iseq_reset_jit_func(iseq) }; + }); + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().no_trace_point_patch_points); + + compile_patch_points!(cb, patch_points, "TracePoint is enabled, invalidating no TracePoint assumption"); + + cb.mark_all_executable(); + }); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +/// This is used to avoid an invalidation loop where we repeatedly compile code +/// that assumes no singleton class, only to have it invalidated. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + ZJITState::get_invariants() + .no_singleton_class_patch_points + .get(&klass) + .map_or(false, |patch_points| patch_points.is_empty()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_singleton_class(klass: VALUE) { + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + match invariants.no_singleton_class_patch_points.get_mut(&klass) { + Some(patch_points) => { + // Invalidate existing patch points and let has_singleton_class_of() + // return true when they are compiled again + let patch_points = mem::take(patch_points); + if !patch_points.is_empty() { + let cb = ZJITState::get_code_block(); + debug!("Singleton class created for {:?}", klass); + compile_patch_points!(cb, patch_points, "Singleton class created for {:?}", klass); + cb.mark_all_executable(); + } + } + None => { + // Let has_singleton_class_of() return true for this class + invariants.no_singleton_class_patch_points.insert(klass, HashSet::new()); + } + } + }); +} diff --git a/zjit/src/json.rs b/zjit/src/json.rs new file mode 100644 index 0000000000..fa4b216821 --- /dev/null +++ b/zjit/src/json.rs @@ -0,0 +1,700 @@ +//! Single file JSON serializer for iongraph output of ZJIT HIR. + +use std::{ + fmt, + io::{self, Write}, +}; + +pub trait Jsonable { + fn to_json(&self) -> Json; +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Json { + Null, + Bool(bool), + Integer(isize), + UnsignedInteger(usize), + Floating(f64), + String(String), + Array(Vec<Json>), + Object(Vec<(String, Json)>), +} + +impl Json { + /// Convenience method for constructing a JSON array. + pub fn array<I, T>(iter: I) -> Self + where + I: IntoIterator<Item = T>, + T: Into<Json>, + { + Json::Array(iter.into_iter().map(Into::into).collect()) + } + + pub fn empty_array() -> Self { + Json::Array(Vec::new()) + } + + pub fn object() -> JsonObjectBuilder { + JsonObjectBuilder::new() + } + + pub fn marshal<W: Write>(&self, writer: &mut W) -> JsonResult<()> { + match self { + Json::Null => writer.write_all(b"null"), + Json::Bool(b) => writer.write_all(if *b { b"true" } else { b"false" }), + Json::Integer(i) => write!(writer, "{i}"), + Json::UnsignedInteger(u) => write!(writer, "{u}"), + Json::Floating(f) => write!(writer, "{f}"), + Json::String(s) => return Self::write_str(writer, s), + Json::Array(jsons) => return Self::write_array(writer, jsons), + Json::Object(map) => return Self::write_object(writer, map), + }?; + Ok(()) + } + + pub fn write_str<W: Write>(writer: &mut W, s: &str) -> JsonResult<()> { + writer.write_all(b"\"")?; + + for ch in s.chars() { + match ch { + '"' => write!(writer, "\\\"")?, + '\\' => write!(writer, "\\\\")?, + // The following characters are control, but have a canonical representation. + // https://datatracker.ietf.org/doc/html/rfc8259#section-7 + '\n' => write!(writer, "\\n")?, + '\r' => write!(writer, "\\r")?, + '\t' => write!(writer, "\\t")?, + '\x08' => write!(writer, "\\b")?, + '\x0C' => write!(writer, "\\f")?, + ch if ch.is_control() => { + let code_point = ch as u32; + write!(writer, "\\u{code_point:04X}")? + } + _ => write!(writer, "{ch}")?, + }; + } + + writer.write_all(b"\"")?; + Ok(()) + } + + pub fn write_array<W: Write>(writer: &mut W, jsons: &[Json]) -> JsonResult<()> { + writer.write_all(b"[")?; + let mut prefix = ""; + for item in jsons { + write!(writer, "{prefix}")?; + item.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"]")?; + Ok(()) + } + + pub fn write_object<W: Write>(writer: &mut W, pairs: &[(String, Json)]) -> JsonResult<()> { + writer.write_all(b"{")?; + let mut prefix = ""; + for (k, v) in pairs { + // Escape the keys, despite not being `Json::String` objects. + write!(writer, "{prefix}")?; + Self::write_str(writer, k)?; + writer.write_all(b":")?; + v.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"}")?; + Ok(()) + } +} + +impl std::fmt::Display for Json { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut buf = Vec::new(); + self.marshal(&mut buf).map_err(|_| std::fmt::Error)?; + let s = String::from_utf8(buf).map_err(|_| std::fmt::Error)?; + write!(f, "{s}") + } +} + +pub struct JsonObjectBuilder { + pairs: Vec<(String, Json)>, +} + +impl JsonObjectBuilder { + pub fn new() -> Self { + Self { pairs: Vec::new() } + } + + pub fn insert<K, V>(mut self, key: K, value: V) -> Self + where + K: Into<String>, + V: Into<Json>, + { + self.pairs.push((key.into(), value.into())); + self + } + + pub fn build(self) -> Json { + Json::Object(self.pairs) + } +} + +impl From<&str> for Json { + fn from(s: &str) -> Json { + Json::String(s.to_string()) + } +} + +impl From<String> for Json { + fn from(s: String) -> Json { + Json::String(s) + } +} + +impl From<i32> for Json { + fn from(i: i32) -> Json { + Json::Integer(i as isize) + } +} + +impl From<i64> for Json { + fn from(i: i64) -> Json { + Json::Integer(i as isize) + } +} + +impl From<u32> for Json { + fn from(u: u32) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<u64> for Json { + fn from(u: u64) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<usize> for Json { + fn from(u: usize) -> Json { + Json::UnsignedInteger(u) + } +} + +impl From<bool> for Json { + fn from(b: bool) -> Json { + Json::Bool(b) + } +} + +impl TryFrom<f64> for Json { + type Error = JsonError; + fn try_from(f: f64) -> Result<Self, Self::Error> { + if f.is_finite() { + Ok(Json::Floating(f)) + } else { + Err(JsonError::FloatError(f)) + } + } +} + +impl<T: Into<Json>> From<Vec<T>> for Json { + fn from(v: Vec<T>) -> Json { + Json::Array(v.into_iter().map(|item| item.into()).collect()) + } +} + +/// Convenience type for a result in JSON serialization. +pub type JsonResult<W> = std::result::Result<W, JsonError>; + +#[derive(Debug)] +pub enum JsonError { + /// Wrapper for a standard `io::Error`. + IoError(io::Error), + /// On attempting to serialize an invalid `f32` or `f64`. + /// Stores invalid values as 64 bit float. + FloatError(f64), +} + +impl From<io::Error> for JsonError { + fn from(err: io::Error) -> Self { + JsonError::IoError(err) + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::FloatError(v) => write!(f, "Cannot serialize float {v}"), + JsonError::IoError(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for JsonError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JsonError::IoError(e) => Some(e), + JsonError::FloatError(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + + fn marshal_to_string(json: &Json) -> String { + let mut buf = Vec::new(); + json.marshal(&mut buf).unwrap(); + String::from_utf8(buf).unwrap() + } + + #[test] + fn test_null() { + let json = Json::Null; + assert_snapshot!(marshal_to_string(&json), @"null"); + } + + #[test] + fn test_bool() { + let json: Json = true.into(); + assert_snapshot!(marshal_to_string(&json), @"true"); + let json: Json = false.into(); + assert_snapshot!(marshal_to_string(&json), @"false"); + } + + #[test] + fn test_integer_positive() { + let json: Json = 42.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_integer_negative() { + let json: Json = (-123).into(); + assert_snapshot!(marshal_to_string(&json), @"-123"); + } + + #[test] + fn test_integer_zero() { + let json: Json = 0.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_floating() { + let json = 2.14159.try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"2.14159"); + } + + #[test] + fn test_floating_negative() { + let json = (-2.5).try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"-2.5"); + } + + #[test] + fn test_floating_error() { + let json: Result<Json, JsonError> = f64::NAN.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::NEG_INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + } + + #[test] + fn test_string_simple() { + let json: Json = "hello".into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello""#); + } + + #[test] + fn test_string_empty() { + let json: Json = "".into(); + assert_snapshot!(marshal_to_string(&json), @r#""""#); + } + + #[test] + fn test_string_with_quotes() { + let json: Json = r#"hello "world""#.into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello \"world\"""#); + } + + #[test] + fn test_string_with_backslash() { + let json: Json = r"path\to\file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path\\to\\file""#); + } + + #[test] + fn test_string_with_slash() { + let json: Json = "path/to/file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path/to/file""#); + } + + #[test] + fn test_string_with_newline() { + let json: Json = "line1\nline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\nline2""#); + } + + #[test] + fn test_string_with_carriage_return() { + let json: Json = "line1\rline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\rline2""#); + } + + #[test] + fn test_string_with_tab() { + let json: Json = "col1\tcol2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""col1\tcol2""#); + } + + #[test] + fn test_string_with_backspace() { + let json: Json = "text\x08back".into(); + assert_snapshot!(marshal_to_string(&json), @r#""text\bback""#); + } + + #[test] + fn test_string_with_form_feed() { + let json: Json = "page\x0Cnew".into(); + assert_snapshot!(marshal_to_string(&json), @r#""page\fnew""#); + } + + #[test] + fn test_string_with_control_chars() { + let json: Json = "test\x01\x02\x03".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test\u0001\u0002\u0003""#); + } + + #[test] + fn test_string_with_all_escapes() { + let json: Json = "\"\\/\n\r\t\x08\x0C".into(); + assert_snapshot!(marshal_to_string(&json), @r#""\"\\/\n\r\t\b\f""#); + } + + #[test] + fn test_array_empty() { + let json: Json = Vec::<i32>::new().into(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_array_single_element() { + let json: Json = vec![42].into(); + assert_snapshot!(marshal_to_string(&json), @"[42]"); + } + + #[test] + fn test_array_multiple_elements() { + let json: Json = vec![1, 2, 3].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_array_mixed_types() { + let json = Json::Array(vec![ + Json::Null, + true.into(), + 42.into(), + 3.134.try_into().unwrap(), + "hello".into(), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"[null, true, 42, 3.134, "hello"]"#); + } + + #[test] + fn test_array_nested() { + let json = Json::Array(vec![1.into(), vec![2, 3].into(), 4.into()]); + assert_snapshot!(marshal_to_string(&json), @"[1, [2, 3], 4]"); + } + + #[test] + fn test_object_empty() { + let json = Json::Object(vec![]); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_single_field() { + let json = Json::Object(vec![("key".to_string(), "value".into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_multiple_fields() { + let json = Json::Object(vec![ + ("name".to_string(), "Alice".into()), + ("age".to_string(), 30.into()), + ("active".to_string(), true.into()), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_with_escaped_key() { + let json = Json::Object(vec![("key\nwith\nnewlines".to_string(), 42.into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key\nwith\nnewlines":42}"#); + } + + #[test] + fn test_object_nested() { + let inner = Json::Object(vec![("inner_key".to_string(), "inner_value".into())]); + let json = Json::Object(vec![("outer_key".to_string(), inner)]); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_from_str() { + let json: Json = "test string".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test string""#); + } + + #[test] + fn test_from_i32() { + let json: Json = 42i32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_i64() { + let json: Json = 9223372036854775807i64.into(); + assert_snapshot!(marshal_to_string(&json), @"9223372036854775807"); + } + + #[test] + fn test_from_u32() { + let json: Json = 42u32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_u64() { + let json: Json = 18446744073709551615u64.into(); + assert_snapshot!(marshal_to_string(&json), @"18446744073709551615"); + } + + #[test] + fn test_unsigned_integer_zero() { + let json: Json = 0u64.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_from_bool() { + let json_true: Json = true.into(); + let json_false: Json = false.into(); + assert_snapshot!(marshal_to_string(&json_true), @"true"); + assert_snapshot!(marshal_to_string(&json_false), @"false"); + } + + #[test] + fn test_from_vec() { + let json: Json = vec![1i32, 2i32, 3i32].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_from_vec_strings() { + let json: Json = vec!["a", "b", "c"].into(); + assert_snapshot!(marshal_to_string(&json), @r#"["a", "b", "c"]"#); + } + + #[test] + fn test_complex_nested_structure() { + let settings = Json::Object(vec![ + ("notifications".to_string(), true.into()), + ("theme".to_string(), "dark".into()), + ]); + + let json = Json::Object(vec![ + ("id".to_string(), 1.into()), + ("name".to_string(), "Alice".into()), + ("tags".to_string(), vec!["admin", "user"].into()), + ("settings".to_string(), settings), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"id":1, "name":"Alice", "tags":["admin", "user"], "settings":{"notifications":true, "theme":"dark"}}"#); + } + + #[test] + fn test_deeply_nested_arrays() { + let json = Json::Array(vec![ + Json::Array(vec![vec![1, 2].into(), 3.into()]), + 4.into(), + ]); + assert_snapshot!(marshal_to_string(&json), @"[[[1, 2], 3], 4]"); + } + + #[test] + fn test_unicode_string() { + let json: Json = "兵马俑".into(); + assert_snapshot!(marshal_to_string(&json), @r#""兵马俑""#); + } + + #[test] + fn test_json_array_convenience() { + let json = Json::array(vec![1, 2, 3]); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_json_array_from_iterator() { + let json = Json::array([1, 2, 3].iter().map(|&x| x * 2)); + assert_snapshot!(marshal_to_string(&json), @"[2, 4, 6]"); + } + + #[test] + fn test_json_empty_array() { + let json = Json::empty_array(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_object_builder_empty() { + let json = Json::object().build(); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_builder_single_field() { + let json = Json::object().insert("key", "value").build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_builder_multiple_fields() { + let json = Json::object() + .insert("name", "Alice") + .insert("age", 30) + .insert("active", true) + .build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_builder_with_nested_objects() { + let inner = Json::object().insert("inner_key", "inner_value").build(); + let json = Json::object().insert("outer_key", inner).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_object_builder_with_array() { + let json = Json::object().insert("items", vec![1, 2, 3]).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"items":[1, 2, 3]}"#); + } + + #[test] + fn test_display_trait() { + let json = Json::object() + .insert("name", "Bob") + .insert("count", 42) + .build(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#"{"name":"Bob", "count":42}"#); + } + + #[test] + fn test_display_trait_array() { + let json: Json = vec![1, 2, 3].into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @"[1, 2, 3]"); + } + + #[test] + fn test_display_trait_string() { + let json: Json = "test".into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#""test""#); + } + + #[test] + fn test_from_usize() { + let json: Json = 123usize.into(); + assert_snapshot!(marshal_to_string(&json), @"123"); + } + + #[test] + fn test_from_usize_large() { + let json: Json = usize::MAX.into(); + let expected = format!("{}", usize::MAX); + assert_eq!(marshal_to_string(&json), expected); + } + + #[test] + fn test_json_error_float_display() { + let err = JsonError::FloatError(f64::NAN); + let display_output = format!("{}", err); + assert!(display_output.contains("Cannot serialize float")); + assert!(display_output.contains("NaN")); + } + + #[test] + fn test_json_error_float_display_infinity() { + let err = JsonError::FloatError(f64::INFINITY); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"Cannot serialize float inf"); + } + + #[test] + fn test_json_error_io_display() { + let io_err = io::Error::new(io::ErrorKind::WriteZero, "write error"); + let err = JsonError::IoError(io_err); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"write error"); + } + + #[test] + fn test_io_error_during_marshal() { + struct FailingWriter; + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result<usize> { + Err(io::Error::other("simulated write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let json: Json = "test".into(); + let mut writer = FailingWriter; + let result = json.marshal(&mut writer); + assert!(result.is_err()); + assert!(matches!(result, Err(JsonError::IoError(_)))); + } + + #[test] + fn test_clone_json() { + let json1: Json = vec![1, 2, 3].into(); + let json2 = json1.clone(); + assert_eq!(json1, json2); + } + + #[test] + fn test_debug_json() { + let json: Json = "test".into(); + let debug_output = format!("{:?}", json); + assert!(debug_output.contains("String")); + assert!(debug_output.contains("test")); + } + + #[test] + fn test_partial_eq_json() { + let json1: Json = 42.into(); + let json2: Json = 42.into(); + let json3: Json = 43.into(); + assert_eq!(json1, json2); + assert_ne!(json1, json3); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs new file mode 100644 index 0000000000..a79989c912 --- /dev/null +++ b/zjit/src/lib.rs @@ -0,0 +1,34 @@ +#![allow(dead_code)] +#![allow(static_mut_refs)] + +#![allow(clippy::enum_variant_names)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::needless_bool)] + +// Add std docs to cargo doc. +#[doc(inline)] +pub use std; + +mod state; +mod distribution; +mod cruby; +mod cruby_methods; +mod hir; +mod hir_type; +mod hir_effect; +mod codegen; +mod stats; +mod cast; +mod virtualmem; +mod asm; +mod backend; +#[cfg(feature = "disasm")] +mod disasm; +mod options; +mod profile; +mod invariants; +mod bitset; +mod gc; +mod payload; +mod json; +mod ttycolors; diff --git a/zjit/src/options.rs b/zjit/src/options.rs new file mode 100644 index 0000000000..9121e49bff --- /dev/null +++ b/zjit/src/options.rs @@ -0,0 +1,526 @@ +//! Configurable options for ZJIT. + +use std::{ffi::{CStr, CString}, ptr::null}; +use std::os::raw::{c_char, c_int, c_uint}; +use crate::cruby::*; +use crate::stats::Counter; +use std::collections::HashSet; + +/// Default --zjit-num-profiles +const DEFAULT_NUM_PROFILES: NumProfiles = 5; +pub type NumProfiles = u32; + +/// Default --zjit-call-threshold. This should be large enough to avoid compiling +/// warmup code, but small enough to perform well on micro-benchmarks. +pub const DEFAULT_CALL_THRESHOLD: CallThreshold = 30; +pub type CallThreshold = u64; + +/// Number of calls to start profiling YARV instructions. +/// They are profiled `rb_zjit_call_threshold - rb_zjit_profile_threshold` times, +/// which is equal to --zjit-num-profiles. +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +pub static mut rb_zjit_profile_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD - DEFAULT_NUM_PROFILES as CallThreshold; + +/// Number of calls to compile ISEQ with ZJIT at jit_compile() in vm.c. +/// --zjit-call-threshold=1 compiles on first execution without profiling information. +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +pub static mut rb_zjit_call_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD; + +/// ZJIT command-line options. This is set before rb_zjit_init() sets +/// ZJITState so that we can query some options while loading builtins. +pub static mut OPTIONS: Option<Options> = None; + +#[derive(Clone, Debug)] +pub struct Options { + /// Hard limit of the executable memory block to allocate in bytes. + /// Note that the command line argument is expressed in MiB and not bytes. + pub exec_mem_bytes: usize, + + /// Hard limit of ZJIT's total memory usage. + /// Note that the command line argument is expressed in MiB and not bytes. + pub mem_bytes: usize, + + /// Number of times YARV instructions should be profiled. + pub num_profiles: NumProfiles, + + /// Enable ZJIT statistics + pub stats: bool, + + /// Print stats on exit (when stats is also true) + pub print_stats: bool, + + /// Print stats to file on exit (when stats is also true) + pub print_stats_file: Option<std::path::PathBuf>, + + /// Enable debug logging + pub debug: bool, + + // Whether to enable JIT at boot. This option prevents other + // ZJIT tuning options from enabling ZJIT at boot. + pub disable: bool, + + /// Turn off the HIR optimizer + pub disable_hir_opt: bool, + + /// Dump initial High-level IR before optimization + pub dump_hir_init: Option<DumpHIR>, + + /// Dump High-level IR after optimization, right before codegen. + pub dump_hir_opt: Option<DumpHIR>, + + /// Dump High-level IR to the given file in Graphviz format after optimization + pub dump_hir_graphviz: Option<std::path::PathBuf>, + + /// Dump High-level IR in Iongraph JSON format after optimization to /tmp/zjit-iongraph-{$PID} + pub dump_hir_iongraph: bool, + + /// Dump low-level IR + pub dump_lir: Option<HashSet<DumpLIR>>, + + /// Dump all compiled machine code. + pub dump_disasm: bool, + + /// Trace and write side exit source maps to /tmp for stackprof. + pub trace_side_exits: Option<TraceExits>, + + /// Frequency of tracing side exits. + pub trace_side_exits_sample_interval: usize, + + /// Dump code map to /tmp for performance profilers. + pub perf: bool, + + /// List of ISEQs that can be compiled, identified by their iseq_get_location() + pub allowed_iseqs: Option<HashSet<String>>, + + /// Path to a file where compiled ISEQs will be saved. + pub log_compiled_iseqs: Option<std::path::PathBuf>, +} + +impl Default for Options { + fn default() -> Self { + Options { + exec_mem_bytes: 64 * 1024 * 1024, + mem_bytes: 128 * 1024 * 1024, + num_profiles: DEFAULT_NUM_PROFILES, + stats: false, + print_stats: false, + print_stats_file: None, + debug: false, + disable: false, + disable_hir_opt: false, + dump_hir_init: None, + dump_hir_opt: None, + dump_hir_graphviz: None, + dump_hir_iongraph: false, + dump_lir: None, + dump_disasm: false, + trace_side_exits: None, + trace_side_exits_sample_interval: 0, + perf: false, + allowed_iseqs: None, + log_compiled_iseqs: None, + } + } +} + +/// `ruby --help` descriptions for user-facing options. Do not add options for ZJIT developers. +/// Note that --help allows only 80 chars per line, including indentation, and it also puts the +/// description in a separate line if the option name is too long. 80-char limit --> | (any character beyond this `|` column fails the test) +pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ + ("--zjit-mem-size=num", + "Max amount of memory that ZJIT can use in MiB (default: 128)."), + ("--zjit-call-threshold=num", + "Number of calls to trigger JIT (default: 30)."), + ("--zjit-num-profiles=num", + "Number of profiled calls before JIT (default: 5)."), + ("--zjit-stats-quiet", + "Collect ZJIT stats and suppress output."), + ("--zjit-stats[=file]", + "Collect ZJIT stats (=file to write to a file)."), + ("--zjit-disable", + "Disable ZJIT for lazily enabling it with RubyVM::ZJIT.enable."), + ("--zjit-perf", "Dump ISEQ symbols into /tmp/perf-{}.map for Linux perf."), + ("--zjit-log-compiled-iseqs=path", + "Log compiled ISEQs to the file. The file will be truncated."), + ("--zjit-trace-exits[=counter]", + "Record source on side-exit. `Counter` picks specific counter."), + ("--zjit-trace-exits-sample-rate=num", + "Frequency at which to record side exits. Must be `usize`.") +]; + +#[derive(Copy, Clone, Debug)] +pub enum TraceExits { + // Trace all exits + All, + // Trace exits for a specific `Counter` + Counter(Counter), +} + +#[derive(Clone, Copy, Debug)] +pub enum DumpHIR { + // Dump High-level IR without Snapshot + WithoutSnapshot, + // Dump High-level IR with Snapshot + All, + // Pretty-print bare High-level IR structs + Debug, +} + +/// --zjit-dump-lir values. Using snake_case to stringify the exact filter value. +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum DumpLIR { + /// Dump the initial LIR + init, + /// Dump LIR after {arch}_split + split, + /// Dump LIR after alloc_regs + alloc_regs, + /// Dump LIR after compile_exits + compile_exits, + /// Dump LIR after resolve_parallel_mov + resolve_parallel_mov, + /// Dump LIR after {arch}_scratch_split + scratch_split, +} + +/// All compiler stages for --zjit-dump-lir=all. +const DUMP_LIR_ALL: &[DumpLIR] = &[ + DumpLIR::init, + DumpLIR::split, + DumpLIR::alloc_regs, + DumpLIR::compile_exits, + DumpLIR::resolve_parallel_mov, + DumpLIR::scratch_split, +]; + +/// Maximum value for --zjit-mem-size/--zjit-exec-mem-size in MiB. +/// We set 1TiB just to avoid overflow. We could make it smaller. +const MAX_MEM_MIB: usize = 1024 * 1024; + +/// Macro to dump LIR if --zjit-dump-lir is specified +macro_rules! asm_dump { + ($asm:expr, $target:ident) => { + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::$target) { + println!("LIR {}:\n{}", stringify!($target), $asm); + } + } + }; +} +pub(crate) use asm_dump; + +/// Macro to get an option value by name +macro_rules! get_option { + // Unsafe is ok here because options are initialized + // once before any Ruby code executes + ($option_name:ident) => { + unsafe { crate::options::OPTIONS.as_ref() }.unwrap().$option_name + }; +} +pub(crate) use get_option; + +/// Set default values to ZJIT options. Setting Some to OPTIONS will make `#with_jit` +/// enable the JIT hook while not enabling compilation yet. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_prepare_options() { + // rb_zjit_prepare_options() could be called for feature flags or $RUBY_ZJIT_ENABLE + // after rb_zjit_parse_option() is called, so we need to handle the already-initialized case. + if unsafe { OPTIONS.is_none() } { + unsafe { OPTIONS = Some(Options::default()); } + } +} + +/// Parse a --zjit* command-line flag +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_parse_option(str_ptr: *const c_char) -> bool { + parse_option(str_ptr).is_some() +} + +fn parse_jit_list(path_like: &str) -> HashSet<String> { + // Read lines from the file + let mut result = HashSet::new(); + if let Ok(lines) = std::fs::read_to_string(path_like) { + for line in lines.lines() { + let trimmed = line.trim(); + if !trimmed.is_empty() { + result.insert(trimmed.to_string()); + } + } + } else { + eprintln!("Failed to read JIT list from '{path_like}'"); + } + eprintln!("JIT list:"); + for item in &result { + eprintln!(" {item}"); + } + result +} + +/// Expected to receive what comes after the third dash in "--zjit-*". +/// Empty string means user passed only "--zjit". C code rejects when +/// they pass exact "--zjit-". +fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { + rb_zjit_prepare_options(); + let options = unsafe { OPTIONS.as_mut().unwrap() }; + + let c_str: &CStr = unsafe { CStr::from_ptr(str_ptr) }; + let opt_str: &str = c_str.to_str().ok()?; + + // Split the option name and value strings + // Note that some options do not contain an assignment + let parts = opt_str.split_once('='); + let (opt_name, opt_val) = match parts { + Some((before_eq, after_eq)) => (before_eq, after_eq), + None => (opt_str, ""), + }; + + // Match on the option name and value strings + match (opt_name, opt_val) { + ("", "") => {}, // Simply --zjit + + ("mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + + ("exec-mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.exec_mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + + ("call-threshold", _) => match opt_val.parse() { + Ok(n) => { + unsafe { rb_zjit_call_threshold = n; } + update_profile_threshold(); + }, + Err(_) => return None, + }, + + ("num-profiles", _) => match opt_val.parse() { + Ok(n) => { + options.num_profiles = n; + update_profile_threshold(); + }, + Err(_) => return None, + }, + + + ("stats-quiet", _) => { + options.stats = true; + options.print_stats = false; + } + + ("stats", "") => { + options.stats = true; + options.print_stats = true; + } + ("stats", path) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path) + .map_err(|e| eprintln!("Failed to open file '{}': {}", path, e)) + .ok(); + let canonical_path = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.stats = true; + options.print_stats_file = Some(canonical_path); + } + + ("trace-exits", exits) => { + options.trace_side_exits = match exits { + "" => Some(TraceExits::All), + name => Some(Counter::get(name).map(TraceExits::Counter)?), + } + } + + ("trace-exits-sample-rate", sample_interval) => { + // If not already set, then set it to `TraceExits::All` by default. + if options.trace_side_exits.is_none() { + options.trace_side_exits = Some(TraceExits::All); + } + // `sample_interval ` must provide a string that can be validly parsed to a `usize`. + options.trace_side_exits_sample_interval = sample_interval.parse::<usize>().ok()?; + } + + ("debug", "") => options.debug = true, + + ("disable", "") => options.disable = true, + + ("disable-hir-opt", "") => options.disable_hir_opt = true, + + // --zjit-dump-hir dumps the actual input to the codegen, which is currently the same as --zjit-dump-hir-opt. + ("dump-hir" | "dump-hir-opt", "") => options.dump_hir_opt = Some(DumpHIR::WithoutSnapshot), + ("dump-hir" | "dump-hir-opt", "all") => options.dump_hir_opt = Some(DumpHIR::All), + ("dump-hir" | "dump-hir-opt", "debug") => options.dump_hir_opt = Some(DumpHIR::Debug), + + ("dump-hir-init", "") => options.dump_hir_init = Some(DumpHIR::WithoutSnapshot), + ("dump-hir-init", "all") => options.dump_hir_init = Some(DumpHIR::All), + ("dump-hir-init", "debug") => options.dump_hir_init = Some(DumpHIR::Debug), + + ("dump-hir-graphviz", "") => options.dump_hir_graphviz = Some("/dev/stderr".into()), + ("dump-hir-graphviz", _) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.dump_hir_graphviz = Some(opt_val); + } + + ("dump-hir-iongraph", "") => options.dump_hir_iongraph = true, + + ("dump-lir", "") => options.dump_lir = Some(HashSet::from([DumpLIR::init])), + ("dump-lir", filters) => { + let mut dump_lirs = HashSet::new(); + for filter in filters.split(',') { + let dump_lir = match filter { + "all" => { + for &dump_lir in DUMP_LIR_ALL { + dump_lirs.insert(dump_lir); + } + continue; + } + "init" => DumpLIR::init, + "split" => DumpLIR::split, + "alloc_regs" => DumpLIR::alloc_regs, + "compile_exits" => DumpLIR::compile_exits, + "scratch_split" => DumpLIR::scratch_split, + _ => { + let valid_options = DUMP_LIR_ALL.iter().map(|opt| format!("{opt:?}")).collect::<Vec<_>>().join(", "); + eprintln!("invalid --zjit-dump-lir option: '{filter}'"); + eprintln!("valid --zjit-dump-lir options: all, {valid_options}"); + return None; + } + }; + dump_lirs.insert(dump_lir); + } + options.dump_lir = Some(dump_lirs); + } + + ("dump-disasm", "") => options.dump_disasm = true, + + ("perf", "") => options.perf = true, + + ("allowed-iseqs", _) if !opt_val.is_empty() => options.allowed_iseqs = Some(parse_jit_list(opt_val)), + ("log-compiled-iseqs", _) if !opt_val.is_empty() => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.log_compiled_iseqs = Some(opt_val); + } + + _ => return None, // Option name not recognized + } + + // Option successfully parsed + Some(()) +} + +/// Update rb_zjit_profile_threshold based on rb_zjit_call_threshold and options.num_profiles +fn update_profile_threshold() { + if unsafe { rb_zjit_call_threshold == 1 } { + // If --zjit-call-threshold=1, never rewrite ISEQs to profile instructions. + unsafe { rb_zjit_profile_threshold = 0; } + } else { + // Otherwise, profile instructions at least once. + let num_profiles = get_option!(num_profiles); + unsafe { rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(num_profiles.into()).max(1) }; + } +} + +/// Update --zjit-call-threshold for testing +#[cfg(test)] +pub fn set_call_threshold(call_threshold: CallThreshold) { + unsafe { rb_zjit_call_threshold = call_threshold; } + rb_zjit_prepare_options(); + update_profile_threshold(); +} + +/// Print YJIT options for `ruby --help`. `width` is width of option parts, and +/// `columns` is indent width of descriptions. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_show_usage(help: c_int, highlight: c_int, width: c_uint, columns: c_int) { + for &(name, description) in ZJIT_OPTIONS.iter() { + unsafe extern "C" { + fn ruby_show_usage_line(name: *const c_char, secondary: *const c_char, description: *const c_char, + help: c_int, highlight: c_int, width: c_uint, columns: c_int); + } + let name = CString::new(name).unwrap(); + let description = CString::new(description).unwrap(); + unsafe { ruby_show_usage_line(name.as_ptr(), null(), description.as_ptr(), help, highlight, width, columns) } + } +} + +/// Macro to print a message only when --zjit-debug is given +macro_rules! debug { + ($($msg:tt)*) => { + if $crate::options::get_option!(debug) { + eprintln!($($msg)*); + } + }; +} +pub(crate) use debug; + +/// Return true if ZJIT should be enabled at boot. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_option_enable() -> bool { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| !opts.disable) { + true + } else { + false + } +} + +/// Return Qtrue if --zjit-stats has been specified. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return Qtrue if stats should be printed at exit. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_print_stats_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats && opts.print_stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return path if stats should be printed at exit to a specified file, else Qnil. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_stats_file_path_p(_ec: EcPtr, _self: VALUE) -> VALUE { + if let Some(opts) = unsafe { OPTIONS.as_ref() } { + if let Some(ref path) = opts.print_stats_file { + return rust_str_to_ruby(path.as_os_str().to_str().unwrap()); + } + } + Qnil +} diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs new file mode 100644 index 0000000000..8540d5e35c --- /dev/null +++ b/zjit/src/payload.rs @@ -0,0 +1,116 @@ +use std::ffi::c_void; +use std::ptr::NonNull; +use crate::codegen::IseqCallRef; +use crate::stats::CompileError; +use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; + +/// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. +#[derive(Debug)] +pub struct IseqPayload { + /// Type information of YARV instruction operands + pub profile: IseqProfile, + /// JIT code versions. Different versions should have different assumptions. + pub versions: Vec<IseqVersionRef>, +} + +impl IseqPayload { + fn new(iseq_size: u32) -> Self { + Self { + profile: IseqProfile::new(iseq_size), + versions: vec![], + } + } +} + +/// JIT code version. When the same ISEQ is compiled with a different assumption, a new version is created. +#[derive(Debug)] +pub struct IseqVersion { + /// ISEQ pointer. Stored here to minimize the size of PatchPoint. + pub iseq: IseqPtr, + + /// Compilation status of the ISEQ. It has the JIT code address of the first block if Compiled. + pub status: IseqStatus, + + /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. + pub gc_offsets: Vec<CodePtr>, + + /// JIT-to-JIT calls from the ISEQ. The IseqPayload's ISEQ is the caller of it. + pub outgoing: Vec<IseqCallRef>, + + /// JIT-to-JIT calls to the ISEQ. The IseqPayload's ISEQ is the callee of it. + pub incoming: Vec<IseqCallRef>, +} + +/// We use a raw pointer instead of Rc to save space for refcount +pub type IseqVersionRef = NonNull<IseqVersion>; + +impl IseqVersion { + /// Allocate a new IseqVersion to be compiled + pub fn new(iseq: IseqPtr) -> IseqVersionRef { + let version = Self { + iseq, + status: IseqStatus::NotCompiled, + gc_offsets: vec![], + outgoing: vec![], + incoming: vec![], + }; + let version_ptr = Box::into_raw(Box::new(version)); + NonNull::new(version_ptr).expect("no null from Box") + } +} + +/// Set of CodePtrs for an ISEQ +#[derive(Clone, Debug, PartialEq)] +pub struct IseqCodePtrs { + /// Entry for the interpreter + pub start_ptr: CodePtr, + /// Entries for JIT-to-JIT calls + pub jit_entry_ptrs: Vec<CodePtr>, +} + +#[derive(Debug, PartialEq)] +pub enum IseqStatus { + Compiled(IseqCodePtrs), + CantCompile(CompileError), + NotCompiled, + Invalidated, +} + +/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { + type VoidPtr = *mut c_void; + + unsafe { + let payload = rb_iseq_get_zjit_payload(iseq); + if payload.is_null() { + // Allocate a new payload with Box and transfer ownership to the GC. + // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. + // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. + // We allocate in those cases anyways. + let iseq_size = get_iseq_encoded_size(iseq); + let new_payload = IseqPayload::new(iseq_size); + let new_payload = Box::into_raw(Box::new(new_payload)); + rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); + + new_payload + } else { + payload as *mut IseqPayload + } + } +} + +/// Get the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { + let payload_non_null = get_or_create_iseq_payload_ptr(iseq); + payload_ptr_as_mut(payload_non_null) +} + +/// Convert an IseqPayload pointer to a mutable reference. Only one reference +/// should be kept at a time. +pub fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { + // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have + // exclusive mutable access. + // Hmm, nothing seems to stop calling this on the same + // iseq twice, though, which violates aliasing rules. + unsafe { payload_ptr.as_mut() }.unwrap() +} diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs new file mode 100644 index 0000000000..7a584afd6f --- /dev/null +++ b/zjit/src/profile.rs @@ -0,0 +1,434 @@ +//! Profiler for runtime information. + +// We use the YARV bytecode constants which have a CRuby-style name +#![allow(non_upper_case_globals)] + +use std::collections::HashMap; +use crate::{cruby::*, payload::get_or_create_iseq_payload, options::{get_option, NumProfiles}}; +use crate::distribution::{Distribution, DistributionSummary}; +use crate::stats::Counter::profile_time_ns; +use crate::stats::with_time_stat; + +/// Ephemeral state for profiling runtime information +struct Profiler { + cfp: CfpPtr, + iseq: IseqPtr, + insn_idx: usize, +} + +impl Profiler { + fn new(ec: EcPtr) -> Self { + let cfp = unsafe { get_ec_cfp(ec) }; + let iseq = unsafe { get_cfp_iseq(cfp) }; + Profiler { + cfp, + iseq, + insn_idx: unsafe { get_cfp_pc(cfp).offset_from(get_iseq_body_iseq_encoded(iseq)) as usize }, + } + } + + // Get an instruction operand that sits next to the opcode at PC. + fn insn_opnd(&self, idx: usize) -> VALUE { + unsafe { get_cfp_pc(self.cfp).add(1 + idx).read() } + } + + // Peek at the nth topmost value on the Ruby stack. + // Returns the topmost value when n == 0. + fn peek_at_stack(&self, n: isize) -> VALUE { + unsafe { + let sp: *mut VALUE = get_cfp_sp(self.cfp); + *(sp.offset(-1 - n)) + } + } + + fn peek_at_self(&self) -> VALUE { + unsafe { rb_get_cfp_self(self.cfp) } + } + + fn peek_at_block_handler(&self) -> VALUE { + unsafe { rb_vm_get_untagged_block_handler(self.cfp) } + } +} + +/// API called from zjit_* instruction. opcode is the bare (non-zjit_*) instruction. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_profile_insn(bare_opcode: u32, ec: EcPtr) { + with_vm_lock(src_loc!(), || { + with_time_stat(profile_time_ns, || profile_insn(bare_opcode as ruby_vminsn_type, ec)); + }); +} + +/// Profile a YARV instruction +fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { + let profiler = &mut Profiler::new(ec); + let profile = &mut get_or_create_iseq_payload(profiler.iseq).profile; + match bare_opcode { + YARVINSN_opt_nil_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_plus => profile_operands(profiler, profile, 2), + YARVINSN_opt_minus => profile_operands(profiler, profile, 2), + YARVINSN_opt_mult => profile_operands(profiler, profile, 2), + YARVINSN_opt_div => profile_operands(profiler, profile, 2), + YARVINSN_opt_mod => profile_operands(profiler, profile, 2), + YARVINSN_opt_eq => profile_operands(profiler, profile, 2), + YARVINSN_opt_neq => profile_operands(profiler, profile, 2), + YARVINSN_opt_lt => profile_operands(profiler, profile, 2), + YARVINSN_opt_le => profile_operands(profiler, profile, 2), + YARVINSN_opt_gt => profile_operands(profiler, profile, 2), + YARVINSN_opt_ge => profile_operands(profiler, profile, 2), + YARVINSN_opt_and => profile_operands(profiler, profile, 2), + YARVINSN_opt_or => profile_operands(profiler, profile, 2), + YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_aref => profile_operands(profiler, profile, 2), + YARVINSN_opt_ltlt => profile_operands(profiler, profile, 2), + YARVINSN_opt_aset => profile_operands(profiler, profile, 3), + YARVINSN_opt_not => profile_operands(profiler, profile, 1), + YARVINSN_getinstancevariable => profile_self(profiler, profile), + YARVINSN_setinstancevariable => profile_self(profiler, profile), + YARVINSN_definedivar => profile_self(profiler, profile), + YARVINSN_opt_regexpmatch2 => profile_operands(profiler, profile, 2), + YARVINSN_objtostring => profile_operands(profiler, profile, 1), + YARVINSN_opt_length => profile_operands(profiler, profile, 1), + YARVINSN_opt_size => profile_operands(profiler, profile, 1), + YARVINSN_opt_succ => profile_operands(profiler, profile, 1), + YARVINSN_invokeblock => profile_block_handler(profiler, profile), + YARVINSN_invokesuper => profile_invokesuper(profiler, profile), + YARVINSN_opt_send_without_block | YARVINSN_send => { + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, (argc + 1) as usize); + } + _ => {} + } + + // Once we profile the instruction num_profiles times, we stop profiling it. + profile.num_profiles[profiler.insn_idx] = profile.num_profiles[profiler.insn_idx].saturating_add(1); + if profile.num_profiles[profiler.insn_idx] == get_option!(num_profiles) { + unsafe { rb_zjit_iseq_insn_set(profiler.iseq, profiler.insn_idx as u32, bare_opcode); } + } +} + +const DISTRIBUTION_SIZE: usize = 4; + +pub type TypeDistribution = Distribution<ProfiledType, DISTRIBUTION_SIZE>; + +pub type TypeDistributionSummary = DistributionSummary<ProfiledType, DISTRIBUTION_SIZE>; + +/// Profile the Type of top-`n` stack operands +fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(n, TypeDistribution::new()); + } + + for (i, profile_type) in types.iter_mut().enumerate() { + let obj = profiler.peek_at_stack((n - i - 1) as isize); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + profile_type.observe(ty); + } +} + +fn profile_self(profiler: &mut Profiler, profile: &mut IseqProfile) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_self(); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + types[0].observe(ty); +} + +fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_block_handler(); + let ty = ProfiledType::object(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + types[0].observe(ty); +} + +fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { + let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; + let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE + + match profile.super_cme.get(&profiler.insn_idx) { + None => { + // If `None`, then this is our first time looking at `super` for this instruction. + profile.super_cme.insert(profiler.insn_idx, Some(cme_value)); + }, + Some(Some(existing_cme)) => { + // Check if the stored method entry is the same as the current one. If it isn't, then + // mark the call site as polymorphic. + if *existing_cme != cme_value { + profile.super_cme.insert(profiler.insn_idx, None); + } + } + Some(None) => { + // We've visited this instruction and explicitly stored `None` to mark the call site + // as polymorphic. + } + } + + unsafe { rb_gc_writebarrier(profiler.iseq.into(), cme_value) }; + + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, (argc + 1) as usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Flags(u32); + +impl Flags { + const NONE: u32 = 0; + const IS_IMMEDIATE: u32 = 1 << 0; + /// Object is embedded and the ivar index lands within the object + const IS_EMBEDDED: u32 = 1 << 1; + /// Object is a T_OBJECT + const IS_T_OBJECT: u32 = 1 << 2; + /// Object is a struct with embedded fields + const IS_STRUCT_EMBEDDED: u32 = 1 << 3; + /// Set if the ProfiledType is used for profiling specific objects, not just classes/shapes + const IS_OBJECT_PROFILING: u32 = 1 << 4; + + pub fn none() -> Self { Self(Self::NONE) } + + pub fn immediate() -> Self { Self(Self::IS_IMMEDIATE) } + pub fn is_immediate(self) -> bool { (self.0 & Self::IS_IMMEDIATE) != 0 } + pub fn is_embedded(self) -> bool { (self.0 & Self::IS_EMBEDDED) != 0 } + pub fn is_t_object(self) -> bool { (self.0 & Self::IS_T_OBJECT) != 0 } + pub fn is_struct_embedded(self) -> bool { (self.0 & Self::IS_STRUCT_EMBEDDED) != 0 } + pub fn is_object_profiling(self) -> bool { (self.0 & Self::IS_OBJECT_PROFILING) != 0 } +} + +/// opt_send_without_block/opt_plus/... should store: +/// * the class of the receiver, so we can do method lookup +/// * the shape of the receiver, so we can optimize ivar lookup +/// +/// with those two, pieces of information, we can also determine when an object is an immediate: +/// * Integer + IS_IMMEDIATE == Fixnum +/// * Float + IS_IMMEDIATE == Flonum +/// * Symbol + IS_IMMEDIATE == StaticSymbol +/// * NilClass == Nil +/// * TrueClass == True +/// * FalseClass == False +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProfiledType { + class: VALUE, + shape: ShapeId, + flags: Flags, +} + +impl Default for ProfiledType { + fn default() -> Self { + Self::empty() + } +} + +impl ProfiledType { + /// Profile the object itself + fn object(obj: VALUE) -> Self { + let mut flags = Flags::none(); + flags.0 |= Flags::IS_OBJECT_PROFILING; + Self { class: obj, shape: INVALID_SHAPE_ID, flags } + } + + /// Profile the class and shape of the given object + fn new(obj: VALUE) -> Self { + if obj == Qfalse { + return Self { class: unsafe { rb_cFalseClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qtrue { + return Self { class: unsafe { rb_cTrueClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qnil { + return Self { class: unsafe { rb_cNilClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.fixnum_p() { + return Self { class: unsafe { rb_cInteger }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.flonum_p() { + return Self { class: unsafe { rb_cFloat }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.static_sym_p() { + return Self { class: unsafe { rb_cSymbol }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + let mut flags = Flags::none(); + if obj.embedded_p() { + flags.0 |= Flags::IS_EMBEDDED; + } + if obj.struct_embedded_p() { + flags.0 |= Flags::IS_STRUCT_EMBEDDED; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_OBJECT) } { + flags.0 |= Flags::IS_T_OBJECT; + } + Self { class: obj.class_of(), shape: obj.shape_id_of(), flags } + } + + pub fn empty() -> Self { + Self { class: VALUE(0), shape: INVALID_SHAPE_ID, flags: Flags::none() } + } + + pub fn is_empty(&self) -> bool { + self.class == VALUE(0) + } + + pub fn class(&self) -> VALUE { + self.class + } + + pub fn shape(&self) -> ShapeId { + self.shape + } + + pub fn flags(&self) -> Flags { + self.flags + } + + pub fn is_fixnum(&self) -> bool { + self.class == unsafe { rb_cInteger } && self.flags.is_immediate() + } + + pub fn is_string(&self) -> bool { + if self.flags.is_object_profiling() { + panic!("should not call is_string on object-profiled ProfiledType"); + } + // Fast paths for immediates and exact-class + if self.flags.is_immediate() { + return false; + } + + let string = unsafe { rb_cString }; + if self.class == string{ + return true; + } + + self.class.is_subclass_of(string) == ClassRelationship::Subclass + } + + pub fn is_flonum(&self) -> bool { + self.class == unsafe { rb_cFloat } && self.flags.is_immediate() + } + + pub fn is_static_symbol(&self) -> bool { + self.class == unsafe { rb_cSymbol } && self.flags.is_immediate() + } + + pub fn is_nil(&self) -> bool { + self.class == unsafe { rb_cNilClass } && self.flags.is_immediate() + } + + pub fn is_true(&self) -> bool { + self.class == unsafe { rb_cTrueClass } && self.flags.is_immediate() + } + + pub fn is_false(&self) -> bool { + self.class == unsafe { rb_cFalseClass } && self.flags.is_immediate() + } +} + +#[derive(Debug)] +pub struct IseqProfile { + /// Type information of YARV instruction operands, indexed by the instruction index + opnd_types: Vec<Vec<TypeDistribution>>, + + /// Number of profiled executions for each YARV instruction, indexed by the instruction index + num_profiles: Vec<NumProfiles>, + + /// Method entries for `super` calls (stored as VALUE to be GC-safe) + super_cme: HashMap<usize, Option<VALUE>> +} + +impl IseqProfile { + pub fn new(iseq_size: u32) -> Self { + Self { + opnd_types: vec![vec![]; iseq_size as usize], + num_profiles: vec![0; iseq_size as usize], + super_cme: HashMap::new(), + } + } + + /// Get profiled operand types for a given instruction index + pub fn get_operand_types(&self, insn_idx: usize) -> Option<&[TypeDistribution]> { + self.opnd_types.get(insn_idx).map(|v| &**v) + } + + pub fn get_super_method_entry(&self, insn_idx: usize) -> Option<*const rb_callable_method_entry_t> { + self.super_cme.get(&insn_idx) + .and_then(|opt| opt.map(|v| v.0 as *const rb_callable_method_entry_t)) + } + + /// Run a given callback with every object in IseqProfile + pub fn each_object(&self, callback: impl Fn(VALUE)) { + for operands in &self.opnd_types { + for distribution in operands { + for profiled_type in distribution.each_item() { + // If the type is a GC object, call the callback + callback(profiled_type.class); + } + } + } + + for cme_value in self.super_cme.values() { + if let Some(cme) = cme_value { + callback(*cme); + } + } + } + + /// Run a given callback with a mutable reference to every object in IseqProfile. + pub fn each_object_mut(&mut self, callback: impl Fn(&mut VALUE)) { + for operands in &mut self.opnd_types { + for distribution in operands { + for ref mut profiled_type in distribution.each_item_mut() { + // If the type is a GC object, call the callback + callback(&mut profiled_type.class); + } + } + } + + // Update CME references if they move during compaction. + for cme_value in self.super_cme.values_mut() { + if let Some(cme) = cme_value { + callback(cme); + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::cruby::*; + + #[test] + fn can_profile_block_handler() { + with_rubyvm(|| eval(" + def foo = yield + foo rescue 0 + foo rescue 0 + ")); + } +} diff --git a/zjit/src/state.rs b/zjit/src/state.rs new file mode 100644 index 0000000000..a807be3f12 --- /dev/null +++ b/zjit/src/state.rs @@ -0,0 +1,549 @@ +//! Runtime state of ZJIT. + +use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_exit_trampoline_with_counter, gen_function_stub_hit_trampoline}; +use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_vm_insn_addr2opcode, rb_profile_frames, VALUE, VM_INSTRUCTION_SIZE, size_t, rb_gc_mark, with_vm_lock}; +use crate::cruby_methods; +use crate::invariants::Invariants; +use crate::asm::CodeBlock; +use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::stats::{Counters, InsnCounters, SideExitLocations}; +use crate::virtualmem::CodePtr; +use std::collections::HashMap; +use std::ptr::null; + +/// Shared trampoline to enter ZJIT. Not null when ZJIT is enabled. +#[allow(non_upper_case_globals)] +#[unsafe(no_mangle)] +pub static mut rb_zjit_entry: *const u8 = null(); + +/// Like rb_zjit_enabled_p, but for Rust code. +pub fn zjit_enabled_p() -> bool { + unsafe { rb_zjit_entry != null() } +} + +/// Global state needed for code generation +pub struct ZJITState { + /// Inline code block (fast path) + code_block: CodeBlock, + + /// ZJIT statistics + counters: Counters, + + /// Side-exit counters + exit_counters: InsnCounters, + + /// Send fallback counters + send_fallback_counters: InsnCounters, + + /// Assumptions that require invalidation + invariants: Invariants, + + /// Assert successful compilation if set to true + assert_compiles: bool, + + /// Properties of core library methods + method_annotations: cruby_methods::Annotations, + + /// Trampoline to side-exit without restoring PC or the stack + exit_trampoline: CodePtr, + + /// Trampoline to side-exit and increment exit_compilation_failure + exit_trampoline_with_counter: CodePtr, + + /// Trampoline to call function_stub_hit + function_stub_hit_trampoline: CodePtr, + + /// Counter pointers for full frame C functions + full_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for un-annotated C functions + not_annotated_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for all calls to any kind of C function from JIT code + ccall_counter_pointers: HashMap<String, Box<u64>>, + + /// Locations of side exists within generated code + exit_locations: Option<SideExitLocations>, +} + +/// Tracks the initialization progress +enum InitializationState { + Uninitialized, + + /// At boot time, rb_zjit_init will be called regardless of whether + /// ZJIT is enabled, in this phase we initialize any states that must + /// be captured at during boot. + Initialized(cruby_methods::Annotations), + + /// When ZJIT is enabled, either during boot with `--zjit`, or lazily + /// at a later time with `RubyVM::ZJIT.enable`, we perform the rest + /// of the initialization steps and produce the `ZJITState` instance. + Enabled(ZJITState), + + /// Indicates that ZJITState::init has panicked. Should never be + /// encountered in practice since we abort immediately when that + /// happens. + Panicked, +} + +/// Private singleton instance of the codegen globals +static mut ZJIT_STATE: InitializationState = InitializationState::Uninitialized; + +impl ZJITState { + /// Initialize the ZJIT globals. Return the address of the JIT entry trampoline. + pub fn init() -> *const u8 { + use InitializationState::*; + + let initialization_state = unsafe { + std::mem::replace(&mut ZJIT_STATE, Panicked) + }; + + let Initialized(method_annotations) = initialization_state else { + panic!("rb_zjit_init was never called"); + }; + + let mut cb = { + use crate::options::*; + use crate::virtualmem::*; + use std::rc::Rc; + use std::cell::RefCell; + + let mem_block = VirtualMem::alloc(get_option!(exec_mem_bytes), Some(get_option!(mem_bytes))); + let mem_block = Rc::new(RefCell::new(mem_block)); + + CodeBlock::new(mem_block.clone(), get_option!(dump_disasm)) + }; + + let entry_trampoline = gen_entry_trampoline(&mut cb).unwrap().raw_ptr(&cb); + let exit_trampoline = gen_exit_trampoline(&mut cb).unwrap(); + let function_stub_hit_trampoline = gen_function_stub_hit_trampoline(&mut cb).unwrap(); + + let exit_locations = if get_option!(trace_side_exits).is_some() { + Some(SideExitLocations::default()) + } else { + None + }; + + // Initialize the codegen globals instance + let zjit_state = ZJITState { + code_block: cb, + counters: Counters::default(), + exit_counters: [0; VM_INSTRUCTION_SIZE as usize], + send_fallback_counters: [0; VM_INSTRUCTION_SIZE as usize], + invariants: Invariants::default(), + assert_compiles: false, + method_annotations, + exit_trampoline, + function_stub_hit_trampoline, + exit_trampoline_with_counter: exit_trampoline, + full_frame_cfunc_counter_pointers: HashMap::new(), + not_annotated_frame_cfunc_counter_pointers: HashMap::new(), + ccall_counter_pointers: HashMap::new(), + exit_locations, + }; + unsafe { ZJIT_STATE = Enabled(zjit_state); } + + // With --zjit-stats, use a different trampoline on function stub exits + // to count exit_compilation_failure. Note that the trampoline code depends + // on the counter, so ZJIT_STATE needs to be initialized first. + if get_option!(stats) { + let cb = ZJITState::get_code_block(); + let code_ptr = gen_exit_trampoline_with_counter(cb, exit_trampoline).unwrap(); + ZJITState::get_instance().exit_trampoline_with_counter = code_ptr; + } + + entry_trampoline + } + + /// Return true if zjit_state has been initialized + pub fn has_instance() -> bool { + matches!(unsafe { &ZJIT_STATE }, InitializationState::Enabled(_)) + } + + /// Get a mutable reference to the codegen globals instance + fn get_instance() -> &'static mut ZJITState { + if let InitializationState::Enabled(instance) = unsafe { &mut ZJIT_STATE } { + instance + } else { + panic!("ZJITState::get_instance called when ZJIT is not enabled") + } + } + + /// Get a mutable reference to the inline code block + pub fn get_code_block() -> &'static mut CodeBlock { + &mut ZJITState::get_instance().code_block + } + + /// Get a mutable reference to the invariants + pub fn get_invariants() -> &'static mut Invariants { + &mut ZJITState::get_instance().invariants + } + + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { + &ZJITState::get_instance().method_annotations + } + + /// Return true if successful compilation should be asserted + pub fn assert_compiles_enabled() -> bool { + ZJITState::get_instance().assert_compiles + } + + /// Start asserting successful compilation + pub fn enable_assert_compiles() { + let instance = ZJITState::get_instance(); + instance.assert_compiles = true; + } + + /// Get a mutable reference to counters for ZJIT stats + pub fn get_counters() -> &'static mut Counters { + &mut ZJITState::get_instance().counters + } + + /// Get a mutable reference to side-exit counters + pub fn get_exit_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().exit_counters + } + + /// Get a mutable reference to fallback counters + pub fn get_send_fallback_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().send_fallback_counters + } + + /// Get a mutable reference to full frame cfunc counter pointers + pub fn get_not_inlined_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().full_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to non-annotated cfunc counter pointers + pub fn get_not_annotated_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().not_annotated_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to ccall counter pointers + pub fn get_ccall_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().ccall_counter_pointers + } + + /// Was --zjit-save-compiled-iseqs specified? + pub fn should_log_compiled_iseqs() -> bool { + get_option!(log_compiled_iseqs).is_some() + } + + /// Log the name of a compiled ISEQ to the file specified in options.log_compiled_iseqs + pub fn log_compile(iseq_name: String) { + assert!(ZJITState::should_log_compiled_iseqs()); + let filename = get_option!(log_compiled_iseqs).as_ref().unwrap(); + use std::io::Write; + let mut file = match std::fs::OpenOptions::new().create(true).append(true).open(filename) { + Ok(f) => f, + Err(e) => { + eprintln!("ZJIT: Failed to create file '{}': {}", filename.display(), e); + return; + } + }; + if let Err(e) = writeln!(file, "{iseq_name}") { + eprintln!("ZJIT: Failed to write to file '{}': {}", filename.display(), e); + } + } + + /// Check if we are allowed to compile a given ISEQ based on --zjit-allowed-iseqs + pub fn can_compile_iseq(iseq: cruby::IseqPtr) -> bool { + if let Some(ref allowed_iseqs) = get_option!(allowed_iseqs) { + let name = cruby::iseq_get_location(iseq, 0); + allowed_iseqs.contains(&name) + } else { + true // If no restrictions, allow all ISEQs + } + } + + /// Return a code pointer to the side-exit trampoline + pub fn get_exit_trampoline() -> CodePtr { + ZJITState::get_instance().exit_trampoline + } + + /// Return a code pointer to the exit trampoline for function stubs + pub fn get_exit_trampoline_with_counter() -> CodePtr { + ZJITState::get_instance().exit_trampoline_with_counter + } + + /// Return a code pointer to the function stub hit trampoline + pub fn get_function_stub_hit_trampoline() -> CodePtr { + ZJITState::get_instance().function_stub_hit_trampoline + } + + /// Get a mutable reference to the ZJIT raw samples Vec + pub fn get_raw_samples() -> Option<&'static mut Vec<VALUE>> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.raw_samples) + } + + /// Get a mutable reference to the ZJIT line samples Vec. + pub fn get_line_samples() -> Option<&'static mut Vec<i32>> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.line_samples) + } + + /// Get number of skipped samples. + pub fn get_skipped_samples() -> Option<&'static mut usize> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.skipped_samples) + } + + /// Get number of skipped samples. + pub fn set_skipped_samples(n: usize) -> Option<()> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| el.skipped_samples = n) + } +} + +/// Initialize ZJIT at boot. This is called even if ZJIT is disabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_init(zjit_enabled: bool) { + use InitializationState::*; + + debug_assert!( + matches!(unsafe { &ZJIT_STATE }, Uninitialized), + "rb_zjit_init should only be called once during boot", + ); + + // Initialize IDs and method annotations. + // cruby_methods::init() must be called at boot, + // as cmes could have been re-defined after boot. + cruby::ids::init(); + + let method_annotations = cruby_methods::init(); + + unsafe { ZJIT_STATE = Initialized(method_annotations); } + + // If --zjit, enable ZJIT immediately + if zjit_enabled { + zjit_enable(); + } +} + +/// Enable ZJIT compilation. +fn zjit_enable() { + // TODO: call RubyVM::ZJIT::call_jit_hooks here + + // Catch panics to avoid UB for unwinding into C frames. + // See https://doc.rust-lang.org/nomicon/exception-safety.html + let result = std::panic::catch_unwind(|| { + // Initialize ZJIT states + let zjit_entry = ZJITState::init(); + + // Install a panic hook for ZJIT + rb_bug_panic_hook(); + + // Discard the instruction count for boot which we never compile + unsafe { rb_vm_insn_count = 0; } + + // ZJIT enabled and initialized successfully + assert!(unsafe{ rb_zjit_entry == null() }); + unsafe { rb_zjit_entry = zjit_entry; } + }); + + if result.is_err() { + println!("ZJIT: zjit_enable() panicked. Aborting."); + std::process::abort(); + } +} + +/// Enable ZJIT compilation, returning Qtrue if ZJIT was previously disabled +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_enable(_ec: EcPtr, _self: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || { + // Options would not have been initialized during boot if no flags were specified + rb_zjit_prepare_options(); + + // Initialize and enable ZJIT + zjit_enable(); + + // Add "+ZJIT" to RUBY_DESCRIPTION + unsafe { + unsafe extern "C" { + fn ruby_set_zjit_description(); + } + ruby_set_zjit_description(); + } + + Qtrue + }) +} + +/// Assert that any future ZJIT compilation will return a function pointer (not fail to compile) +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_assert_compiles(_ec: EcPtr, _self: VALUE) -> VALUE { + ZJITState::enable_assert_compiles(); + Qnil +} + +/// Call `rb_profile_frames` and write the result into buffers to be consumed by `rb_zjit_record_exit_stack`. +fn record_profiling_frames() -> (i32, Vec<VALUE>, Vec<i32>) { + // Stackprof uses a buffer of length 2048 when collating the frames into statistics. + // Since eventually the collected information will be used by Stackprof, collect only + // 2048 frames at a time. + // https://github.com/tmm1/stackprof/blob/5d832832e4afcb88521292d6dfad4a9af760ef7c/ext/stackprof/stackprof.c#L21 + const BUFF_LEN: usize = 2048; + + let mut frames_buffer = vec![VALUE(0_usize); BUFF_LEN]; + let mut lines_buffer = vec![0; BUFF_LEN]; + + let stack_length = unsafe { + rb_profile_frames( + 0, + BUFF_LEN as i32, + frames_buffer.as_mut_ptr(), + lines_buffer.as_mut_ptr(), + ) + }; + + // Trim at `stack_length` since anything past it is redundant + frames_buffer.truncate(stack_length as usize); + lines_buffer.truncate(stack_length as usize); + + (stack_length, frames_buffer, lines_buffer) +} + +/// Write samples in `frames_buffer` and `lines_buffer` from profiling into +/// `raw_samples` and `line_samples`. Also write opcode, number of frames, +/// and stack size to be consumed by Stackprof. +fn write_exit_stack_samples( + raw_samples: &'static mut Vec<VALUE>, + line_samples: &'static mut Vec<i32>, + frames_buffer: &[VALUE], + lines_buffer: &[i32], + stack_length: i32, + exit_pc: *const VALUE, +) { + raw_samples.push(VALUE(stack_length as usize)); + line_samples.push(stack_length); + + // Push frames and their lines in reverse order. + for i in (0..stack_length as usize).rev() { + raw_samples.push(frames_buffer[i]); + line_samples.push(lines_buffer[i]); + } + + // Get the opcode from instruction handler at exit PC. + let exit_opcode = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) }; + raw_samples.push(VALUE(exit_opcode as usize)); + // Push a dummy line number since we don't know where this insn is from. + line_samples.push(0); + + // Push number of times seen onto the stack. + raw_samples.push(VALUE(1usize)); + line_samples.push(1); +} + +fn try_increment_existing_stack( + raw_samples: &mut [VALUE], + line_samples: &mut [i32], + frames_buffer: &[VALUE], + stack_length: i32, + samples_length: usize, +) -> bool { + let prev_stack_len_index = raw_samples.len() - samples_length; + let prev_stack_len = i64::from(raw_samples[prev_stack_len_index]); + + if prev_stack_len == stack_length as i64 { + // Check if all stack lengths match and all frames are identical + let frames_match = (0..stack_length).all(|i| { + let current_frame = frames_buffer[stack_length as usize - 1 - i as usize]; + let prev_frame = raw_samples[prev_stack_len_index + i as usize + 1]; + current_frame == prev_frame + }); + + if frames_match { + let counter_idx = raw_samples.len() - 1; + let new_count = i64::from(raw_samples[counter_idx]) + 1; + + raw_samples[counter_idx] = VALUE(new_count as usize); + line_samples[counter_idx] = new_count as i32; + return true; + } + } + false +} + +/// Record a backtrace with ZJIT side exits +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_record_exit_stack(exit_pc: *const VALUE) { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + // When `trace_side_exits_sample_interval` is zero, then the feature is disabled. + if get_option!(trace_side_exits_sample_interval) != 0 { + // If `trace_side_exits_sample_interval` is set, then can safely unwrap + // both `get_skipped_samples` and `set_skipped_samples`. + let skipped_samples = *ZJITState::get_skipped_samples().unwrap(); + if skipped_samples < get_option!(trace_side_exits_sample_interval) { + // Skip sample and increment counter. + ZJITState::set_skipped_samples(skipped_samples + 1).unwrap(); + return; + } else { + ZJITState::set_skipped_samples(0).unwrap(); + } + } + + let (stack_length, frames_buffer, lines_buffer) = record_profiling_frames(); + + // Can safely unwrap since `trace_side_exits` must be true at this point + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + let zjit_line_samples = ZJITState::get_line_samples().unwrap(); + assert_eq!(zjit_raw_samples.len(), zjit_line_samples.len()); + + // Represents pushing the stack length, the instruction opcode, and the sample count. + const SAMPLE_METADATA_SIZE: usize = 3; + let samples_length = (stack_length as usize) + SAMPLE_METADATA_SIZE; + + // If zjit_raw_samples is greater than or equal to the current length of the samples + // we might have seen this stack trace previously. + if zjit_raw_samples.len() >= samples_length + && try_increment_existing_stack( + zjit_raw_samples, + zjit_line_samples, + &frames_buffer, + stack_length, + samples_length, + ) + { + return; + } + + write_exit_stack_samples( + zjit_raw_samples, + zjit_line_samples, + &frames_buffer, + &lines_buffer, + stack_length, + exit_pc, + ); +} + +/// Mark `raw_samples` so they can be used by rb_zjit_add_frame. +pub fn gc_mark_raw_samples() { + // Return if ZJIT is not enabled + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + let mut idx: size_t = 0; + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + + while idx < zjit_raw_samples.len() as size_t { + let num = zjit_raw_samples[idx as usize]; + let mut i = 0; + idx += 1; + + // Mark the zjit_raw_samples at the given index. These represent + // the data that needs to be GC'd which are the current frames. + while i < i32::from(num) { + unsafe { rb_gc_mark(zjit_raw_samples[idx as usize]); } + i += 1; + idx += 1; + } + + // Increase index for exit instruction. + idx += 1; + // Increase index for bookeeping value (number of times we've seen this + // row in a stack). + idx += 1; + } +} diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs new file mode 100644 index 0000000000..cf100dcda2 --- /dev/null +++ b/zjit/src/stats.rs @@ -0,0 +1,949 @@ +//! Counters and associated methods for events when ZJIT is run. + +use std::time::Instant; +use std::sync::atomic::Ordering; +use crate::options::OPTIONS; + +#[cfg(feature = "stats_allocator")] +#[path = "../../jit/src/lib.rs"] +mod jit; + +use crate::{cruby::*, hir::ParseError, options::get_option, state::{zjit_enabled_p, ZJITState}}; + +macro_rules! make_counters { + ( + default { + $($default_counter_name:ident,)+ + } + exit { + $($exit_counter_name:ident,)+ + } + dynamic_send { + $($dynamic_send_counter_name:ident,)+ + } + optimized_send { + $($optimized_send_counter_name:ident,)+ + } + dynamic_setivar { + $($dynamic_setivar_counter_name:ident,)+ + } + dynamic_getivar { + $($dynamic_getivar_counter_name:ident,)+ + } + dynamic_definedivar { + $($dynamic_definedivar_counter_name:ident,)+ + } + $($counter_name:ident,)+ + ) => { + /// Struct containing the counter values + #[derive(Default, Debug)] + pub struct Counters { + $(pub $default_counter_name: u64,)+ + $(pub $exit_counter_name: u64,)+ + $(pub $dynamic_send_counter_name: u64,)+ + $(pub $optimized_send_counter_name: u64,)+ + $(pub $dynamic_setivar_counter_name: u64,)+ + $(pub $dynamic_getivar_counter_name: u64,)+ + $(pub $dynamic_definedivar_counter_name: u64,)+ + $(pub $counter_name: u64,)+ + } + + /// Enum to represent a counter + #[allow(non_camel_case_types)] + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub enum Counter { + $($default_counter_name,)+ + $($exit_counter_name,)+ + $($dynamic_send_counter_name,)+ + $($optimized_send_counter_name,)+ + $($dynamic_setivar_counter_name,)+ + $($dynamic_getivar_counter_name,)+ + $($dynamic_definedivar_counter_name,)+ + $($counter_name,)+ + } + + impl Counter { + pub fn name(&self) -> &'static str { + match self { + $( Counter::$default_counter_name => stringify!($default_counter_name), )+ + $( Counter::$exit_counter_name => stringify!($exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => stringify!($dynamic_send_counter_name), )+ + $( Counter::$optimized_send_counter_name => stringify!($optimized_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => stringify!($dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => stringify!($dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => stringify!($dynamic_definedivar_counter_name), )+ + $( Counter::$counter_name => stringify!($counter_name), )+ + } + } + + pub fn get(name: &str) -> Option<Counter> { + match name { + $( stringify!($default_counter_name) => Some(Counter::$default_counter_name), )+ + $( stringify!($exit_counter_name) => Some(Counter::$exit_counter_name), )+ + $( stringify!($dynamic_send_counter_name) => Some(Counter::$dynamic_send_counter_name), )+ + $( stringify!($optimized_send_counter_name) => Some(Counter::$optimized_send_counter_name), )+ + $( stringify!($dynamic_setivar_counter_name) => Some(Counter::$dynamic_setivar_counter_name), )+ + $( stringify!($dynamic_getivar_counter_name) => Some(Counter::$dynamic_getivar_counter_name), )+ + $( stringify!($dynamic_definedivar_counter_name) => Some(Counter::$dynamic_definedivar_counter_name), )+ + $( stringify!($counter_name) => Some(Counter::$counter_name), )+ + _ => None, + } + } + } + + /// Map a counter to a pointer + pub fn counter_ptr(counter: Counter) -> *mut u64 { + let counters = $crate::state::ZJITState::get_counters(); + match counter { + $( Counter::$default_counter_name => std::ptr::addr_of_mut!(counters.$default_counter_name), )+ + $( Counter::$exit_counter_name => std::ptr::addr_of_mut!(counters.$exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_definedivar_counter_name), )+ + $( Counter::$optimized_send_counter_name => std::ptr::addr_of_mut!(counters.$optimized_send_counter_name), )+ + $( Counter::$counter_name => std::ptr::addr_of_mut!(counters.$counter_name), )+ + } + } + + /// List of counters that are available without --zjit-stats. + /// They are incremented only by `incr_counter()` and don't use `gen_incr_counter()`. + pub const DEFAULT_COUNTERS: &'static [Counter] = &[ + $( Counter::$default_counter_name, )+ + ]; + + /// List of other counters that are summed as side_exit_count. + pub const EXIT_COUNTERS: &'static [Counter] = &[ + $( Counter::$exit_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_send_count. + pub const DYNAMIC_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_send_counter_name, )+ + ]; + + /// List of other counters that are summed as optimized_send_count. + pub const OPTIMIZED_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$optimized_send_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_setivar_count. + pub const DYNAMIC_SETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_setivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_getivar_count. + pub const DYNAMIC_GETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_getivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_definedivar_count. + pub const DYNAMIC_DEFINEDIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_definedivar_counter_name, )+ + ]; + + /// List of other counters that are available only for --zjit-stats. + pub const OTHER_COUNTERS: &'static [Counter] = &[ + $( Counter::$counter_name, )+ + ]; + } +} + +// Declare all the counters we track +make_counters! { + // Default counters that are available without --zjit-stats + default { + compiled_iseq_count, + failed_iseq_count, + + compile_time_ns, + profile_time_ns, + gc_time_ns, + invalidation_time_ns, + } + + // Exit counters that are summed as side_exit_count + exit { + // exit_: Side exits reasons + exit_compile_error, + exit_unhandled_newarray_send_min, + exit_unhandled_newarray_send_hash, + exit_unhandled_newarray_send_pack, + exit_unhandled_newarray_send_pack_buffer, + exit_unhandled_newarray_send_unknown, + exit_unhandled_duparray_send, + exit_unhandled_tailcall, + exit_unhandled_splat, + exit_unhandled_kwarg, + exit_unhandled_block_arg, + exit_unknown_special_variable, + exit_unhandled_hir_insn, + exit_unhandled_yarv_insn, + exit_fixnum_add_overflow, + exit_fixnum_sub_overflow, + exit_fixnum_mult_overflow, + exit_fixnum_lshift_overflow, + exit_fixnum_mod_by_zero, + exit_fixnum_div_by_zero, + exit_box_fixnum_overflow, + exit_guard_type_failure, + exit_guard_type_not_failure, + exit_guard_bit_equals_failure, + exit_guard_int_equals_failure, + exit_guard_shape_failure, + exit_expandarray_failure, + exit_guard_not_frozen_failure, + exit_guard_not_shared_failure, + exit_guard_less_failure, + exit_guard_greater_eq_failure, + exit_guard_super_method_entry, + exit_patchpoint_bop_redefined, + exit_patchpoint_method_redefined, + exit_patchpoint_stable_constant_names, + exit_patchpoint_no_tracepoint, + exit_patchpoint_no_ep_escape, + exit_patchpoint_single_ractor_mode, + exit_patchpoint_no_singleton_class, + exit_callee_side_exit, + exit_obj_to_string_fallback, + exit_interrupt, + exit_stackoverflow, + exit_block_param_proxy_modified, + exit_block_param_proxy_not_iseq_or_ifunc, + exit_too_many_keyword_parameters, + } + + // Send fallback counters that are summed as dynamic_send_count + dynamic_send { + // send_fallback_: Fallback reasons for send-ish instructions + send_fallback_send_without_block_polymorphic, + send_fallback_send_without_block_megamorphic, + send_fallback_send_without_block_no_profiles, + send_fallback_send_without_block_cfunc_not_variadic, + send_fallback_send_without_block_cfunc_array_variadic, + send_fallback_send_without_block_not_optimized_method_type, + send_fallback_send_without_block_not_optimized_method_type_optimized, + send_fallback_send_without_block_not_optimized_need_permission, + send_fallback_too_many_args_for_lir, + send_fallback_send_without_block_bop_redefined, + send_fallback_send_without_block_operands_not_fixnum, + send_fallback_send_without_block_direct_keyword_mismatch, + send_fallback_send_without_block_direct_keyword_count_mismatch, + send_fallback_send_without_block_direct_missing_keyword, + send_fallback_send_without_block_direct_too_many_keywords, + send_fallback_send_polymorphic, + send_fallback_send_megamorphic, + send_fallback_send_no_profiles, + send_fallback_send_not_optimized_method_type, + send_fallback_send_not_optimized_need_permission, + send_fallback_ccall_with_frame_too_many_args, + send_fallback_argc_param_mismatch, + // The call has at least one feature on the caller or callee side + // that the optimizer does not support. + send_fallback_one_or_more_complex_arg_pass, + // Caller has keyword arguments but callee doesn't expect them. + send_fallback_unexpected_keyword_args, + // Singleton class previously created for receiver class. + send_fallback_singleton_class_seen, + send_fallback_bmethod_non_iseq_proc, + send_fallback_obj_to_string_not_string, + send_fallback_send_cfunc_variadic, + send_fallback_send_cfunc_array_variadic, + send_fallback_super_call_with_block, + send_fallback_super_class_not_found, + send_fallback_super_complex_args_pass, + send_fallback_super_fallback_no_profile, + send_fallback_super_not_optimized_method_type, + send_fallback_super_polymorphic, + send_fallback_super_target_not_found, + send_fallback_super_target_complex_args_pass, + send_fallback_cannot_send_direct, + send_fallback_uncategorized, + } + + // Optimized send counters that are summed as optimized_send_count + optimized_send { + iseq_optimized_send_count, + inline_cfunc_optimized_send_count, + inline_iseq_optimized_send_count, + non_variadic_cfunc_optimized_send_count, + variadic_cfunc_optimized_send_count, + } + + // Ivar fallback counters that are summed as dynamic_setivar_count + dynamic_setivar { + // setivar_fallback_: Fallback reasons for dynamic setivar instructions + setivar_fallback_not_monomorphic, + setivar_fallback_immediate, + setivar_fallback_not_t_object, + setivar_fallback_too_complex, + setivar_fallback_frozen, + setivar_fallback_shape_transition, + setivar_fallback_new_shape_too_complex, + setivar_fallback_new_shape_needs_extension, + } + + // Ivar fallback counters that are summed as dynamic_getivar_count + dynamic_getivar { + // getivar_fallback_: Fallback reasons for dynamic getivar instructions + getivar_fallback_not_monomorphic, + getivar_fallback_immediate, + getivar_fallback_not_t_object, + getivar_fallback_too_complex, + } + + // Ivar fallback counters that are summed as dynamic_definedivar_count + dynamic_definedivar { + // definedivar_fallback_: Fallback reasons for dynamic definedivar instructions + definedivar_fallback_not_monomorphic, + definedivar_fallback_immediate, + definedivar_fallback_not_t_object, + definedivar_fallback_too_complex, + } + + // compile_error_: Compile error reasons + compile_error_iseq_version_limit_reached, + compile_error_iseq_stack_too_large, + compile_error_exception_handler, + compile_error_out_of_memory, + compile_error_label_linking_failure, + compile_error_jit_to_jit_optional, + compile_error_register_spill_on_ccall, + compile_error_register_spill_on_alloc, + compile_error_parse_stack_underflow, + compile_error_parse_malformed_iseq, + compile_error_parse_not_allowed, + compile_error_validation_block_has_no_terminator, + compile_error_validation_terminator_not_at_end, + compile_error_validation_mismatched_block_arity, + compile_error_validation_jump_target_not_in_rpo, + compile_error_validation_operand_not_defined, + compile_error_validation_duplicate_instruction, + compile_error_validation_type_check_failure, + compile_error_validation_misc_validation_error, + + // unhandled_hir_insn_: Unhandled HIR instructions + unhandled_hir_insn_array_max, + unhandled_hir_insn_fixnum_div, + unhandled_hir_insn_throw, + unhandled_hir_insn_invokebuiltin, + unhandled_hir_insn_unknown, + + // The number of times YARV instructions are executed on JIT code + zjit_insn_count, + + // Method call def_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_iseq, + unspecialized_send_without_block_def_type_cfunc, + unspecialized_send_without_block_def_type_attrset, + unspecialized_send_without_block_def_type_ivar, + unspecialized_send_without_block_def_type_bmethod, + unspecialized_send_without_block_def_type_zsuper, + unspecialized_send_without_block_def_type_alias, + unspecialized_send_without_block_def_type_undef, + unspecialized_send_without_block_def_type_not_implemented, + unspecialized_send_without_block_def_type_optimized, + unspecialized_send_without_block_def_type_missing, + unspecialized_send_without_block_def_type_refined, + unspecialized_send_without_block_def_type_null, + + // Method call optimized_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_optimized_send, + unspecialized_send_without_block_def_type_optimized_call, + unspecialized_send_without_block_def_type_optimized_block_call, + unspecialized_send_without_block_def_type_optimized_struct_aref, + unspecialized_send_without_block_def_type_optimized_struct_aset, + + // Method call def_type related to send fallback to dynamic dispatch + unspecialized_send_def_type_iseq, + unspecialized_send_def_type_cfunc, + unspecialized_send_def_type_attrset, + unspecialized_send_def_type_ivar, + unspecialized_send_def_type_bmethod, + unspecialized_send_def_type_zsuper, + unspecialized_send_def_type_alias, + unspecialized_send_def_type_undef, + unspecialized_send_def_type_not_implemented, + unspecialized_send_def_type_optimized, + unspecialized_send_def_type_missing, + unspecialized_send_def_type_refined, + unspecialized_send_def_type_null, + + // Super call def_type related to send fallback to dynamic dispatch + unspecialized_super_def_type_iseq, + unspecialized_super_def_type_cfunc, + unspecialized_super_def_type_attrset, + unspecialized_super_def_type_ivar, + unspecialized_super_def_type_bmethod, + unspecialized_super_def_type_zsuper, + unspecialized_super_def_type_alias, + unspecialized_super_def_type_undef, + unspecialized_super_def_type_not_implemented, + unspecialized_super_def_type_optimized, + unspecialized_super_def_type_missing, + unspecialized_super_def_type_refined, + unspecialized_super_def_type_null, + + // Unsupported parameter features + complex_arg_pass_param_rest, + complex_arg_pass_param_post, + complex_arg_pass_param_kwrest, + complex_arg_pass_param_block, + complex_arg_pass_param_forwardable, + + // Unsupported caller side features + complex_arg_pass_caller_splat, + complex_arg_pass_caller_blockarg, + complex_arg_pass_caller_kwarg, + complex_arg_pass_caller_kw_splat, + complex_arg_pass_caller_tailcall, + complex_arg_pass_caller_super, + complex_arg_pass_caller_zsuper, + complex_arg_pass_caller_forwarding, + + // Writes to the VM frame + vm_write_pc_count, + vm_write_sp_count, + vm_write_locals_count, + vm_write_stack_count, + vm_write_to_parent_iseq_local_count, + vm_read_from_parent_iseq_local_count, + // TODO(max): Implement + // vm_reify_stack_count, + + // The number of times we ran a dynamic check + guard_type_count, + guard_shape_count, + + invokeblock_handler_monomorphic_iseq, + invokeblock_handler_monomorphic_ifunc, + invokeblock_handler_monomorphic_other, + invokeblock_handler_polymorphic, + invokeblock_handler_megamorphic, + invokeblock_handler_no_profiles, +} + +/// Increase a counter by a specified amount +pub fn incr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr += amount; } +} + +/// Decrease a counter by a specified amount +pub fn decr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr -= amount; } +} + +/// Increment a counter by its identifier +macro_rules! incr_counter { + ($counter_name:ident) => { + $crate::stats::incr_counter_by($crate::stats::Counter::$counter_name, 1) + } +} +pub(crate) use incr_counter; + +/// The number of side exits from each YARV instruction +pub type InsnCounters = [u64; VM_INSTRUCTION_SIZE as usize]; + +/// Return a raw pointer to the exit counter for a given YARV opcode +pub fn exit_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let exit_counters = ZJITState::get_exit_counters(); + unsafe { exit_counters.get_unchecked_mut(opcode as usize) } +} + +/// Return a raw pointer to the fallback counter for a given YARV opcode +pub fn send_fallback_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let fallback_counters = ZJITState::get_send_fallback_counters(); + unsafe { fallback_counters.get_unchecked_mut(opcode as usize) } +} + +/// Reason why ZJIT failed to produce any JIT code +#[derive(Clone, Debug, PartialEq)] +pub enum CompileError { + IseqVersionLimitReached, + IseqStackTooLarge, + ExceptionHandler, + OutOfMemory, + ParseError(ParseError), + /// When a ZJIT function is too large, the branches may have + /// offsets that don't fit in one instruction. We error in + /// error that case. + LabelLinkingFailure, +} + +/// Return a raw pointer to the exit counter for a given CompileError +pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { + use crate::hir::ParseError::*; + use crate::hir::ValidationError::*; + use crate::stats::CompileError::*; + use crate::stats::Counter::*; + match compile_error { + IseqVersionLimitReached => compile_error_iseq_version_limit_reached, + IseqStackTooLarge => compile_error_iseq_stack_too_large, + ExceptionHandler => compile_error_exception_handler, + OutOfMemory => compile_error_out_of_memory, + LabelLinkingFailure => compile_error_label_linking_failure, + ParseError(parse_error) => match parse_error { + StackUnderflow(_) => compile_error_parse_stack_underflow, + MalformedIseq(_) => compile_error_parse_malformed_iseq, + NotAllowed => compile_error_parse_not_allowed, + Validation(validation) => match validation { + BlockHasNoTerminator(_) => compile_error_validation_block_has_no_terminator, + TerminatorNotAtEnd(_, _, _) => compile_error_validation_terminator_not_at_end, + MismatchedBlockArity(_, _, _) => compile_error_validation_mismatched_block_arity, + JumpTargetNotInRPO(_) => compile_error_validation_jump_target_not_in_rpo, + OperandNotDefined(_, _, _) => compile_error_validation_operand_not_defined, + DuplicateInstruction(_, _) => compile_error_validation_duplicate_instruction, + MismatchedOperandType(..) => compile_error_validation_type_check_failure, + MiscValidationError(..) => compile_error_validation_misc_validation_error, + }, + } + } +} + +pub fn exit_counter_for_unhandled_hir_insn(insn: &crate::hir::Insn) -> Counter { + use crate::hir::Insn::*; + use crate::stats::Counter::*; + match insn { + ArrayMax { .. } => unhandled_hir_insn_array_max, + FixnumDiv { .. } => unhandled_hir_insn_fixnum_div, + Throw { .. } => unhandled_hir_insn_throw, + InvokeBuiltin { .. } => unhandled_hir_insn_invokebuiltin, + _ => unhandled_hir_insn_unknown, + } +} + +pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { + use crate::hir::SideExitReason::*; + use crate::hir::CallType::*; + use crate::hir::Invariant; + use crate::stats::Counter::*; + match reason { + UnhandledNewarraySend(send_type) => match send_type { + VM_OPT_NEWARRAY_SEND_MIN => exit_unhandled_newarray_send_min, + VM_OPT_NEWARRAY_SEND_HASH => exit_unhandled_newarray_send_hash, + VM_OPT_NEWARRAY_SEND_PACK => exit_unhandled_newarray_send_pack, + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => exit_unhandled_newarray_send_pack_buffer, + _ => exit_unhandled_newarray_send_unknown, + } + UnhandledDuparraySend(_) => exit_unhandled_duparray_send, + UnhandledCallType(Tailcall) => exit_unhandled_tailcall, + UnhandledCallType(Splat) => exit_unhandled_splat, + UnhandledCallType(Kwarg) => exit_unhandled_kwarg, + UnknownSpecialVariable(_) => exit_unknown_special_variable, + UnhandledHIRInsn(_) => exit_unhandled_hir_insn, + UnhandledYARVInsn(_) => exit_unhandled_yarv_insn, + UnhandledBlockArg => exit_unhandled_block_arg, + FixnumAddOverflow => exit_fixnum_add_overflow, + FixnumSubOverflow => exit_fixnum_sub_overflow, + FixnumMultOverflow => exit_fixnum_mult_overflow, + FixnumLShiftOverflow => exit_fixnum_lshift_overflow, + FixnumModByZero => exit_fixnum_mod_by_zero, + FixnumDivByZero => exit_fixnum_div_by_zero, + BoxFixnumOverflow => exit_box_fixnum_overflow, + GuardType(_) => exit_guard_type_failure, + GuardTypeNot(_) => exit_guard_type_not_failure, + GuardShape(_) => exit_guard_shape_failure, + ExpandArray => exit_expandarray_failure, + GuardNotFrozen => exit_guard_not_frozen_failure, + GuardNotShared => exit_guard_not_shared_failure, + GuardLess => exit_guard_less_failure, + GuardGreaterEq => exit_guard_greater_eq_failure, + GuardSuperMethodEntry => exit_guard_super_method_entry, + CalleeSideExit => exit_callee_side_exit, + ObjToStringFallback => exit_obj_to_string_fallback, + Interrupt => exit_interrupt, + StackOverflow => exit_stackoverflow, + BlockParamProxyModified => exit_block_param_proxy_modified, + BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + TooManyKeywordParameters => exit_too_many_keyword_parameters, + PatchPoint(Invariant::BOPRedefined { .. }) + => exit_patchpoint_bop_redefined, + PatchPoint(Invariant::MethodRedefined { .. }) + => exit_patchpoint_method_redefined, + PatchPoint(Invariant::StableConstantNames { .. }) + => exit_patchpoint_stable_constant_names, + PatchPoint(Invariant::NoTracePoint) + => exit_patchpoint_no_tracepoint, + PatchPoint(Invariant::NoEPEscape(_)) + => exit_patchpoint_no_ep_escape, + PatchPoint(Invariant::SingleRactorMode) + => exit_patchpoint_single_ractor_mode, + PatchPoint(Invariant::NoSingletonClass { .. }) + => exit_patchpoint_no_singleton_class, + } +} + +pub fn exit_counter_ptr(reason: crate::hir::SideExitReason) -> *mut u64 { + let counter = side_exit_counter(reason); + counter_ptr(counter) +} + +pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter { + use crate::hir::SendFallbackReason::*; + use crate::stats::Counter::*; + match reason { + SendWithoutBlockPolymorphic => send_fallback_send_without_block_polymorphic, + SendWithoutBlockMegamorphic => send_fallback_send_without_block_megamorphic, + SendWithoutBlockNoProfiles => send_fallback_send_without_block_no_profiles, + SendWithoutBlockCfuncNotVariadic => send_fallback_send_without_block_cfunc_not_variadic, + SendWithoutBlockCfuncArrayVariadic => send_fallback_send_without_block_cfunc_array_variadic, + SendWithoutBlockNotOptimizedMethodType(_) => send_fallback_send_without_block_not_optimized_method_type, + SendWithoutBlockNotOptimizedMethodTypeOptimized(_) + => send_fallback_send_without_block_not_optimized_method_type_optimized, + SendWithoutBlockNotOptimizedNeedPermission + => send_fallback_send_without_block_not_optimized_need_permission, + TooManyArgsForLir => send_fallback_too_many_args_for_lir, + SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, + SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, + SendWithoutBlockDirectKeywordMismatch => send_fallback_send_without_block_direct_keyword_mismatch, + SendWithoutBlockDirectKeywordCountMismatch=> send_fallback_send_without_block_direct_keyword_count_mismatch, + SendWithoutBlockDirectMissingKeyword => send_fallback_send_without_block_direct_missing_keyword, + SendWithoutBlockDirectTooManyKeywords => send_fallback_send_without_block_direct_too_many_keywords, + SendPolymorphic => send_fallback_send_polymorphic, + SendMegamorphic => send_fallback_send_megamorphic, + SendNoProfiles => send_fallback_send_no_profiles, + SendCfuncVariadic => send_fallback_send_cfunc_variadic, + SendCfuncArrayVariadic => send_fallback_send_cfunc_array_variadic, + ComplexArgPass => send_fallback_one_or_more_complex_arg_pass, + UnexpectedKeywordArgs => send_fallback_unexpected_keyword_args, + SingletonClassSeen => send_fallback_singleton_class_seen, + ArgcParamMismatch => send_fallback_argc_param_mismatch, + BmethodNonIseqProc => send_fallback_bmethod_non_iseq_proc, + SendNotOptimizedMethodType(_) => send_fallback_send_not_optimized_method_type, + SendNotOptimizedNeedPermission => send_fallback_send_not_optimized_need_permission, + CCallWithFrameTooManyArgs => send_fallback_ccall_with_frame_too_many_args, + ObjToStringNotString => send_fallback_obj_to_string_not_string, + SuperCallWithBlock => send_fallback_super_call_with_block, + SuperClassNotFound => send_fallback_super_class_not_found, + SuperComplexArgsPass => send_fallback_super_complex_args_pass, + SuperNoProfiles => send_fallback_super_fallback_no_profile, + SuperNotOptimizedMethodType(_) => send_fallback_super_not_optimized_method_type, + SuperPolymorphic => send_fallback_super_polymorphic, + SuperTargetNotFound => send_fallback_super_target_not_found, + SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + Uncategorized(_) => send_fallback_uncategorized, + } +} + +pub fn send_without_block_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_without_block_def_type_iseq, + Cfunc => unspecialized_send_without_block_def_type_cfunc, + Attrset => unspecialized_send_without_block_def_type_attrset, + Ivar => unspecialized_send_without_block_def_type_ivar, + Bmethod => unspecialized_send_without_block_def_type_bmethod, + Zsuper => unspecialized_send_without_block_def_type_zsuper, + Alias => unspecialized_send_without_block_def_type_alias, + Undefined => unspecialized_send_without_block_def_type_undef, + NotImplemented => unspecialized_send_without_block_def_type_not_implemented, + Optimized => unspecialized_send_without_block_def_type_optimized, + Missing => unspecialized_send_without_block_def_type_missing, + Refined => unspecialized_send_without_block_def_type_refined, + Null => unspecialized_send_without_block_def_type_null, + } +} + +pub fn send_without_block_fallback_counter_for_optimized_method_type(method_type: crate::hir::OptimizedMethodType) -> Counter { + use crate::hir::OptimizedMethodType::*; + use crate::stats::Counter::*; + + match method_type { + Send => unspecialized_send_without_block_def_type_optimized_send, + Call => unspecialized_send_without_block_def_type_optimized_call, + BlockCall => unspecialized_send_without_block_def_type_optimized_block_call, + StructAref => unspecialized_send_without_block_def_type_optimized_struct_aref, + StructAset => unspecialized_send_without_block_def_type_optimized_struct_aset, + } +} + +pub fn send_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_def_type_iseq, + Cfunc => unspecialized_send_def_type_cfunc, + Attrset => unspecialized_send_def_type_attrset, + Ivar => unspecialized_send_def_type_ivar, + Bmethod => unspecialized_send_def_type_bmethod, + Zsuper => unspecialized_send_def_type_zsuper, + Alias => unspecialized_send_def_type_alias, + Undefined => unspecialized_send_def_type_undef, + NotImplemented => unspecialized_send_def_type_not_implemented, + Optimized => unspecialized_send_def_type_optimized, + Missing => unspecialized_send_def_type_missing, + Refined => unspecialized_send_def_type_refined, + Null => unspecialized_send_def_type_null, + } +} + +pub fn send_fallback_counter_for_super_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_super_def_type_iseq, + Cfunc => unspecialized_super_def_type_cfunc, + Attrset => unspecialized_super_def_type_attrset, + Ivar => unspecialized_super_def_type_ivar, + Bmethod => unspecialized_super_def_type_bmethod, + Zsuper => unspecialized_super_def_type_zsuper, + Alias => unspecialized_super_def_type_alias, + Undefined => unspecialized_super_def_type_undef, + NotImplemented => unspecialized_super_def_type_not_implemented, + Optimized => unspecialized_super_def_type_optimized, + Missing => unspecialized_super_def_type_missing, + Refined => unspecialized_super_def_type_refined, + Null => unspecialized_super_def_type_null, + } +} + +/// Primitive called in zjit.rb. Zero out all the counters. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_reset_stats_bang(_ec: EcPtr, _self: VALUE) -> VALUE { + let counters = ZJITState::get_counters(); + let exit_counters = ZJITState::get_exit_counters(); + + // Reset all counters to zero + *counters = Counters::default(); + + // Reset exit counters for YARV instructions + exit_counters.as_mut_slice().fill(0); + + Qnil +} + +/// Return a Hash object that contains ZJIT statistics +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE { + if !zjit_enabled_p() { + return Qnil; + } + + macro_rules! set_stat { + ($hash:ident, $key:expr, $value:expr) => { + let key = rust_str_to_sym($key); + if key == target_key { + return $value; + } else if $hash != Qnil { + #[allow(unused_unsafe)] + unsafe { rb_hash_aset($hash, key, $value); } + } + }; + } + + macro_rules! set_stat_usize { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, VALUE::fixnum_from_usize($value as usize)) + } + } + + macro_rules! set_stat_f64 { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, unsafe { rb_float_new($value) }) + } + } + + let hash = if target_key.nil_p() { + unsafe { rb_hash_new() } + } else { + Qnil + }; + + // Set default counters + for &counter in DEFAULT_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Memory usage stats + let code_region_bytes = ZJITState::get_code_block().mapped_region_size(); + set_stat_usize!(hash, "code_region_bytes", code_region_bytes); + set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes()); + set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes()); + + // End of default stats. Every counter beyond this is provided only for --zjit-stats. + if !get_option!(stats) { + return hash; + } + + // Set other stats-only counters + for &counter in OTHER_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Set side-exit counters for each SideExitReason + let mut side_exit_count = 0; + for &counter in EXIT_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + side_exit_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "side_exit_count", side_exit_count); + + // Set side-exit counters for UnhandledYARVInsn + let exit_counters = ZJITState::get_exit_counters(); + for (op_idx, count) in exit_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "unhandled_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Set send fallback counters for each DynamicSendReason + let mut dynamic_send_count = 0; + for &counter in DYNAMIC_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_send_count", dynamic_send_count); + + // Set optimized send counters + let mut optimized_send_count = 0; + for &counter in OPTIMIZED_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + optimized_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "optimized_send_count", optimized_send_count); + set_stat_usize!(hash, "send_count", dynamic_send_count + optimized_send_count); + + // Set send fallback counters for each setivar fallback reason + let mut dynamic_setivar_count = 0; + for &counter in DYNAMIC_SETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_setivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_setivar_count", dynamic_setivar_count); + + // Set send fallback counters for each getivar fallback reason + let mut dynamic_getivar_count = 0; + for &counter in DYNAMIC_GETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_getivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_getivar_count", dynamic_getivar_count); + + // Set send fallback counters for each definedivar fallback reason + let mut dynamic_definedivar_count = 0; + for &counter in DYNAMIC_DEFINEDIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_definedivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_definedivar_count", dynamic_definedivar_count); + + // Set send fallback counters for Uncategorized + let send_fallback_counters = ZJITState::get_send_fallback_counters(); + for (op_idx, count) in send_fallback_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "uncategorized_fallback_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Only ZJIT_STATS builds support rb_vm_insn_count + if unsafe { rb_vm_insn_count } > 0 { + let vm_insn_count = unsafe { rb_vm_insn_count }; + set_stat_usize!(hash, "vm_insn_count", vm_insn_count); + + let zjit_insn_count = ZJITState::get_counters().zjit_insn_count; + let total_insn_count = vm_insn_count + zjit_insn_count; + set_stat_usize!(hash, "total_insn_count", total_insn_count); + + set_stat_f64!(hash, "ratio_in_zjit", 100.0 * zjit_insn_count as f64 / total_insn_count as f64); + } + + // Set not inlined cfunc counters + let not_inlined_cfuncs = ZJITState::get_not_inlined_cfunc_counter_pointers(); + for (signature, counter) in not_inlined_cfuncs.iter() { + let key_string = format!("not_inlined_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set not annotated cfunc counters + let not_annotated_cfuncs = ZJITState::get_not_annotated_cfunc_counter_pointers(); + for (signature, counter) in not_annotated_cfuncs.iter() { + let key_string = format!("not_annotated_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set ccall counters + let ccall = ZJITState::get_ccall_counter_pointers(); + for (signature, counter) in ccall.iter() { + let key_string = format!("ccall_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + hash +} + +/// Measure the time taken by func() and add that to zjit_compile_time. +pub fn with_time_stat<F, R>(counter: Counter, func: F) -> R where F: FnOnce() -> R { + let start = Instant::now(); + let ret = func(); + let nanos = Instant::now().duration_since(start).as_nanos(); + incr_counter_by(counter, nanos as u64); + ret +} + +/// The number of bytes ZJIT has allocated on the Rust heap. +pub fn zjit_alloc_bytes() -> usize { + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) +} + +/// Struct of arrays for --zjit-trace-exits. +#[derive(Default)] +pub struct SideExitLocations { + /// Control frames of method entries. + pub raw_samples: Vec<VALUE>, + /// Line numbers of the iseq caller. + pub line_samples: Vec<i32>, + /// Skipped samples + pub skipped_samples: usize +} + +/// Primitive called in zjit.rb +/// +/// Check if trace_exits generation is enabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_trace_exit_locations_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.trace_side_exits.is_some()) { + Qtrue + } else { + Qfalse + } +} + +/// Call the C function to parse the raw_samples and line_samples +/// into raw, lines, and frames hash for RubyVM::YJIT.exit_locations. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_exit_locations(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return Qnil; + } + + // Can safely unwrap since `trace_side_exits` must be true at this point + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + let zjit_line_samples = ZJITState::get_line_samples().unwrap(); + + assert_eq!(zjit_raw_samples.len(), zjit_line_samples.len()); + + // zjit_raw_samples and zjit_line_samples are the same length so + // pass only one of the lengths in the C function. + let samples_len = zjit_raw_samples.len() as i32; + + unsafe { + rb_zjit_exit_locations_dict( + zjit_raw_samples.as_mut_ptr(), + zjit_line_samples.as_mut_ptr(), + samples_len + ) + } +} diff --git a/zjit/src/ttycolors.rs b/zjit/src/ttycolors.rs new file mode 100644 index 0000000000..f325772431 --- /dev/null +++ b/zjit/src/ttycolors.rs @@ -0,0 +1,31 @@ +use std::io::IsTerminal; + +pub fn stdout_supports_colors() -> bool { + std::io::stdout().is_terminal() +} + +#[cfg_attr(not(feature = "disasm"), allow(dead_code))] +#[derive(Copy, Clone, Debug)] +pub struct TerminalColor { + pub bold_begin: &'static str, + pub bold_end: &'static str, +} + +pub static TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "\x1b[1m", + bold_end: "\x1b[22m", +}; + +pub static NON_TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "", + bold_end: "", +}; + +/// Terminal escape codes for colors, font weight, etc. Only enabled if stdout is a TTY. +pub fn get_colors() -> &'static TerminalColor { + if stdout_supports_colors() { + &TTY_TERMINAL_COLOR + } else { + &NON_TTY_TERMINAL_COLOR + } +} diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs new file mode 100644 index 0000000000..9741a7b138 --- /dev/null +++ b/zjit/src/virtualmem.rs @@ -0,0 +1,488 @@ +//! Memory management stuff for ZJIT's code storage. Deals with virtual memory. +// I'm aware that there is an experiment in Rust Nightly right now for to see if banning +// usize->pointer casts is viable. It seems like a lot of work for us to participate for not much +// benefit. + +use std::ptr::NonNull; +use crate::cruby::*; +use crate::stats::zjit_alloc_bytes; + +pub type VirtualMem = VirtualMemory<sys::SystemAllocator>; + +/// Memory for generated executable machine code. When not testing, we reserve address space for +/// the entire region upfront and map physical memory into the reserved address space as needed. On +/// Linux, this is basically done using an `mmap` with `PROT_NONE` upfront and gradually using +/// `mprotect` with `PROT_READ|PROT_WRITE` as needed. The WIN32 equivalent seems to be +/// `VirtualAlloc` with `MEM_RESERVE` then later with `MEM_COMMIT`. +/// +/// This handles ["W^X"](https://en.wikipedia.org/wiki/W%5EX) semi-automatically. Writes +/// are always accepted and once writes are done a call to [Self::mark_all_executable] makes +/// the code in the region executable. +pub struct VirtualMemory<A: Allocator> { + /// Location of the virtual memory region. + region_start: NonNull<u8>, + + /// Size of this virtual memory region in bytes. + region_size_bytes: usize, + + /// mapped_region_bytes + zjit_alloc_size may not increase beyond this limit. + memory_limit_bytes: Option<usize>, + + /// Number of bytes per "page", memory protection permission can only be controlled at this + /// granularity. + page_size_bytes: usize, + + /// Number of bytes that have we have allocated physical memory for starting at + /// [Self::region_start]. + mapped_region_bytes: usize, + + /// Keep track of the address of the last written to page. + /// Used for changing protection to implement W^X. + current_write_page: Option<usize>, + + /// Zero size member for making syscalls to get physical memory during normal operation. + /// When testing this owns some memory. + allocator: A, +} + +/// Groups together the two syscalls to get get new physical memory and to change +/// memory protection. See [VirtualMemory] for details. +pub trait Allocator { + #[must_use] + fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool; + + fn mark_executable(&mut self, ptr: *const u8, size: u32); + + fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool; +} + +/// Pointer into a [VirtualMemory] represented as an offset from the base. +/// Note: there is no NULL constant for [CodePtr]. You should use `Option<CodePtr>` instead. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Debug)] +#[repr(C, packed)] +pub struct CodePtr(u32); + +impl CodePtr { + /// Advance the CodePtr. Can return a dangling pointer. + pub fn add_bytes(self, bytes: usize) -> Self { + let CodePtr(raw) = self; + let bytes: u32 = bytes.try_into().unwrap(); + CodePtr(raw + bytes) + } + + /// Subtract bytes from the CodePtr + pub fn sub_bytes(self, bytes: usize) -> Self { + let CodePtr(raw) = self; + let bytes: u32 = bytes.try_into().unwrap(); + CodePtr(raw.saturating_sub(bytes)) + } + + /// Note that the raw pointer might be dangling if there hasn't + /// been any writes to it through the [VirtualMemory] yet. + pub fn raw_ptr(self, base: &impl CodePtrBase) -> *const u8 { + let CodePtr(offset) = self; + base.base_ptr().as_ptr().wrapping_add(offset as usize) + } + + /// Get the address of the code pointer. + pub fn raw_addr(self, base: &impl CodePtrBase) -> usize { + self.raw_ptr(base).addr() + } + + /// Get the offset component for the code pointer. Useful finding the distance between two + /// code pointers that share the same [VirtualMem]. + pub fn as_offset(self) -> i64 { + let CodePtr(offset) = self; + offset.into() + } +} + +/// Errors that can happen when writing to [VirtualMemory] +#[derive(Debug, PartialEq)] +pub enum WriteError { + OutOfBounds, + FailedPageMapping, +} + +use WriteError::*; + +impl VirtualMem { + /// Allocate a VirtualMem insntace with a requested size + pub fn alloc(exec_mem_bytes: usize, mem_bytes: Option<usize>) -> Self { + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_bytes as u32) }; + + // Memory protection syscalls need page-aligned addresses, so check it here. Assuming + // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the + // page size in bytes is a power of two 2¹⁹ or smaller. This is because the user + // requested size is half of mem_option × 2²⁰ as it's in MiB. + // + // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB + // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. + let page_size = unsafe { rb_jit_get_page_size() }; + assert_eq!( + virt_block as usize % page_size as usize, 0, + "Start of virtual address block should be page-aligned", + ); + + Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes) + } +} + +impl<A: Allocator> VirtualMemory<A> { + /// Bring a part of the address space under management. + pub fn new( + allocator: A, + page_size: u32, + virt_region_start: NonNull<u8>, + region_size_bytes: usize, + memory_limit_bytes: Option<usize>, + ) -> Self { + assert_ne!(0, page_size); + let page_size_bytes = page_size as usize; + + Self { + region_start: virt_region_start, + region_size_bytes, + memory_limit_bytes, + page_size_bytes, + mapped_region_bytes: 0, + current_write_page: None, + allocator, + } + } + + /// Return the start of the region as a raw pointer. Note that it could be a dangling + /// pointer so be careful dereferencing it. + pub fn start_ptr(&self) -> CodePtr { + CodePtr(0) + } + + pub fn mapped_end_ptr(&self) -> CodePtr { + self.start_ptr().add_bytes(self.mapped_region_bytes) + } + + pub fn virtual_end_ptr(&self) -> CodePtr { + self.start_ptr().add_bytes(self.region_size_bytes) + } + + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mapped_region_bytes + } + + /// Size of the region in bytes where writes could be attempted. + pub fn virtual_region_size(&self) -> usize { + self.region_size_bytes + } + + /// The granularity at which we can control memory permission. + /// On Linux, this is the page size that mmap(2) talks about. + pub fn system_page_size(&self) -> usize { + self.page_size_bytes + } + + /// Write a single byte. The first write to a page makes it readable. + pub fn write_byte(&mut self, write_ptr: CodePtr, byte: u8) -> Result<(), WriteError> { + let page_size = self.page_size_bytes; + let raw: *mut u8 = write_ptr.raw_ptr(self) as *mut u8; + let page_addr = (raw as usize / page_size) * page_size; + + if self.current_write_page == Some(page_addr) { + // Writing within the last written to page, nothing to do + } else { + // Switching to a different and potentially new page + let start = self.region_start.as_ptr(); + let mapped_region_end = start.wrapping_add(self.mapped_region_bytes); + let whole_region_end = start.wrapping_add(self.region_size_bytes); + let alloc = &mut self.allocator; + + // Ignore zjit_alloc_size() if self.memory_limit_bytes is None for testing + let mut required_region_bytes = page_addr + page_size - start as usize; + if self.memory_limit_bytes.is_some() { + required_region_bytes += zjit_alloc_bytes(); + } + + assert!((start..=whole_region_end).contains(&mapped_region_end)); + + if (start..mapped_region_end).contains(&raw) { + // Writing to a previously written to page. + // Need to make page writable, but no need to fill. + let page_size: u32 = page_size.try_into().unwrap(); + if !alloc.mark_writable(page_addr as *const _, page_size) { + return Err(FailedPageMapping); + } + + self.current_write_page = Some(page_addr); + } else if (start..whole_region_end).contains(&raw) && + required_region_bytes < self.memory_limit_bytes.unwrap_or(self.region_size_bytes) { + // Writing to a brand new page + let mapped_region_end_addr = mapped_region_end as usize; + let alloc_size = page_addr - mapped_region_end_addr + page_size; + + assert_eq!(0, alloc_size % page_size, "allocation size should be page aligned"); + assert_eq!(0, mapped_region_end_addr % page_size, "pointer should be page aligned"); + + if alloc_size > page_size { + // This is unusual for the current setup, so keep track of it. + //crate::stats::incr_counter!(exec_mem_non_bump_alloc); // TODO + } + + // Allocate new chunk + let alloc_size_u32: u32 = alloc_size.try_into().unwrap(); + unsafe { + if !alloc.mark_writable(mapped_region_end.cast(), alloc_size_u32) { + return Err(FailedPageMapping); + } + if cfg!(target_arch = "x86_64") { + // Fill new memory with PUSH DS (0x1E) so that executing uninitialized memory + // will fault with #UD in 64-bit mode. On Linux it becomes SIGILL and use the + // usual Ruby crash reporter. + std::slice::from_raw_parts_mut(mapped_region_end, alloc_size).fill(0x1E); + } else if cfg!(target_arch = "aarch64") { + // In aarch64, all zeros encodes UDF, so it's already what we want. + } else { + unreachable!("unknown arch"); + } + } + self.mapped_region_bytes += alloc_size; + + self.current_write_page = Some(page_addr); + } else { + return Err(OutOfBounds); + } + } + + // We have permission to write if we get here + unsafe { raw.write(byte) }; + + Ok(()) + } + + /// Return true if write_byte() can allocate a new page + pub fn can_allocate(&self) -> bool { + let memory_usage_bytes = self.mapped_region_bytes + zjit_alloc_bytes(); + let memory_limit_bytes = self.memory_limit_bytes.unwrap_or(self.region_size_bytes); + memory_usage_bytes + self.page_size_bytes < memory_limit_bytes + } + + /// Make all the code in the region executable. Call this at the end of a write session. + /// See [Self] for usual usage flow. + pub fn mark_all_executable(&mut self) { + self.current_write_page = None; + + let region_start = self.region_start; + let mapped_region_bytes: u32 = self.mapped_region_bytes.try_into().unwrap(); + + // Make mapped region executable + self.allocator.mark_executable(region_start.as_ptr(), mapped_region_bytes); + } + + /// Free a range of bytes. start_ptr must be memory page-aligned. + pub fn free_bytes(&mut self, start_ptr: CodePtr, size: u32) { + assert_eq!(start_ptr.raw_ptr(self) as usize % self.page_size_bytes, 0); + + // Bounds check the request. We should only free memory we manage. + let mapped_region = self.start_ptr().raw_ptr(self)..self.mapped_end_ptr().raw_ptr(self); + let virtual_region = self.start_ptr().raw_ptr(self)..self.virtual_end_ptr().raw_ptr(self); + let last_byte_to_free = start_ptr.add_bytes(size.saturating_sub(1) as usize).raw_ptr(self); + assert!(mapped_region.contains(&start_ptr.raw_ptr(self))); + // On platforms where code page size != memory page size (e.g. Linux), we often need + // to free code pages that contain unmapped memory pages. When it happens on the last + // code page, it's more appropriate to check the last byte against the virtual region. + assert!(virtual_region.contains(&last_byte_to_free)); + + self.allocator.mark_unused(start_ptr.raw_ptr(self), size); + } +} + +/// Something that could provide a base pointer to compute a raw pointer from a [CodePtr]. +pub trait CodePtrBase { + fn base_ptr(&self) -> NonNull<u8>; +} + +impl<A: Allocator> CodePtrBase for VirtualMemory<A> { + fn base_ptr(&self) -> NonNull<u8> { + self.region_start + } +} + +/// Requires linking with CRuby to work +pub mod sys { + use crate::cruby::*; + + /// Zero size! This just groups together syscalls that require linking with CRuby. + pub struct SystemAllocator; + + type VoidPtr = *mut std::os::raw::c_void; + + impl super::Allocator for SystemAllocator { + fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } + } + + fn mark_executable(&mut self, ptr: *const u8, size: u32) { + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } + } + + fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } + } + } +} + + +#[cfg(test)] +pub mod tests { + use super::*; + + // Track allocation requests and owns some fixed size backing memory for requests. + // While testing we don't execute generated code. + pub struct TestingAllocator { + requests: Vec<AllocRequest>, + memory: Vec<u8>, + } + + #[derive(Debug)] + enum AllocRequest { + MarkWritable{ start_idx: usize, length: usize }, + MarkExecutable{ start_idx: usize, length: usize }, + MarkUnused, + } + use AllocRequest::*; + + impl TestingAllocator { + pub fn new(mem_size: usize) -> Self { + Self { requests: Vec::default(), memory: vec![0; mem_size] } + } + + pub fn mem_start(&self) -> *const u8 { + self.memory.as_ptr() + } + + // Verify that write_byte() bounds checks. Return `ptr` as an index. + fn bounds_check_request(&self, ptr: *const u8, size: u32) -> usize { + let mem_start = self.memory.as_ptr() as usize; + let index = ptr as usize - mem_start; + + assert!(index < self.memory.len()); + assert!(index + size as usize <= self.memory.len()); + + index + } + } + + // Bounds check and then record the request + impl super::Allocator for TestingAllocator { + fn mark_writable(&mut self, ptr: *const u8, length: u32) -> bool { + let index = self.bounds_check_request(ptr, length); + self.requests.push(MarkWritable { start_idx: index, length: length as usize }); + + true + } + + fn mark_executable(&mut self, ptr: *const u8, length: u32) { + let index = self.bounds_check_request(ptr, length); + self.requests.push(MarkExecutable { start_idx: index, length: length as usize }); + + // We don't try to execute generated code in cfg(test) + // so no need to actually request executable memory. + } + + fn mark_unused(&mut self, ptr: *const u8, length: u32) -> bool { + self.bounds_check_request(ptr, length); + self.requests.push(MarkUnused); + + true + } + } + + // Fictional architecture where each page is 4 bytes long + const PAGE_SIZE: usize = 4; + fn new_dummy_virt_mem() -> VirtualMemory<TestingAllocator> { + unsafe { + if crate::options::OPTIONS.is_none() { + crate::options::OPTIONS = Some(crate::options::Options::default()); + } + } + + let mem_size = PAGE_SIZE * 10; + let alloc = TestingAllocator::new(mem_size); + let mem_start: *const u8 = alloc.mem_start(); + + VirtualMemory::new( + alloc, + PAGE_SIZE.try_into().unwrap(), + NonNull::new(mem_start as *mut u8).unwrap(), + mem_size, + None, + ) + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn new_memory_is_initialized() { + let mut virt = new_dummy_virt_mem(); + + virt.write_byte(virt.start_ptr(), 1).unwrap(); + assert!( + virt.allocator.memory[..PAGE_SIZE].iter().all(|&byte| byte != 0), + "Entire page should be initialized", + ); + + // Skip a few page + let three_pages = 3 * PAGE_SIZE; + virt.write_byte(virt.start_ptr().add_bytes(three_pages), 1).unwrap(); + assert!( + virt.allocator.memory[..three_pages].iter().all(|&byte| byte != 0), + "Gaps between write requests should be filled", + ); + } + + #[test] + fn no_redundant_syscalls_when_writing_to_the_same_page() { + let mut virt = new_dummy_virt_mem(); + + virt.write_byte(virt.start_ptr(), 1).unwrap(); + virt.write_byte(virt.start_ptr(), 0).unwrap(); + + assert!( + matches!( + virt.allocator.requests[..], + [MarkWritable { start_idx: 0, length: PAGE_SIZE }], + ) + ); + } + + #[test] + fn bounds_checking() { + use super::WriteError::*; + let mut virt = new_dummy_virt_mem(); + + let one_past_end = virt.start_ptr().add_bytes(virt.virtual_region_size()); + assert_eq!(Err(OutOfBounds), virt.write_byte(one_past_end, 0)); + + let end_of_addr_space = CodePtr(u32::MAX); + assert_eq!(Err(OutOfBounds), virt.write_byte(end_of_addr_space, 0)); + } + + #[test] + fn only_written_to_regions_become_executable() { + // ... so we catch attempts to read/write/execute never-written-to regions + const THREE_PAGES: usize = PAGE_SIZE * 3; + let mut virt = new_dummy_virt_mem(); + let page_two_start = virt.start_ptr().add_bytes(PAGE_SIZE * 2); + virt.write_byte(page_two_start, 1).unwrap(); + virt.mark_all_executable(); + + assert!(virt.virtual_region_size() > THREE_PAGES); + assert!( + matches!( + virt.allocator.requests[..], + [ + MarkWritable { start_idx: 0, length: THREE_PAGES }, + MarkExecutable { start_idx: 0, length: THREE_PAGES }, + ] + ), + ); + } +} |