diff options
Diffstat (limited to 'lib/ruby_vm/mjit')
| -rw-r--r-- | lib/ruby_vm/mjit/assembler.rb | 1052 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/block.rb | 10 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/branch_stub.rb | 24 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/c_pointer.rb | 360 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/c_type.rb | 91 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/code_block.rb | 86 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/compiler.rb | 291 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/context.rb | 30 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/exit_compiler.rb | 146 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/hooks.rb | 30 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/insn_compiler.rb | 2515 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/invariants.rb | 116 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/jit_state.rb | 37 | ||||
| -rw-r--r-- | lib/ruby_vm/mjit/stats.rb | 99 |
14 files changed, 0 insertions, 4887 deletions
diff --git a/lib/ruby_vm/mjit/assembler.rb b/lib/ruby_vm/mjit/assembler.rb deleted file mode 100644 index 989d069776..0000000000 --- a/lib/ruby_vm/mjit/assembler.rb +++ /dev/null @@ -1,1052 +0,0 @@ -# frozen_string_literal: true -module RubyVM::MJIT - # 8-bit memory access - class BytePtr < Data.define(:reg, :disp); end - - # 32-bit memory access - class DwordPtr < Data.define(:reg, :disp); end - - # SystemV x64 calling convention - C_ARGS = [:rdi, :rsi, :rdx, :rcx, :r8, :r9] - C_RET = :rax - - # https://www.intel.com/content/dam/develop/public/us/en/documents/325383-sdm-vol-2abcd.pdf - # Mostly an x86_64 assembler, but this also has some stuff that is useful for any architecture. - class Assembler - # rel8 jumps are made with labels - class Label < Data.define(:id, :name); end - - # rel32 is inserted as [Rel32, Rel32Pad..] and converted on #resolve_rel32 - class Rel32 < Data.define(:addr); end - Rel32Pad = Object.new - - # A set of ModR/M values encoded on #insn - class ModRM < Data.define(:mod, :reg, :rm); end - Mod00 = 0b00 # Mod 00: [reg] - Mod01 = 0b01 # Mod 01: [reg]+disp8 - Mod10 = 0b10 # Mod 10: [reg]+disp32 - Mod11 = 0b11 # Mod 11: reg - - # REX = 0100WR0B - REX_B = 0b01000001 - REX_R = 0b01000100 - REX_W = 0b01001000 - - def initialize - @bytes = [] - @labels = {} - @label_id = 0 - @comments = Hash.new { |h, k| h[k] = [] } - @blocks = Hash.new { |h, k| h[k] = [] } - @stub_starts = Hash.new { |h, k| h[k] = [] } - @stub_ends = Hash.new { |h, k| h[k] = [] } - @pos_markers = Hash.new { |h, k| h[k] = [] } - end - - def assemble(addr) - set_code_addrs(addr) - resolve_rel32(addr) - resolve_labels - - write_bytes(addr) - - @pos_markers.each do |write_pos, markers| - markers.each { |marker| marker.call(addr + write_pos) } - end - @bytes.size - ensure - @bytes.clear - end - - def size - @bytes.size - end - - # - # Instructions - # - - def add(dst, src) - case [dst, src] - # ADD r/m64, imm8 (Mod 00: [reg]) - in [Array[Symbol => dst_reg], Integer => src_imm] if r64?(dst_reg) && imm8?(src_imm) - # REX.W + 83 /0 ib - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod00, reg: 0, rm: dst_reg], - imm: imm8(src_imm), - ) - # ADD r/m64, imm8 (Mod 11: reg) - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm8?(src_imm) - # REX.W + 83 /0 ib - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg], - imm: imm8(src_imm), - ) - # ADD r/m64 imm32 (Mod 11: reg) - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm32?(src_imm) - # REX.W + 81 /0 id - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x81, - mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg], - imm: imm32(src_imm), - ) - # ADD r/m64, r64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg) - # REX.W + 01 /r - # MR: Operand 1: ModRM:r/m (r, w), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x01, - mod_rm: ModRM[mod: Mod11, reg: src_reg, rm: dst_reg], - ) - else - raise NotImplementedError, "add: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def and(dst, src) - case [dst, src] - # AND r/m64, imm8 (Mod 11: reg) - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm8?(src_imm) - # REX.W + 83 /4 ib - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg], - imm: imm8(src_imm), - ) - # AND r/m64, imm32 (Mod 11: reg) - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm32?(src_imm) - # REX.W + 81 /4 id - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x81, - mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg], - imm: imm32(src_imm), - ) - # AND r64, r/m64 (Mod 01: [reg]+disp8) - in [Symbol => dst_reg, Array[Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp) - # REX.W + 23 /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x23, - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: imm8(src_disp), - ) - else - raise NotImplementedError, "and: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def call(dst) - case dst - # CALL rel32 - in Integer => dst_addr - # E8 cd - # D: Operand 1: Offset - insn(opcode: 0xe8, imm: rel32(dst_addr)) - # CALL r/m64 (Mod 11: reg) - in Symbol => dst_reg - # FF /2 - # M: Operand 1: ModRM:r/m (r) - insn( - opcode: 0xff, - mod_rm: ModRM[mod: Mod11, reg: 2, rm: dst_reg], - ) - else - raise NotImplementedError, "call: not-implemented operands: #{dst.inspect}" - end - end - - def cmove(dst, src) - case [dst, src] - # CMOVE r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] - # REX.W + 0F 44 /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x44], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmove: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovg(dst, src) - case [dst, src] - # CMOVG r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] - # REX.W + 0F 4F /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x4f], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmovg: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovge(dst, src) - case [dst, src] - # CMOVGE r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] - # REX.W + 0F 4D /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x4d], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmovge: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovl(dst, src) - case [dst, src] - # CMOVL r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] - # REX.W + 0F 4C /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x4c], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmovl: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovle(dst, src) - case [dst, src] - # CMOVLE r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] - # REX.W + 0F 4E /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x4e], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmovle: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovnz(dst, src) - case [dst, src] - # CMOVNZ r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg) - # REX.W + 0F 45 /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x45], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - else - raise NotImplementedError, "cmovnz: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmovz(dst, src) - case [dst, src] - # CMOVZ r64, r/m64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg) - # REX.W + 0F 44 /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x44], - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - # CMOVZ r64, r/m64 (Mod 01: [reg]+disp8) - in [Symbol => dst_reg, Array[Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp) - # REX.W + 0F 44 /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: [0x0f, 0x44], - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: imm8(src_disp), - ) - else - raise NotImplementedError, "cmovz: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def cmp(left, right) - case [left, right] - # CMP r/m8, imm8 (Mod 01: [reg]+disp8) - in [BytePtr[reg: left_reg, disp: left_disp], Integer => right_imm] if r64?(left_reg) && imm8?(left_disp) && imm8?(right_imm) - # 80 /7 ib - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - opcode: 0x80, - mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg], - disp: left_disp, - imm: imm8(right_imm), - ) - # CMP r/m32, imm32 (Mod 01: [reg]+disp8) - in [DwordPtr[reg: left_reg, disp: left_disp], Integer => right_imm] if imm8?(left_disp) && imm32?(right_imm) - # 81 /7 id - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - opcode: 0x81, - mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg], - disp: left_disp, - imm: imm32(right_imm), - ) - # CMP r/m64, imm8 (Mod 01: [reg]+disp8) - in [Array[Symbol => left_reg, Integer => left_disp], Integer => right_imm] if r64?(left_reg) && imm8?(left_disp) && imm8?(right_imm) - # REX.W + 83 /7 ib - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg], - disp: left_disp, - imm: imm8(right_imm), - ) - # CMP r/m64, imm8 (Mod 11: reg) - in [Symbol => left_reg, Integer => right_imm] if r64?(left_reg) && imm8?(right_imm) - # REX.W + 83 /7 ib - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod11, reg: 7, rm: left_reg], - imm: imm8(right_imm), - ) - # CMP r/m64, imm32 (Mod 11: reg) - in [Symbol => left_reg, Integer => right_imm] if r64?(left_reg) && imm32?(right_imm) - # REX.W + 81 /7 id - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x81, - mod_rm: ModRM[mod: Mod11, reg: 7, rm: left_reg], - imm: imm32(right_imm), - ) - # CMP r/m64, r64 (Mod 01: [reg]+disp8) - in [Array[Symbol => left_reg, Integer => left_disp], Symbol => right_reg] if r64?(right_reg) - # REX.W + 39 /r - # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x39, - mod_rm: ModRM[mod: Mod01, reg: right_reg, rm: left_reg], - disp: left_disp, - ) - # CMP r/m64, r64 (Mod 11: reg) - in [Symbol => left_reg, Symbol => right_reg] if r64?(left_reg) && r64?(right_reg) - # REX.W + 39 /r - # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x39, - mod_rm: ModRM[mod: Mod11, reg: right_reg, rm: left_reg], - ) - else - raise NotImplementedError, "cmp: not-implemented operands: #{left.inspect}, #{right.inspect}" - end - end - - def jbe(dst) - case dst - # JBE rel32 - in Integer => dst_addr - # 0F 86 cd - insn(opcode: [0x0f, 0x86], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jbe: not-implemented operands: #{dst.inspect}" - end - end - - def je(dst) - case dst - # JE rel8 - in Label => dst_label - # 74 cb - insn(opcode: 0x74, imm: dst_label) - # JE rel32 - in Integer => dst_addr - # 0F 84 cd - insn(opcode: [0x0f, 0x84], imm: rel32(dst_addr)) - else - raise NotImplementedError, "je: not-implemented operands: #{dst.inspect}" - end - end - - def jl(dst) - case dst - # JL rel32 - in Integer => dst_addr - # 0F 8C cd - insn(opcode: [0x0f, 0x8c], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jl: not-implemented operands: #{dst.inspect}" - end - end - - def jmp(dst) - case dst - # JZ rel8 - in Label => dst_label - # EB cb - insn(opcode: 0xeb, imm: dst_label) - # JMP rel32 - in Integer => dst_addr - # E9 cd - insn(opcode: 0xe9, imm: rel32(dst_addr)) - # JMP r/m64 (Mod 01: [reg]+disp8) - in Array[Symbol => dst_reg, Integer => dst_disp] if imm8?(dst_disp) - # FF /4 - insn(opcode: 0xff, mod_rm: ModRM[mod: Mod01, reg: 4, rm: dst_reg], disp: dst_disp) - # JMP r/m64 (Mod 11: reg) - in Symbol => dst_reg - # FF /4 - insn(opcode: 0xff, mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg]) - else - raise NotImplementedError, "jmp: not-implemented operands: #{dst.inspect}" - end - end - - def jne(dst) - case dst - # JNE rel32 - in Integer => dst_addr - # 0F 85 cd - insn(opcode: [0x0f, 0x85], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jne: not-implemented operands: #{dst.inspect}" - end - end - - def jnz(dst) - case dst - # JNZ rel32 - in Integer => dst_addr - # 0F 85 cd - insn(opcode: [0x0f, 0x85], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jnz: not-implemented operands: #{dst.inspect}" - end - end - - def jo(dst) - case dst - # JO rel32 - in Integer => dst_addr - # 0F 80 cd - insn(opcode: [0x0f, 0x80], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jo: not-implemented operands: #{dst.inspect}" - end - end - - def jz(dst) - case dst - # JZ rel8 - in Label => dst_label - # 74 cb - insn(opcode: 0x74, imm: dst_label) - # JZ rel32 - in Integer => dst_addr - # 0F 84 cd - insn(opcode: [0x0f, 0x84], imm: rel32(dst_addr)) - else - raise NotImplementedError, "jz: not-implemented operands: #{dst.inspect}" - end - end - - def lea(dst, src) - case [dst, src] - # LEA r64,m (Mod 01: [reg]+disp8) - in [Symbol => dst_reg, Array[Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp) - # REX.W + 8D /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8d, - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: imm8(src_disp), - ) - # LEA r64,m (Mod 10: [reg]+disp32) - in [Symbol => dst_reg, Array[Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm32?(src_disp) - # REX.W + 8D /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8d, - mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg], - disp: imm32(src_disp), - ) - else - raise NotImplementedError, "lea: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def mov(dst, src) - case dst - in Symbol => dst_reg - case src - # MOV r64, r/m64 (Mod 00: [reg]) - in Array[Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg) - # REX.W + 8B /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8b, - mod_rm: ModRM[mod: Mod00, reg: dst_reg, rm: src_reg], - ) - # MOV r64, r/m64 (Mod 01: [reg]+disp8) - in Array[Symbol => src_reg, Integer => src_disp] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp) - # REX.W + 8B /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8b, - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: src_disp, - ) - # MOV r64, r/m64 (Mod 10: [reg]+disp16) - in Array[Symbol => src_reg, Integer => src_disp] if r64?(dst_reg) && r64?(src_reg) && imm32?(src_disp) - # REX.W + 8B /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8b, - mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg], - disp: imm32(src_disp), - ) - # MOV r64, r/m64 (Mod 11: reg) - in Symbol => src_reg if r64?(dst_reg) && r64?(src_reg) - # REX.W + 8B /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x8b, - mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg], - ) - # MOV r32 r/m32 (Mod 01: [reg]+disp8) - in Array[Symbol => src_reg, Integer => src_disp] if r32?(dst_reg) && imm8?(src_disp) - # 8B /r - # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r) - insn( - opcode: 0x8b, - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: src_disp, - ) - # MOV r32, imm32 (Mod 11: reg) - in Integer => src_imm if r32?(dst_reg) && imm32?(src_imm) - # B8+ rd id - # OI: Operand 1: opcode + rd (w), Operand 2: imm8/16/32/64 - insn( - opcode: 0xb8, - rd: dst_reg, - imm: imm32(src_imm), - ) - # MOV r/m64, imm32 (Mod 11: reg) - in Integer => src_imm if r64?(dst_reg) && imm32?(src_imm) - # REX.W + C7 /0 id - # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64 - insn( - prefix: REX_W, - opcode: 0xc7, - mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg], - imm: imm32(src_imm), - ) - # MOV r64, imm64 - in Integer => src_imm if r64?(dst_reg) && imm64?(src_imm) - # REX.W + B8+ rd io - # OI: Operand 1: opcode + rd (w), Operand 2: imm8/16/32/64 - insn( - prefix: REX_W, - opcode: 0xb8, - rd: dst_reg, - imm: imm64(src_imm), - ) - else - raise NotImplementedError, "mov: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - in Array[Symbol => dst_reg] - case src - # MOV r/m64, imm32 (Mod 00: [reg]) - in Integer => src_imm if r64?(dst_reg) && imm32?(src_imm) - # REX.W + C7 /0 id - # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64 - insn( - prefix: REX_W, - opcode: 0xc7, - mod_rm: ModRM[mod: Mod00, reg: 0, rm: dst_reg], - imm: imm32(src_imm), - ) - # MOV r/m64, r64 (Mod 00: [reg]) - in Symbol => src_reg if r64?(dst_reg) && r64?(src_reg) - # REX.W + 89 /r - # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x89, - mod_rm: ModRM[mod: Mod00, reg: src_reg, rm: dst_reg], - ) - else - raise NotImplementedError, "mov: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - in Array[Symbol => dst_reg, Integer => dst_disp] - # Optimize encoding when disp is 0 - return mov([dst_reg], src) if dst_disp == 0 - - case src - # MOV r/m64, imm32 (Mod 01: [reg]+disp8) - in Integer => src_imm if r64?(dst_reg) && imm8?(dst_disp) && imm32?(src_imm) - # REX.W + C7 /0 id - # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64 - insn( - prefix: REX_W, - opcode: 0xc7, - mod_rm: ModRM[mod: Mod01, reg: 0, rm: dst_reg], - disp: dst_disp, - imm: imm32(src_imm), - ) - # MOV r/m64, imm32 (Mod 10: [reg]+disp32) - in Integer => src_imm if r64?(dst_reg) && imm32?(dst_disp) && imm32?(src_imm) - # REX.W + C7 /0 id - # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64 - insn( - prefix: REX_W, - opcode: 0xc7, - mod_rm: ModRM[mod: Mod10, reg: 0, rm: dst_reg], - disp: imm32(dst_disp), - imm: imm32(src_imm), - ) - # MOV r/m64, r64 (Mod 01: [reg]+disp8) - in Symbol => src_reg if r64?(dst_reg) && imm8?(dst_disp) && r64?(src_reg) - # REX.W + 89 /r - # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x89, - mod_rm: ModRM[mod: Mod01, reg: src_reg, rm: dst_reg], - disp: dst_disp, - ) - # MOV r/m64, r64 (Mod 10: [reg]+disp32) - in Symbol => src_reg if r64?(dst_reg) && imm32?(dst_disp) && r64?(src_reg) - # REX.W + 89 /r - # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x89, - mod_rm: ModRM[mod: Mod10, reg: src_reg, rm: dst_reg], - disp: imm32(dst_disp), - ) - else - raise NotImplementedError, "mov: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - else - raise NotImplementedError, "mov: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def or(dst, src) - case [dst, src] - # OR r64, r/m64 (Mod 01: [reg]+disp8) - in [Symbol => dst_reg, Array[Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp) - # REX.W + 0B /r - # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r) - insn( - prefix: REX_W, - opcode: 0x0b, - mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg], - disp: imm8(src_disp), - ) - else - raise NotImplementedError, "or: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def push(src) - case src - # PUSH r64 - in Symbol => src_reg if r64?(src_reg) - # 50+rd - # O: Operand 1: opcode + rd (r) - insn(opcode: 0x50, rd: src_reg) - else - raise NotImplementedError, "push: not-implemented operands: #{src.inspect}" - end - end - - def pop(dst) - case dst - # POP r64 - in Symbol => dst_reg if r64?(dst_reg) - # 58+ rd - # O: Operand 1: opcode + rd (r) - insn(opcode: 0x58, rd: dst_reg) - else - raise NotImplementedError, "pop: not-implemented operands: #{dst.inspect}" - end - end - - def ret - # RET - # Near return: A return to a procedure within the current code segment - insn(opcode: 0xc3) - end - - def sar(dst, src) - case [dst, src] - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm8?(src_imm) - # REX.W + C1 /7 ib - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8 - insn( - prefix: REX_W, - opcode: 0xc1, - mod_rm: ModRM[mod: Mod11, reg: 7, rm: dst_reg], - imm: imm8(src_imm), - ) - else - raise NotImplementedError, "sar: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def sub(dst, src) - case [dst, src] - # SUB r/m64, imm8 - in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm8?(src_imm) - # REX.W + 83 /5 ib - # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0x83, - mod_rm: ModRM[mod: Mod11, reg: 5, rm: dst_reg], - imm: imm8(src_imm), - ) - # SUB r/m64, r64 (Mod 11: reg) - in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg) - # REX.W + 29 /r - # MR: Operand 1: ModRM:r/m (r, w), Operand 2: ModRM:reg (r) - insn( - prefix: REX_W, - opcode: 0x29, - mod_rm: ModRM[mod: Mod11, reg: src_reg, rm: dst_reg], - ) - else - raise NotImplementedError, "sub: not-implemented operands: #{dst.inspect}, #{src.inspect}" - end - end - - def test(left, right) - case [left, right] - # TEST r/m8*, imm8 (Mod 01: [reg]+disp8) - in [Array[Symbol => left_reg, Integer => left_disp], Integer => right_imm] if imm8?(right_imm) && right_imm >= 0 - # REX + F6 /0 ib - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - opcode: 0xf6, - mod_rm: ModRM[mod: Mod01, reg: 0, rm: left_reg], - disp: left_disp, - imm: imm8(right_imm), - ) - # TEST r/m64, imm32 (Mod 01: [reg]+disp8) - in [Array[Symbol => left_reg, Integer => left_disp], Integer => right_imm] if imm32?(right_imm) - # REX.W + F7 /0 id - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0xf7, - mod_rm: ModRM[mod: Mod01, reg: 0, rm: left_reg], - disp: left_disp, - imm: imm32(right_imm), - ) - # TEST r/m64, imm32 (Mod 11: reg) - in [Symbol => left_reg, Integer => right_imm] if r64?(left_reg) && imm32?(right_imm) - # REX.W + F7 /0 id - # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 - insn( - prefix: REX_W, - opcode: 0xf7, - mod_rm: ModRM[mod: Mod11, reg: 0, rm: left_reg], - imm: imm32(right_imm), - ) - # TEST r/m32, r32 (Mod 11: reg) - in [Symbol => left_reg, Symbol => right_reg] if r32?(left_reg) && r32?(right_reg) - # 85 /r - # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r) - insn( - opcode: 0x85, - mod_rm: ModRM[mod: Mod11, reg: right_reg, rm: left_reg], - ) - else - raise NotImplementedError, "test: not-implemented operands: #{left.inspect}, #{right.inspect}" - end - end - - # - # Utilities - # - - attr_reader :comments - - def comment(message) - @comments[@bytes.size] << message - end - - # Mark the starting address of a block - def block(block) - @blocks[@bytes.size] << block - end - - # Mark the starting/ending addresses of a stub - def stub(stub) - @stub_starts[@bytes.size] << stub - yield - ensure - @stub_ends[@bytes.size] << stub - end - - def pos_marker(&block) - @pos_markers[@bytes.size] << block - end - - def new_label(name) - Label.new(id: @label_id += 1, name:) - end - - # @param [RubyVM::MJIT::Assembler::Label] label - def write_label(label) - @labels[label] = @bytes.size - end - - def incr_counter(name) - if C.mjit_opts.stats - comment("increment counter #{name}") - mov(:rax, C.rb_mjit_counters[name].to_i) - add([:rax], 1) # TODO: lock - end - end - - def imm32?(imm) - (-0x8000_0000..0x7fff_ffff).include?(imm) # TODO: consider uimm - end - - private - - def insn(prefix: 0, opcode:, rd: nil, mod_rm: nil, disp: nil, imm: nil) - # Determine prefix - if rd - prefix |= REX_B if extended_reg?(rd) - opcode += reg_code(rd) - end - if mod_rm - prefix |= REX_R if mod_rm.reg.is_a?(Symbol) && extended_reg?(mod_rm.reg) - prefix |= REX_B if mod_rm.rm.is_a?(Symbol) && extended_reg?(mod_rm.rm) - end - - # Encode insn - if prefix > 0 - @bytes.push(prefix) - end - @bytes.push(*Array(opcode)) - if mod_rm - mod_rm_byte = encode_mod_rm( - mod: mod_rm.mod, - reg: mod_rm.reg.is_a?(Symbol) ? reg_code(mod_rm.reg) : mod_rm.reg, - rm: mod_rm.rm.is_a?(Symbol) ? reg_code(mod_rm.rm) : mod_rm.rm, - ) - @bytes.push(mod_rm_byte) - end - if disp - @bytes.push(*Array(disp)) - end - if imm - @bytes.push(*imm) - end - end - - def reg_code(reg) - reg_code_extended(reg).first - end - - def extended_reg?(reg) - reg_code_extended(reg).last - end - - def reg_code_extended(reg) - case reg - # Not extended - when :al, :ax, :eax, :rax then [0, false] - when :cl, :cx, :ecx, :rcx then [1, false] - when :dl, :dx, :edx, :rdx then [2, false] - when :bl, :bx, :ebx, :rbx then [3, false] - when :ah, :sp, :esp, :rsp then [4, false] - when :ch, :bp, :ebp, :rbp then [5, false] - when :dh, :si, :esi, :rsi then [6, false] - when :bh, :di, :edi, :rdi then [7, false] - # Extended - when :r8b, :r8w, :r8d, :r8 then [0, true] - when :r9b, :r9w, :r9d, :r9 then [1, true] - when :r10b, :r10w, :r10d, :r10 then [2, true] - when :r11b, :r11w, :r11d, :r11 then [3, true] - when :r12b, :r12w, :r12d, :r12 then [4, true] - when :r13b, :r13w, :r13d, :r13 then [5, true] - when :r14b, :r14w, :r14d, :r14 then [6, true] - when :r15b, :r15w, :r15d, :r15 then [7, true] - else raise ArgumentError, "unexpected reg: #{reg.inspect}" - end - end - - # Table 2-2. 32-Bit Addressing Forms with the ModR/M Byte - # - # 7 6 5 4 3 2 1 0 - # +--+--+--+--+--+--+--+--+ - # | Mod | Reg/ | R/M | - # | | Opcode | | - # +--+--+--+--+--+--+--+--+ - # - # The r/m field can specify a register as an operand or it can be combined - # with the mod field to encode an addressing mode. - # - # /0: R/M is 0 (not used) - # /r: R/M is a register - def encode_mod_rm(mod:, reg: 0, rm: 0) - if mod > 0b11 - raise ArgumentError, "too large Mod: #{mod}" - end - if reg > 0b111 - raise ArgumentError, "too large Reg/Opcode: #{reg}" - end - if rm > 0b111 - raise ArgumentError, "too large R/M: #{rm}" - end - (mod << 6) + (reg << 3) + rm - end - - # ib: 1 byte - def imm8(imm) - unless imm8?(imm) - raise ArgumentError, "unexpected imm8: #{imm}" - end - [imm].pack('c').unpack('c*') # TODO: consider uimm - end - - # id: 4 bytes - def imm32(imm) - unless imm32?(imm) - raise ArgumentError, "unexpected imm32: #{imm}" - end - [imm].pack('l').unpack('c*') # TODO: consider uimm - end - - # io: 8 bytes - def imm64(imm) - unless imm64?(imm) - raise ArgumentError, "unexpected imm64: #{imm}" - end - imm_bytes(imm, 8) - end - - def imm_bytes(imm, num_bytes) - bytes = [] - bits = imm - num_bytes.times do - bytes << (bits & 0xff) - bits >>= 8 - end - if bits != 0 - raise ArgumentError, "unexpected imm with #{num_bytes} bytes: #{imm}" - end - bytes - end - - def imm8?(imm) - (-0x80..0x7f).include?(imm) - end - - def imm64?(imm) - (-0x8000_0000_0000_0000..0xffff_ffff_ffff_ffff).include?(imm) - end - - def r32?(reg) - if extended_reg?(reg) - reg.end_with?('d') - else - reg.start_with?('e') - end - end - - def r64?(reg) - if extended_reg?(reg) - reg.match?(/\Ar\d+\z/) - else - reg.start_with?('r') - end - end - - def rel32(addr) - [Rel32.new(addr), Rel32Pad, Rel32Pad, Rel32Pad] - end - - def set_code_addrs(write_addr) - (@bytes.size + 1).times do |index| - @blocks.fetch(index, []).each do |block| - block.start_addr = write_addr + index - end - @stub_starts.fetch(index, []).each do |stub| - stub.start_addr = write_addr + index - end - @stub_ends.fetch(index, []).each do |stub| - stub.end_addr = write_addr + index - end - end - end - - def resolve_rel32(write_addr) - @bytes.each_with_index do |byte, index| - if byte.is_a?(Rel32) - src_addr = write_addr + index + 4 # offset 4 bytes for rel32 itself - dst_addr = byte.addr - rel32 = dst_addr - src_addr - raise "unexpected offset: #{rel32}" unless imm32?(rel32) - imm32(rel32).each_with_index do |rel_byte, rel_index| - @bytes[index + rel_index] = rel_byte - end - end - end - end - - def resolve_labels - @bytes.each_with_index do |byte, index| - if byte.is_a?(Label) - src_index = index + 1 # offset 1 byte for rel8 itself - dst_index = @labels.fetch(byte) - rel8 = dst_index - src_index - raise "unexpected offset: #{rel8}" unless imm8?(rel8) - @bytes[index] = rel8 - end - end - end - - def write_bytes(addr) - Fiddle::Pointer.new(addr)[0, @bytes.size] = @bytes.pack('c*') - end - end -end diff --git a/lib/ruby_vm/mjit/block.rb b/lib/ruby_vm/mjit/block.rb deleted file mode 100644 index abe0c221b2..0000000000 --- a/lib/ruby_vm/mjit/block.rb +++ /dev/null @@ -1,10 +0,0 @@ -class RubyVM::MJIT::Block < Struct.new( - :pc, # @param [Integer] Starting PC - :ctx, # @param [RubyVM::MJIT::Context] **Starting** Context (TODO: freeze?) - :start_addr, # @param [Integer] Starting address of this block's JIT code - :entry_exit, # @param [Integer] Address of entry exit (optional) - :incoming, # @param [Array<RubyVM::MJIT::BranchStub>] Incoming branches - :invalidated, # @param [TrueClass,FalseClass] true if already invalidated -) - def initialize(incoming: [], invalidated: false, **) = super -end diff --git a/lib/ruby_vm/mjit/branch_stub.rb b/lib/ruby_vm/mjit/branch_stub.rb deleted file mode 100644 index 0f015e2f72..0000000000 --- a/lib/ruby_vm/mjit/branch_stub.rb +++ /dev/null @@ -1,24 +0,0 @@ -module RubyVM::MJIT - # Branch shapes - Next0 = :Next0 # target0 is a fallthrough - Next1 = :Next1 # target1 is a fallthrough - Default = :Default # neither targets is a fallthrough - - class BranchStub < Struct.new( - :iseq, # @param [RubyVM::MJIT::CPointer::Struct_rb_iseq_struct] Branch target ISEQ - :shape, # @param [Symbol] Next0, Next1, or Default - :target0, # @param [RubyVM::MJIT::BranchTarget] First branch target - :target1, # @param [RubyVM::MJIT::BranchTarget,NilClass] Second branch target (optional) - :compile, # @param [Proc] A callback to (re-)generate this branch stub - :start_addr, # @param [Integer] Stub source start address to be re-generated - :end_addr, # @param [Integer] Stub source end address to be re-generated - ) - end - - class BranchTarget < Struct.new( - :pc, - :ctx, - :address, - ) - end -end diff --git a/lib/ruby_vm/mjit/c_pointer.rb b/lib/ruby_vm/mjit/c_pointer.rb deleted file mode 100644 index 73cf267482..0000000000 --- a/lib/ruby_vm/mjit/c_pointer.rb +++ /dev/null @@ -1,360 +0,0 @@ -module RubyVM::MJIT - # Every class under this namespace is a pointer. Even if the type is - # immediate, it shouldn't be dereferenced until `*` is called. - module CPointer - # Note: We'd like to avoid alphabetic method names to avoid a conflict - # with member methods. to_i and to_s are considered an exception. - class Struct - # @param name [String] - # @param sizeof [Integer] - # @param members [Hash{ Symbol => [RubyVM::MJIT::CType::*, Integer, TrueClass] }] - def initialize(addr, sizeof, members) - @addr = addr - @sizeof = sizeof - @members = members - end - - # Get a raw address - def to_i - @addr - end - - # Serialized address for generated code - def to_s - "0x#{@addr.to_s(16)}" - end - - # Pointer diff - def -(struct) - raise ArgumentError if self.class != struct.class - (@addr - struct.to_i) / @sizeof - end - - # Primitive API that does no automatic dereference - # TODO: remove this? - # @param member [Symbol] - def [](member) - type, offset = @members.fetch(member) - type.new(@addr + offset / 8) - end - - private - - # @param member [Symbol] - # @param value [Object] - def []=(member, value) - type, offset = @members.fetch(member) - type[@addr + offset / 8] = value - end - - # @param size [Integer] - # @param members [Hash{ Symbol => [Integer, RubyVM::MJIT::CType::*] }] - def self.define(size, members) - Class.new(self) do - # Return the size of this type - define_singleton_method(:size) { size } - - # Return the offset to a field - define_singleton_method(:offsetof) do |field, *fields| - member, offset = members.fetch(field) - offset /= 8 - unless fields.empty? - offset += member.offsetof(*fields) - end - offset - end - - # Return member names - define_singleton_method(:members) { members.keys } - - define_method(:initialize) do |addr = nil| - if addr.nil? # TODO: get rid of this feature later - addr = Fiddle.malloc(size) - end - super(addr, size, members) - end - - members.each do |member, (type, offset, to_ruby)| - # Intelligent API that does automatic dereference - define_method(member) do - value = self[member] - if value.respond_to?(:*) - value = value.* - end - if to_ruby - value = C.to_ruby(value) - end - value - end - - define_method("#{member}=") do |value| - if to_ruby - value = C.to_value(value) - end - self[member] = value - end - end - end - end - end - - # Note: We'd like to avoid alphabetic method names to avoid a conflict - # with member methods. to_i is considered an exception. - class Union - # @param _name [String] To be used when it starts defining a union pointer class - # @param sizeof [Integer] - # @param members [Hash{ Symbol => RubyVM::MJIT::CType::* }] - def initialize(addr, sizeof, members) - @addr = addr - @sizeof = sizeof - @members = members - end - - # Get a raw address - def to_i - @addr - end - - # Move addr to access this pointer like an array - def +(index) - raise ArgumentError unless index.is_a?(Integer) - self.class.new(@addr + index * @sizeof) - end - - # Pointer diff - def -(union) - raise ArgumentError if self.class != union.class - (@addr - union.instance_variable_get(:@addr)) / @sizeof - end - - # @param sizeof [Integer] - # @param members [Hash{ Symbol => RubyVM::MJIT::CType::* }] - def self.define(sizeof, members) - Class.new(self) do - # Return the size of this type - define_singleton_method(:sizeof) { sizeof } - - # Part of Struct's offsetof implementation - define_singleton_method(:offsetof) do |field, *fields| - member = members.fetch(field) - offset = 0 - unless fields.empty? - offset += member.offsetof(*fields) - end - offset - end - - define_method(:initialize) do |addr| - super(addr, sizeof, members) - end - - members.each do |member, type| - # Intelligent API that does automatic dereference - define_method(member) do - value = type.new(@addr) - if value.respond_to?(:*) - value = value.* - end - value - end - end - end - end - end - - class Immediate - # @param addr [Integer] - # @param size [Integer] - # @param pack [String] - def initialize(addr, size, pack) - @addr = addr - @size = size - @pack = pack - end - - # Get a raw address - def to_i - @addr - end - - # Move addr to addess this pointer like an array - def +(index) - Immediate.new(@addr + index * @size, @size, @pack) - end - - # Dereference - def * - self[0] - end - - # Array access - def [](index) - return nil if @addr == 0 - Fiddle::Pointer.new(@addr + index * @size)[0, @size].unpack1(@pack) - end - - # Array set - def []=(index, value) - Fiddle::Pointer.new(@addr + index * @size)[0, @size] = [value].pack(@pack) - end - - # Serialized address for generated code. Used for embedding things like body->iseq_encoded. - def to_s - "0x#{Integer(@addr).to_s(16)}" - end - - # @param fiddle_type [Integer] Fiddle::TYPE_* - def self.define(fiddle_type) - size = Fiddle::PackInfo::SIZE_MAP.fetch(fiddle_type) - pack = Fiddle::PackInfo::PACK_MAP.fetch(fiddle_type) - - Class.new(self) do - define_method(:initialize) do |addr| - super(addr, size, pack) - end - - define_singleton_method(:size) do - size - end - - # Type-level []=: Used by struct fields - define_singleton_method(:[]=) do |addr, value| - Fiddle::Pointer.new(addr)[0, size] = [value].pack(pack) - end - end - end - end - - # -Fiddle::TYPE_CHAR Immediate with special handling of true/false - class Bool < Immediate.define(-Fiddle::TYPE_CHAR) - # Dereference - def * - return nil if @addr == 0 - super != 0 - end - - def self.[]=(addr, value) - super(addr, value ? 1 : 0) - end - end - - class Pointer - attr_reader :type - - # @param addr [Integer] - # @param type [Class] RubyVM::MJIT::CType::* - def initialize(addr, type) - @addr = addr - @type = type - end - - # Move addr to addess this pointer like an array - def +(index) - raise ArgumentError unless index.is_a?(Integer) - Pointer.new(@addr + index * Fiddle::SIZEOF_VOIDP, @type) - end - - # Dereference - def * - return nil if dest_addr == 0 - @type.new(dest_addr) - end - - # Array access - def [](index) - (self + index).* - end - - # Array set - # @param index [Integer] - # @param value [Integer, RubyVM::MJIT::CPointer::Struct] an address itself or an object that return an address with to_i - def []=(index, value) - Fiddle::Pointer.new(@addr + index * Fiddle::SIZEOF_VOIDP)[0, Fiddle::SIZEOF_VOIDP] = - [value.to_i].pack(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP]) - end - - # Get a raw address - def to_i - @addr - end - - private - - def dest_addr - Fiddle::Pointer.new(@addr)[0, Fiddle::SIZEOF_VOIDP].unpack1(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP]) - end - - def self.define(block) - Class.new(self) do - define_method(:initialize) do |addr| - super(addr, block.call) - end - - # Type-level []=: Used by struct fields - # @param addr [Integer] - # @param value [Integer, RubyVM::MJIT::CPointer::Struct] an address itself, or an object that return an address with to_i - define_singleton_method(:[]=) do |addr, value| - value = value.to_i - Fiddle::Pointer.new(addr)[0, Fiddle::SIZEOF_VOIDP] = [value].pack(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP]) - end - end - end - end - - class BitField - # @param addr [Integer] - # @param width [Integer] - # @param offset [Integer] - def initialize(addr, width, offset) - @addr = addr - @width = width - @offset = offset - end - - # Dereference - def * - byte = Fiddle::Pointer.new(@addr)[0, Fiddle::SIZEOF_CHAR].unpack('c').first - if @width == 1 - bit = (1 & (byte >> @offset)) - bit == 1 - elsif @width <= 8 && @offset == 0 - bitmask = @width.times.map { |i| 1 << i }.sum - byte & bitmask - else - raise NotImplementedError.new("not-implemented bit field access: width=#{@width} offset=#{@offset}") - end - end - - # @param width [Integer] - # @param offset [Integer] - def self.define(width, offset) - Class.new(self) do - define_method(:initialize) do |addr| - super(addr, width, offset) - end - end - end - end - - # Give a name to a dynamic CPointer class to see it on inspect - def self.with_class_name(prefix, name, cache: false, &block) - return block.call if name.empty? - - # Use a cached result only if cache: true - class_name = "#{prefix}_#{name}" - klass = - if cache && self.const_defined?(class_name) - self.const_get(class_name) - else - block.call - end - - # Give it a name unless it's already defined - unless self.const_defined?(class_name) - self.const_set(class_name, klass) - end - - klass - end - end -end diff --git a/lib/ruby_vm/mjit/c_type.rb b/lib/ruby_vm/mjit/c_type.rb deleted file mode 100644 index 9e45d8d41c..0000000000 --- a/lib/ruby_vm/mjit/c_type.rb +++ /dev/null @@ -1,91 +0,0 @@ -require 'fiddle' -require 'fiddle/pack' -require_relative 'c_pointer' - -module RubyVM::MJIT - module CType - module Struct - # @param name [String] - # @param members [Hash{ Symbol => [Integer, RubyVM::MJIT::CType::*] }] - def self.new(name, sizeof, **members) - name = members.keys.join('_') if name.empty? - CPointer.with_class_name('Struct', name) do - CPointer::Struct.define(sizeof, members) - end - end - end - - module Union - # @param name [String] - # @param members [Hash{ Symbol => RubyVM::MJIT::CType::* }] - def self.new(name, sizeof, **members) - name = members.keys.join('_') if name.empty? - CPointer.with_class_name('Union', name) do - CPointer::Union.define(sizeof, members) - end - end - end - - module Immediate - # @param fiddle_type [Integer] - def self.new(fiddle_type) - name = Fiddle.constants.find do |const| - const.start_with?('TYPE_') && Fiddle.const_get(const) == fiddle_type.abs - end&.to_s - name.delete_prefix!('TYPE_') - if fiddle_type.negative? - name.prepend('U') - end - CPointer.with_class_name('Immediate', name, cache: true) do - CPointer::Immediate.define(fiddle_type) - end - end - - # @param type [String] - def self.parse(ctype) - new(Fiddle::Importer.parse_ctype(ctype)) - end - - def self.find(size, signed) - fiddle_type = TYPE_MAP.fetch(size) - fiddle_type = -fiddle_type unless signed - new(fiddle_type) - end - - TYPE_MAP = Fiddle::PackInfo::SIZE_MAP.map { |type, size| [size, type.abs] }.to_h - private_constant :TYPE_MAP - end - - module Bool - def self.new - CPointer::Bool - end - end - - class Pointer - # This takes a block to avoid "stack level too deep" on a cyclic reference - # @param block [Proc] - def self.new(&block) - CPointer.with_class_name('Pointer', block.object_id.to_s) do - CPointer::Pointer.define(block) - end - end - end - - module BitField - # @param width [Integer] - # @param offset [Integer] - def self.new(width, offset) - CPointer.with_class_name('BitField', "#{offset}_#{width}") do - CPointer::BitField.define(width, offset) - end - end - end - - # Types that are referenced but not part of code generation targets - Stub = ::Struct.new(:name) - - # Types that it failed to figure out from the header - Unknown = Module.new - end -end diff --git a/lib/ruby_vm/mjit/code_block.rb b/lib/ruby_vm/mjit/code_block.rb deleted file mode 100644 index 21ae2386b7..0000000000 --- a/lib/ruby_vm/mjit/code_block.rb +++ /dev/null @@ -1,86 +0,0 @@ -module RubyVM::MJIT - class CodeBlock - # @param mem_block [Integer] JIT buffer address - # @param mem_size [Integer] JIT buffer size - # @param outliend [TrueClass,FalseClass] true for outlined CodeBlock - def initialize(mem_block:, mem_size:, outlined: false) - @comments = Hash.new { |h, k| h[k] = [] } - @mem_block = mem_block - @mem_size = mem_size - @write_pos = 0 - @outlined = outlined - end - - # @param asm [RubyVM::MJIT::Assembler] - def write(asm) - return 0 if @write_pos + asm.size >= @mem_size - - start_addr = write_addr - - # Write machine code - C.mjit_mark_writable - @write_pos += asm.assemble(start_addr) - C.mjit_mark_executable - - end_addr = write_addr - - # Convert comment indexes to addresses - asm.comments.each do |index, comments| - @comments[start_addr + index] += comments - end - asm.comments.clear - - # Dump disasm if --mjit-dump-disasm - if C.mjit_opts.dump_disasm && start_addr < end_addr - dump_disasm(start_addr, end_addr) - end - start_addr - end - - def set_write_addr(addr) - @write_pos = addr - @mem_block - @comments.delete(addr) # TODO: clean up old comments for all the overwritten range? - end - - def with_write_addr(addr) - old_write_pos = @write_pos - set_write_addr(addr) - yield - ensure - @write_pos = old_write_pos - end - - def write_addr - @mem_block + @write_pos - end - - def include?(addr) - (@mem_block...(@mem_block + @mem_size)).include?(addr) - end - - private - - def dump_disasm(from, to) - C.dump_disasm(from, to).each do |address, mnemonic, op_str| - @comments.fetch(address, []).each do |comment| - puts colorize(" # #{comment}", bold: true) - end - puts colorize(" 0x#{format("%x", address)}: #{mnemonic} #{op_str}") - end - puts - end - - def colorize(text, bold: false) - buf = +'' - buf << "\e[1m" if bold - buf << "\e[34m" if @outlined - buf << text - buf << "\e[0m" - buf - end - - def bold(text) - "\e[1m#{text}\e[0m" - end - end -end diff --git a/lib/ruby_vm/mjit/compiler.rb b/lib/ruby_vm/mjit/compiler.rb deleted file mode 100644 index 710c3fd23d..0000000000 --- a/lib/ruby_vm/mjit/compiler.rb +++ /dev/null @@ -1,291 +0,0 @@ -require 'ruby_vm/mjit/assembler' -require 'ruby_vm/mjit/block' -require 'ruby_vm/mjit/branch_stub' -require 'ruby_vm/mjit/code_block' -require 'ruby_vm/mjit/context' -require 'ruby_vm/mjit/exit_compiler' -require 'ruby_vm/mjit/insn_compiler' -require 'ruby_vm/mjit/instruction' -require 'ruby_vm/mjit/invariants' -require 'ruby_vm/mjit/jit_state' - -module RubyVM::MJIT - # Compilation status - KeepCompiling = :KeepCompiling - CantCompile = :CantCompile - EndBlock = :EndBlock - - # Ruby constants - Qtrue = Fiddle::Qtrue - Qfalse = Fiddle::Qfalse - Qnil = Fiddle::Qnil - Qundef = Fiddle::Qundef - - # Callee-saved registers - # TODO: support using r12/r13 here - EC = :r14 - CFP = :r15 - SP = :rbx - - # Scratch registers: rax, rcx - - class Compiler - attr_accessor :write_pos - - IseqBlocks = Hash.new do |iseq_pc_ctx, iseq| - iseq_pc_ctx[iseq] = Hash.new do |pc_ctx, pc| - pc_ctx[pc] = {} - end - end - DeadBlocks = [] # invalidated IseqBlocks, but kept for safety - - def self.reset_blocks - DeadBlocks << IseqBlocks.dup - IseqBlocks.clear - end - - def self.decode_insn(encoded) - INSNS.fetch(C.rb_vm_insn_decode(encoded)) - end - - # @param mem_block [Integer] JIT buffer address - # @param mem_size [Integer] JIT buffer size - def initialize(mem_block, mem_size) - @cb = CodeBlock.new(mem_block: mem_block, mem_size: mem_size / 2) - @ocb = CodeBlock.new(mem_block: mem_block + mem_size / 2, mem_size: mem_size / 2, outlined: true) - @exit_compiler = ExitCompiler.new - @insn_compiler = InsnCompiler.new(@cb, @ocb, @exit_compiler) - Invariants.initialize(@cb, @ocb, self, @exit_compiler) - - @leave_exit = Assembler.new.then do |asm| - @exit_compiler.compile_leave_exit(asm) - @ocb.write(asm) - end - end - - # Compile an ISEQ from its entry point. - # @param iseq `RubyVM::MJIT::CPointer::Struct_rb_iseq_t` - # @param cfp `RubyVM::MJIT::CPointer::Struct_rb_control_frame_t` - def compile(iseq, cfp) - # TODO: Support has_opt - return if iseq.body.param.flags.has_opt - - jit = JITState.new(iseq:, cfp:) - asm = Assembler.new - asm.comment("Block: #{iseq.body.location.label}@#{C.rb_iseq_path(iseq)}:#{iseq.body.location.first_lineno}") - compile_prologue(asm) - compile_block(asm, jit:) - iseq.body.jit_func = @cb.write(asm) - rescue Exception => e - $stderr.puts e.full_message - exit 1 - end - - # Compile a branch stub. - # @param branch_stub [RubyVM::MJIT::BranchStub] - # @param cfp `RubyVM::MJIT::CPointer::Struct_rb_control_frame_t` - # @param target0_p [TrueClass,FalseClass] - # @return [Integer] The starting address of the compiled branch stub - def branch_stub_hit(branch_stub, cfp, target0_p) - # Update cfp->pc for `jit.at_current_insn?` - target = target0_p ? branch_stub.target0 : branch_stub.target1 - cfp.pc = target.pc - - # Reuse an existing block if it already exists - block = find_block(branch_stub.iseq, target.pc, target.ctx) - - # If the branch stub's jump is the last code, allow overwriting part of - # the old branch code with the new block code. - fallthrough = block.nil? && @cb.write_addr == branch_stub.end_addr - if fallthrough - # If the branch stub's jump is the last code, allow overwriting part of - # the old branch code with the new block code. - @cb.set_write_addr(branch_stub.start_addr) - branch_stub.shape = target0_p ? Next0 : Next1 - Assembler.new.tap do |branch_asm| - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - - # Reuse or generate a block - if block - target.address = block.start_addr - else - jit = JITState.new(iseq: branch_stub.iseq, cfp:) - target.address = Assembler.new.then do |asm| - compile_block(asm, jit:, pc: target.pc, ctx: target.ctx.dup) - @cb.write(asm) - end - block = jit.block - end - block.incoming << branch_stub # prepare for invalidate_block - - # Re-generate the branch code for non-fallthrough cases - unless fallthrough - @cb.with_write_addr(branch_stub.start_addr) do - branch_asm = Assembler.new - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - - return target.address - rescue Exception => e - $stderr.puts e.full_message - exit 1 - end - - # @param iseq `RubyVM::MJIT::CPointer::Struct_rb_iseq_t` - # @param pc [Integer] - def invalidate_blocks(iseq, pc) - list_blocks(iseq, pc).each do |block| - invalidate_block(iseq, block) - end - end - - private - - # Callee-saved: rbx, rsp, rbp, r12, r13, r14, r15 - # Caller-saved: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11 - # - # @param asm [RubyVM::MJIT::Assembler] - def compile_prologue(asm) - asm.comment('MJIT entry point') - - # Save callee-saved registers used by JITed code - asm.push(CFP) - asm.push(EC) - asm.push(SP) - - # Move arguments EC and CFP to dedicated registers - asm.mov(EC, :rdi) - asm.mov(CFP, :rsi) - - # Load sp to a dedicated register - asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # rbx = cfp->sp - - # Setup cfp->jit_return - asm.mov(:rax, @leave_exit) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:jit_return)], :rax) - end - - # @param asm [RubyVM::MJIT::Assembler] - def compile_block(asm, jit:, pc: jit.iseq.body.iseq_encoded.to_i, ctx: Context.new) - # Mark the block start address and prepare an exit code storage - block = Block.new(pc:, ctx: ctx.dup) - jit.block = block - asm.block(block) - - # Compile each insn - iseq = jit.iseq - index = (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size - while index < iseq.body.iseq_size - insn = self.class.decode_insn(iseq.body.iseq_encoded[index]) - jit.pc = (iseq.body.iseq_encoded + index).to_i - - # If previous instruction requested to record the boundary - if jit.record_boundary_patch_point - # Generate an exit to this instruction and record it - exit_pos = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_side_exit(jit.pc, ctx, ocb_asm) - @ocb.write(ocb_asm) - end - Invariants.record_global_inval_patch(asm, exit_pos) - jit.record_boundary_patch_point = false - end - - case status = @insn_compiler.compile(jit, ctx, asm, insn) - when KeepCompiling - index += insn.len - when EndBlock - # TODO: pad nops if entry exit exists (not needed for x86_64?) - break - when CantCompile - @exit_compiler.compile_side_exit(jit.pc, ctx, asm) - - # If this is the first instruction, this block never needs to be invalidated. - if block.pc == iseq.body.iseq_encoded.to_i + index * C.VALUE.size - block.invalidated = true - end - - break - else - raise "compiling #{insn.name} returned unexpected status: #{status.inspect}" - end - end - - incr_counter(:compiled_block_count) - set_block(iseq, block) - end - - def incr_counter(name) - if C.mjit_opts.stats - C.rb_mjit_counters[name][0] += 1 - end - end - - def invalidate_block(iseq, block) - # Remove this block from the version array - remove_block(iseq, block) - - # Invalidate the block with entry exit - unless block.invalidated - @cb.with_write_addr(block.start_addr) do - asm = Assembler.new - asm.comment('invalidate_block') - asm.jmp(block.entry_exit) - @cb.write(asm) - end - block.invalidated = true - end - - # Re-stub incoming branches - block.incoming.each do |branch_stub| - target = [branch_stub.target0, branch_stub.target1].compact.find do |target| - target.pc == block.pc && target.ctx == block.ctx - end - next if target.nil? - # TODO: Could target.address be a stub address? Is invalidation not needed in that case? - - # If the target being re-generated is currently a fallthrough block, - # the fallthrough code must be rewritten with a jump to the stub. - if target.address == branch_stub.end_addr - branch_stub.shape = Default - end - - target.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(block.ctx, ocb_asm, branch_stub, target == branch_stub.target0) - @ocb.write(ocb_asm) - end - @cb.with_write_addr(branch_stub.start_addr) do - branch_asm = Assembler.new - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - # TODO: Reset jit_func and total_calls if it's the first block after prelude - end - - def list_blocks(iseq, pc) - IseqBlocks[iseq.to_i][pc].values - end - - # @param [Integer] pc - # @param [RubyVM::MJIT::Context] ctx - # @return [RubyVM::MJIT::Block,NilClass] - def find_block(iseq, pc, ctx) - IseqBlocks[iseq.to_i][pc][ctx] - end - - # @param [RubyVM::MJIT::Block] block - def set_block(iseq, block) - IseqBlocks[iseq.to_i][block.pc][block.ctx] = block - end - - # @param [RubyVM::MJIT::Block] block - def remove_block(iseq, block) - IseqBlocks[iseq.to_i][block.pc].delete(block.ctx) - end - end -end diff --git a/lib/ruby_vm/mjit/context.rb b/lib/ruby_vm/mjit/context.rb deleted file mode 100644 index e834b42999..0000000000 --- a/lib/ruby_vm/mjit/context.rb +++ /dev/null @@ -1,30 +0,0 @@ -module RubyVM::MJIT - class Context < Struct.new( - :stack_size, # @param [Integer] The number of values on the stack - :sp_offset, # @param [Integer] JIT sp offset relative to the interpreter's sp - :chain_depth, # @param [Integer] jit_chain_guard depth - ) - def initialize(stack_size: 0, sp_offset: 0, chain_depth: 0) = super - - def stack_push(size = 1) - self.stack_size += size - self.sp_offset += size - stack_opnd(0) - end - - def stack_pop(size = 1) - opnd = stack_opnd(0) - self.stack_size -= size - self.sp_offset -= size - opnd - end - - def stack_opnd(depth_from_top) - [SP, C.VALUE.size * (self.sp_offset - 1 - depth_from_top)] - end - - def sp_opnd(offset_bytes = 0) - [SP, (C.VALUE.size * self.sp_offset) + offset_bytes] - end - end -end diff --git a/lib/ruby_vm/mjit/exit_compiler.rb b/lib/ruby_vm/mjit/exit_compiler.rb deleted file mode 100644 index bf85a340d7..0000000000 --- a/lib/ruby_vm/mjit/exit_compiler.rb +++ /dev/null @@ -1,146 +0,0 @@ -module RubyVM::MJIT - class ExitCompiler - def initialize - @gc_refs = [] # TODO: GC offsets? - end - - # Used for invalidating a block on entry. - # @param pc [Integer] - # @param asm [RubyVM::MJIT::Assembler] - def compile_entry_exit(pc, ctx, asm, cause:) - # Increment per-insn exit counter - incr_insn_exit(pc, asm) - - # Fix pc/sp offsets for the interpreter - save_pc_and_sp(pc, ctx, asm, reset_sp_offset: false) - - # Restore callee-saved registers - asm.comment("#{cause}: entry exit") - asm.pop(SP) - asm.pop(EC) - asm.pop(CFP) - - asm.mov(C_RET, Qundef) - asm.ret - end - - # Set to cfp->jit_return by default for leave insn - # @param asm [RubyVM::MJIT::Assembler] - def compile_leave_exit(asm) - asm.comment('default cfp->jit_return') - - # Restore callee-saved registers - asm.pop(SP) - asm.pop(EC) - asm.pop(CFP) - - # :rax is written by #leave - asm.ret - end - - # Fire cfunc events on invalidation by TracePoint - # @param asm [RubyVM::MJIT::Assembler] - def compile_full_cfunc_return(asm) - # This chunk of code expects REG_EC to be filled properly and - # RAX to contain the return value of the C method. - - asm.comment('full cfunc return') - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], :rax) - asm.call(C.rb_full_cfunc_return) - - # TODO: count the exit - - # Restore callee-saved registers - asm.pop(SP) - asm.pop(EC) - asm.pop(CFP) - - asm.mov(C_RET, Qundef) - asm.ret - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def compile_side_exit(pc, ctx, asm) - # Increment per-insn exit counter - incr_insn_exit(pc, asm) - - # Fix pc/sp offsets for the interpreter - save_pc_and_sp(pc, ctx.dup, asm) # dup to avoid sp_offset update - - # Restore callee-saved registers - asm.comment("exit to interpreter on #{pc_to_insn(pc).name}") - asm.pop(SP) - asm.pop(EC) - asm.pop(CFP) - - asm.mov(C_RET, Qundef) - asm.ret - end - - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - # @param branch_stub [RubyVM::MJIT::BranchStub] - # @param target0_p [TrueClass,FalseClass] - def compile_branch_stub(ctx, asm, branch_stub, target0_p) - # Call rb_mjit_branch_stub_hit - asm.comment("branch stub hit: #{branch_stub.iseq.body.location.label}@#{C.rb_iseq_path(branch_stub.iseq)}:#{iseq_lineno(branch_stub.iseq, target0_p ? branch_stub.target0.pc : branch_stub.target1.pc)}") - asm.mov(:rdi, to_value(branch_stub)) - asm.mov(:esi, ctx.sp_offset) - asm.mov(:edx, target0_p ? 1 : 0) - asm.call(C.rb_mjit_branch_stub_hit) - - # Jump to the address returned by rb_mjit_stub_hit - asm.jmp(:rax) - end - - private - - def pc_to_insn(pc) - Compiler.decode_insn(C.VALUE.new(pc).*) - end - - # @param pc [Integer] - # @param asm [RubyVM::MJIT::Assembler] - def incr_insn_exit(pc, asm) - if C.mjit_opts.stats - insn = Compiler.decode_insn(C.VALUE.new(pc).*) - asm.comment("increment insn exit: #{insn.name}") - asm.mov(:rax, (C.mjit_insn_exits + insn.bin).to_i) - asm.add([:rax], 1) # TODO: lock - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def save_pc_and_sp(pc, ctx, asm, reset_sp_offset: true) - # Update pc (TODO: manage PC offset?) - asm.comment("save PC#{' and SP' if ctx.sp_offset != 0} to CFP") - asm.mov(:rax, pc) # rax = jit.pc - asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) # cfp->pc = rax - - # Update sp - if ctx.sp_offset != 0 - asm.add(SP, C.VALUE.size * ctx.sp_offset) # sp += stack_size - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) # cfp->sp = sp - if reset_sp_offset - ctx.sp_offset = 0 - end - end - end - - def to_value(obj) - @gc_refs << obj - C.to_value(obj) - end - - def iseq_lineno(iseq, pc) - C.rb_iseq_line_no(iseq, (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size) - rescue RangeError # bignum too big to convert into `unsigned long long' (RangeError) - -1 - end - end -end diff --git a/lib/ruby_vm/mjit/hooks.rb b/lib/ruby_vm/mjit/hooks.rb deleted file mode 100644 index f3943504fd..0000000000 --- a/lib/ruby_vm/mjit/hooks.rb +++ /dev/null @@ -1,30 +0,0 @@ -module RubyVM::MJIT - module Hooks # :nodoc: all - def self.on_bop_redefined(_redefined_flag, _bop) - # C.mjit_cancel_all("BOP is redefined") - end - - def self.on_cme_invalidate(cme) - cme = C.rb_callable_method_entry_struct.new(cme) - Invariants.on_cme_invalidate(cme) - end - - def self.on_ractor_spawn - # C.mjit_cancel_all("Ractor is spawned") - end - - def self.on_constant_state_changed(_id) - # to be used later - end - - def self.on_constant_ic_update(iseq, ic, insn_idx) - iseq = C.rb_iseq_t.new(iseq) - ic = C.IC.new(ic) - Invariants.on_constant_ic_update(iseq, ic, insn_idx) - end - - def self.on_tracing_invalidate_all(_new_iseq_events) - Invariants.on_tracing_invalidate_all - end - end -end diff --git a/lib/ruby_vm/mjit/insn_compiler.rb b/lib/ruby_vm/mjit/insn_compiler.rb deleted file mode 100644 index 0ed0924519..0000000000 --- a/lib/ruby_vm/mjit/insn_compiler.rb +++ /dev/null @@ -1,2515 +0,0 @@ -module RubyVM::MJIT - class InsnCompiler - # @param ocb [CodeBlock] - # @param exit_compiler [RubyVM::MJIT::ExitCompiler] - def initialize(cb, ocb, exit_compiler) - @ocb = ocb - @exit_compiler = exit_compiler - @gc_refs = [] # TODO: GC offsets? - - @full_cfunc_return = Assembler.new.then do |asm| - @exit_compiler.compile_full_cfunc_return(asm) - @ocb.write(asm) - end - - # freeze # workaround a binding.irb issue. TODO: resurrect this - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - # @param insn `RubyVM::MJIT::Instruction` - def compile(jit, ctx, asm, insn) - asm.incr_counter(:mjit_insns_count) - asm.comment("Insn: #{insn.name}") - - # 55/101 - case insn.name - when :nop then nop(jit, ctx, asm) - when :getlocal then getlocal(jit, ctx, asm) - when :setlocal then setlocal(jit, ctx, asm) - # getblockparam - # setblockparam - # getblockparamproxy - # getspecial - # setspecial - when :getinstancevariable then getinstancevariable(jit, ctx, asm) - when :setinstancevariable then setinstancevariable(jit, ctx, asm) - # getclassvariable - # setclassvariable - when :opt_getconstant_path then opt_getconstant_path(jit, ctx, asm) - # getconstant - # setconstant - # getglobal - # setglobal - when :putnil then putnil(jit, ctx, asm) - when :putself then putself(jit, ctx, asm) - when :putobject then putobject(jit, ctx, asm) - # putspecialobject - # putstring - # concatstrings - # anytostring - # toregexp - # intern - when :newarray then newarray(jit, ctx, asm) - # newarraykwsplat - when :duparray then duparray(jit, ctx, asm) - # duphash - when :expandarray then expandarray(jit, ctx, asm) - # concatarray - when :splatarray then splatarray(jit, ctx, asm) - # newhash - # newrange - when :pop then pop(jit, ctx, asm) - when :dup then dup(jit, ctx, asm) - when :dupn then dupn(jit, ctx, asm) - # swap - # opt_reverse - when :topn then topn(jit, ctx, asm) - when :setn then setn(jit, ctx, asm) - when :adjuststack then adjuststack(jit, ctx, asm) - # defined - # checkmatch - # checkkeyword - # checktype - # defineclass - # definemethod - # definesmethod - # send - when :opt_send_without_block then opt_send_without_block(jit, ctx, asm) - # objtostring - # opt_str_freeze - when :opt_nil_p then opt_nil_p(jit, ctx, asm) - # opt_str_uminus - # opt_newarray_max - # opt_newarray_min - # invokesuper - # invokeblock - when :leave then leave(jit, ctx, asm) - # throw - when :jump then jump(jit, ctx, asm) - when :branchif then branchif(jit, ctx, asm) - when :branchunless then branchunless(jit, ctx, asm) - # branchnil - # once - # opt_case_dispatch - when :opt_plus then opt_plus(jit, ctx, asm) - when :opt_minus then opt_minus(jit, ctx, asm) - when :opt_mult then opt_mult(jit, ctx, asm) - when :opt_div then opt_div(jit, ctx, asm) - when :opt_mod then opt_mod(jit, ctx, asm) - when :opt_eq then opt_eq(jit, ctx, asm) - when :opt_neq then opt_neq(jit, ctx, asm) - when :opt_lt then opt_lt(jit, ctx, asm) - when :opt_le then opt_le(jit, ctx, asm) - when :opt_gt then opt_gt(jit, ctx, asm) - when :opt_ge then opt_ge(jit, ctx, asm) - when :opt_ltlt then opt_ltlt(jit, ctx, asm) - when :opt_and then opt_and(jit, ctx, asm) - when :opt_or then opt_or(jit, ctx, asm) - when :opt_aref then opt_aref(jit, ctx, asm) - when :opt_aset then opt_aset(jit, ctx, asm) - # opt_aset_with - # opt_aref_with - when :opt_length then opt_length(jit, ctx, asm) - when :opt_size then opt_size(jit, ctx, asm) - when :opt_empty_p then opt_empty_p(jit, ctx, asm) - when :opt_succ then opt_succ(jit, ctx, asm) - when :opt_not then opt_not(jit, ctx, asm) - when :opt_regexpmatch2 then opt_regexpmatch2(jit, ctx, asm) - # invokebuiltin - when :opt_invokebuiltin_delegate then opt_invokebuiltin_delegate(jit, ctx, asm) - when :opt_invokebuiltin_delegate_leave then opt_invokebuiltin_delegate_leave(jit, ctx, asm) - when :getlocal_WC_0 then getlocal_WC_0(jit, ctx, asm) - when :getlocal_WC_1 then getlocal_WC_1(jit, ctx, asm) - when :setlocal_WC_0 then setlocal_WC_0(jit, ctx, asm) - when :setlocal_WC_1 then setlocal_WC_1(jit, ctx, asm) - when :putobject_INT2FIX_0_ then putobject_INT2FIX_0_(jit, ctx, asm) - when :putobject_INT2FIX_1_ then putobject_INT2FIX_1_(jit, ctx, asm) - else CantCompile - end - end - - private - - # - # Insns - # - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def nop(jit, ctx, asm) - # Do nothing - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def getlocal(jit, ctx, asm) - idx = jit.operand(0) - level = jit.operand(1) - jit_getlocal_generic(jit, ctx, asm, idx:, level:) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def setlocal(jit, ctx, asm) - idx = jit.operand(0) - level = jit.operand(1) - jit_setlocal_generic(jit, ctx, asm, idx:, level:) - end - - # getblockparam - # setblockparam - # getblockparamproxy - # getspecial - # setspecial - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def getinstancevariable(jit, ctx, asm) - # Specialize on a compile-time receiver, and split a block for chain guards - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - id = jit.operand(0) - comptime_obj = jit.peek_at_self - - jit_getivar(jit, ctx, asm, comptime_obj, id) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def setinstancevariable(jit, ctx, asm) - id = jit.operand(0) - ivc = jit.operand(1) - - # rb_vm_setinstancevariable could raise exceptions - jit_prepare_routine_call(jit, ctx, asm) - - val_opnd = ctx.stack_pop - - asm.comment('rb_vm_setinstancevariable') - asm.mov(:rdi, jit.iseq.to_i) - asm.mov(:rsi, [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(:rdx, id) - asm.mov(:rcx, val_opnd) - asm.mov(:r8, ivc) - asm.call(C.rb_vm_setinstancevariable) - - KeepCompiling - end - - # getclassvariable - # setclassvariable - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_getconstant_path(jit, ctx, asm) - # Cut the block for invalidation - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - ic = C.iseq_inline_constant_cache.new(jit.operand(0)) - idlist = ic.segments - - # Make sure there is an exit for this block as the interpreter might want - # to invalidate this block from rb_mjit_constant_ic_update(). - # For now, we always take an entry exit even if it was a side exit. - Invariants.ensure_block_entry_exit(jit, cause: 'opt_getconstant_path') - - # See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). - ice = ic.entry - if ice.nil? - # In this case, leave a block that unconditionally side exits - # for the interpreter to invalidate. - asm.incr_counter(:optgetconst_not_cached) - return CantCompile - end - - if ice.ic_cref # with cref - # Not supported yet - asm.incr_counter(:optgetconst_cref) - return CantCompile - else # without cref - # TODO: implement this - # Optimize for single ractor mode. - # if !assume_single_ractor_mode(jit, ocb) - # return CantCompile - # end - - # Invalidate output code on any constant writes associated with - # constants referenced within the current block. - #assume_stable_constant_names(jit, ocb, idlist); - - putobject(jit, ctx, asm, val: ice.value) - end - - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # getconstant - # setconstant - # getglobal - # setglobal - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def putnil(jit, ctx, asm) - putobject(jit, ctx, asm, val: Qnil) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def putself(jit, ctx, asm) - stack_top = ctx.stack_push - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(stack_top, :rax) - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def putobject(jit, ctx, asm, val: jit.operand(0)) - # Push it to the stack - stack_top = ctx.stack_push - if asm.imm32?(val) - asm.mov(stack_top, val) - else # 64-bit immediates can't be directly written to memory - asm.mov(:rax, val) - asm.mov(stack_top, :rax) - end - # TODO: GC offsets? - - KeepCompiling - end - - # putspecialobject - # putstring - # concatstrings - # anytostring - # toregexp - # intern - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def newarray(jit, ctx, asm) - n = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - # If n is 0, then elts is never going to be read, so we can just pass null - if n == 0 - values_ptr = 0 - else - asm.comment('load pointer to array elts') - offset_magnitude = C.VALUE.size * n - values_opnd = ctx.sp_opnd(-(offset_magnitude)) - asm.lea(:rax, values_opnd) - values_ptr = :rax - end - - # call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts); - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], n) - asm.mov(C_ARGS[2], values_ptr) - asm.call(C.rb_ec_ary_new_from_values) - - ctx.stack_pop(n) - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # newarraykwsplat - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def duparray(jit, ctx, asm) - ary = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - # call rb_ary_resurrect(VALUE ary); - asm.comment('call rb_ary_resurrect') - asm.mov(C_ARGS[0], ary) - asm.call(C.rb_ary_resurrect) - - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # duphash - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def expandarray(jit, ctx, asm) - # Both arguments are rb_num_t which is unsigned - num = jit.operand(0) - flag = jit.operand(1) - - # If this instruction has the splat flag, then bail out. - if flag & 0x01 != 0 - asm.incr_counter(:expandarray_splat) - return CantCompile - end - - # If this instruction has the postarg flag, then bail out. - if flag & 0x02 != 0 - asm.incr_counter(:expandarray_postarg) - return CantCompile - end - - side_exit = side_exit(jit, ctx) - - array_opnd = ctx.stack_pop(1) - - # num is the number of requested values. If there aren't enough in the - # array then we're going to push on nils. - # TODO: implement this - - # Move the array from the stack and check that it's an array. - asm.mov(:rax, array_opnd) - guard_object_is_heap(asm, :rax, counted_exit(side_exit, :expandarray_not_array)) - guard_object_is_array(asm, :rax, :rcx, counted_exit(side_exit, :expandarray_not_array)) - - # If we don't actually want any values, then just return. - if num == 0 - return KeepCompiling - end - - jit_array_len(asm, :rax, :rcx) - - # Only handle the case where the number of values in the array is greater - # than or equal to the number of values requested. - asm.cmp(:rcx, num) - asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small)) - - # Conditionally load the address of the heap array into REG1. - # (struct RArray *)(obj)->as.heap.ptr - #asm.mov(:rax, array_opnd) - asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) - asm.test(:rcx, C.RARRAY_EMBED_FLAG); - asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)]) - - # Load the address of the embedded array into REG1. - # (struct RArray *)(obj)->as.ary - asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)]) - - asm.cmovnz(:rcx, :rax) - - # Loop backward through the array and push each element onto the stack. - (num - 1).downto(0).each do |i| - top = ctx.stack_push - asm.mov(:rax, [:rcx, i * C.VALUE.size]) - asm.mov(top, :rax) - end - - KeepCompiling - end - - # concatarray - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def splatarray(jit, ctx, asm) - flag = jit.operand(0) - - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - ary_opnd = ctx.stack_pop(1) - - # Call rb_vm_splat_array(flag, ary) - asm.mov(C_ARGS[0], flag) - asm.mov(C_ARGS[1], ary_opnd) - asm.call(C.rb_vm_splat_array) - - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # newhash - # newrange - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def pop(jit, ctx, asm) - ctx.stack_pop - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def dup(jit, ctx, asm) - val1 = ctx.stack_opnd(0) - val2 = ctx.stack_push - asm.mov(:rax, val1) - asm.mov(val2, :rax) - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def dupn(jit, ctx, asm) - n = jit.operand(0) - - # In practice, seems to be only used for n==2 - if n != 2 - return CantCompile - end - - opnd1 = ctx.stack_opnd(1) - opnd0 = ctx.stack_opnd(0) - - dst1 = ctx.stack_push - asm.mov(:rax, opnd1) - asm.mov(dst1, :rax) - - dst0 = ctx.stack_push - asm.mov(:rax, opnd0) - asm.mov(dst0, :rax) - - KeepCompiling - end - - # swap - # opt_reverse - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def topn(jit, ctx, asm) - n = jit.operand(0) - - top_n_val = ctx.stack_opnd(n) - loc0 = ctx.stack_push - asm.mov(:rax, top_n_val) - asm.mov(loc0, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def setn(jit, ctx, asm) - n = jit.operand(0) - - top_val = ctx.stack_pop(0) - dst_opnd = ctx.stack_opnd(n) - asm.mov(:rax, top_val) - asm.mov(dst_opnd, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def adjuststack(jit, ctx, asm) - n = jit.operand(0) - ctx.stack_pop(n) - KeepCompiling - end - - # defined - # checkmatch - # checkkeyword - # checktype - # defineclass - # definemethod - # definesmethod - # send - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - # @param cd `RubyVM::MJIT::CPointer::Struct_rb_call_data` - def opt_send_without_block(jit, ctx, asm) - cd = C.rb_call_data.new(jit.operand(0)) - jit_call_general(jit, ctx, asm, cd) - end - - # objtostring - # opt_str_freeze - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_nil_p(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # opt_str_uminus - # opt_newarray_max - # opt_newarray_min - # invokesuper - # invokeblock - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def leave(jit, ctx, asm) - assert_equal(ctx.stack_size, 1) - - jit_check_ints(jit, ctx, asm) - - asm.comment('pop stack frame') - asm.lea(:rax, [CFP, C.rb_control_frame_t.size]) - asm.mov(CFP, :rax) - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], :rax) - - # Return a value (for compile_leave_exit) - ret_opnd = ctx.stack_pop - asm.mov(:rax, ret_opnd) - - # Set caller's SP and push a value to its stack (for JIT) - asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # Note: SP is in the position after popping a receiver and arguments - asm.mov([SP], :rax) - - # Jump to cfp->jit_return - asm.jmp([CFP, -C.rb_control_frame_t.size + C.rb_control_frame_t.offsetof(:jit_return)]) - - EndBlock - end - - # throw - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jump(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) - stub_next_block(jit.iseq, pc, ctx, asm) - EndBlock - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def branchif(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - # TODO: skip check for known truthy - - # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. - val = ctx.stack_pop - asm.test(val, ~Qnil) - - # Set stubs - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)), # branch target - target1: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * jit.insn.len), # fallthrough - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.target1.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) - @ocb.write(ocb_asm) - end - - # Jump to target0 on jnz - branch_stub.compile = proc do |branch_asm| - branch_asm.comment("branchif #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jnz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jz(branch_stub.target1.address) - in Next1 - branch_asm.jnz(branch_stub.target0.address) - end - end - end - branch_stub.compile.call(asm) - - EndBlock - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def branchunless(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - # TODO: skip check for known truthy - - # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. - val = ctx.stack_pop - asm.test(val, ~Qnil) - - # Set stubs - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)), # branch target - target1: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * jit.insn.len), # fallthrough - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.target1.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) - @ocb.write(ocb_asm) - end - - # Jump to target0 on jz - branch_stub.compile = proc do |branch_asm| - branch_asm.comment("branchunless #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jnz(branch_stub.target1.address) - in Next1 - branch_asm.jz(branch_stub.target0.address) - end - end - end - branch_stub.compile.call(asm) - - EndBlock - end - - # branchnil - # once - # opt_case_dispatch - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_plus(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - # Generate a side exit before popping operands - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_PLUS) - return CantCompile - end - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.comment('guard recv is fixnum') # TODO: skip this with type information - asm.test(recv_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.comment('guard obj is fixnum') # TODO: skip this with type information - asm.test(obj_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.mov(:rax, recv_opnd) - asm.sub(:rax, 1) # untag - asm.mov(:rcx, obj_opnd) - asm.add(:rax, :rcx) - asm.jo(side_exit) - - dst_opnd = ctx.stack_push - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_minus(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - # Generate a side exit before popping operands - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_MINUS) - return CantCompile - end - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.comment('guard recv is fixnum') # TODO: skip this with type information - asm.test(recv_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.comment('guard obj is fixnum') # TODO: skip this with type information - asm.test(obj_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.mov(:rax, recv_opnd) - asm.mov(:rcx, obj_opnd) - asm.sub(:rax, :rcx) - asm.jo(side_exit) - asm.add(:rax, 1) # re-tag - - dst_opnd = ctx.stack_push - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_mult(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_div(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_mod(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - # Create a side-exit to fall back to the interpreter - # Note: we generate the side-exit before popping operands from the stack - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_MOD) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm, side_exit) - - # Get the operands and destination from the stack - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - # Check for arg0 % 0 - asm.cmp(arg1, 0) - asm.je(side_exit) - - # Call rb_fix_mod_fix(VALUE recv, VALUE obj) - asm.mov(C_ARGS[0], arg0) - asm.mov(C_ARGS[1], arg1) - asm.call(C.rb_fix_mod_fix) - - # Push the return value onto the stack - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_eq(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if jit_equality_specialized(jit, ctx, asm) - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_neq(jit, ctx, asm) - # opt_neq is passed two rb_call_data as arguments: - # first for ==, second for != - neq_cd = C.rb_call_data.new(jit.operand(1)) - jit_call_general(jit, ctx, asm, neq_cd) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_lt(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovl, bop: C.BOP_LT) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_le(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovle, bop: C.BOP_LE) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_gt(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovg, bop: C.BOP_GT) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_ge(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovge, bop: C.BOP_GE) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_ltlt(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_and(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - # Create a side-exit to fall back to the interpreter - # Note: we generate the side-exit before popping operands from the stack - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_AND) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm, side_exit) - - # Get the operands and destination from the stack - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - asm.comment('bitwise and') - asm.mov(:rax, arg0) - asm.and(:rax, arg1) - - # Push the return value onto the stack - dst = ctx.stack_push - asm.mov(dst, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_or(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - # Create a side-exit to fall back to the interpreter - # Note: we generate the side-exit before popping operands from the stack - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_OR) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm, side_exit) - - # Get the operands and destination from the stack - asm.comment('bitwise or') - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - # Do the bitwise or arg0 | arg1 - asm.mov(:rax, arg0) - asm.or(:rax, arg1) - - # Push the return value onto the stack - dst = ctx.stack_push - asm.mov(dst, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_aref(jit, ctx, asm) - cd = C.rb_call_data.new(jit.operand(0)) - argc = C.vm_ci_argc(cd.ci) - - if argc != 1 - asm.incr_counter(:optaref_argc_not_one) - return CantCompile - end - - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - side_exit = side_exit(jit, ctx) - - if comptime_recv.class == Array && fixnum?(comptime_obj) - unless Invariants.assume_bop_not_redefined(jit, C.ARRAY_REDEFINED_OP_FLAG, C.BOP_AREF) - return CantCompile - end - - idx_opnd = ctx.stack_opnd(0) - recv_opnd = ctx.stack_opnd(1) - - not_array_exit = counted_exit(side_exit, :optaref_recv_not_array) - if jit_guard_known_klass(jit, ctx, asm, comptime_recv.class, recv_opnd, comptime_recv, not_array_exit) == CantCompile - return CantCompile - end - - # Bail if idx is not a FIXNUM - asm.mov(:rax, idx_opnd) - asm.test(:rax, C.RUBY_FIXNUM_FLAG) - asm.jz(counted_exit(side_exit, :optaref_arg_not_fixnum)) - - # Call VALUE rb_ary_entry_internal(VALUE ary, long offset). - # It never raises or allocates, so we don't need to write to cfp->pc. - asm.sar(:rax, 1) # Convert fixnum to int - asm.mov(C_ARGS[0], recv_opnd) - asm.mov(C_ARGS[1], :rax) - asm.call(C.rb_ary_entry_internal) - - # Pop the argument and the receiver - ctx.stack_pop(2) - - # Push the return value onto the stack - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - elsif comptime_recv.class == Hash - unless Invariants.assume_bop_not_redefined(jit, C.HASH_REDEFINED_OP_FLAG, C.BOP_AREF) - return CantCompile - end - - recv_opnd = ctx.stack_opnd(1) - - # Guard that the receiver is a Hash - not_hash_exit = counted_exit(side_exit, :optaref_recv_not_hash) - if jit_guard_known_klass(jit, ctx, asm, comptime_recv.class, recv_opnd, comptime_recv, not_hash_exit) == CantCompile - return CantCompile - end - - # Prepare to call rb_hash_aref(). It might call #hash on the key. - jit_prepare_routine_call(jit, ctx, asm) - - asm.comment('call rb_hash_aref') - key_opnd = ctx.stack_opnd(0) - recv_opnd = ctx.stack_opnd(1) - asm.mov(:rdi, recv_opnd) - asm.mov(:rsi, key_opnd) - asm.call(C.rb_hash_aref) - - # Pop the key and the receiver - ctx.stack_pop(2) - - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_aset(jit, ctx, asm) - # Defer compilation so we can specialize on a runtime `self` - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(2) - comptime_key = jit.peek_at_stack(1) - - # Get the operands from the stack - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - _val = ctx.stack_opnd(0) - - if comptime_recv.class == Array && fixnum?(comptime_key) - side_exit = side_exit(jit, ctx) - - # Guard receiver is an Array - if jit_guard_known_klass(jit, ctx, asm, comptime_recv.class, recv, comptime_recv, side_exit) == CantCompile - return CantCompile - end - - # Guard key is a fixnum - if jit_guard_known_klass(jit, ctx, asm, comptime_key.class, key, comptime_key, side_exit) == CantCompile - return CantCompile - end - - # We might allocate or raise - jit_prepare_routine_call(jit, ctx, asm) - - asm.comment('call rb_ary_store') - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - val = ctx.stack_opnd(0) - asm.mov(:rax, key) - asm.sar(:rax, 1) # FIX2LONG(key) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], :rax) - asm.mov(C_ARGS[2], val) - asm.call(C.rb_ary_store) - - # rb_ary_store returns void - # stored value should still be on stack - val = ctx.stack_opnd(0) - - # Push the return value onto the stack - ctx.stack_pop(3) - stack_ret = ctx.stack_push - asm.mov(:rax, val) - asm.mov(stack_ret, :rax) - - jump_to_next_insn(jit, ctx, asm) - EndBlock - elsif comptime_recv.class == Hash - side_exit = side_exit(jit, ctx) - - # Guard receiver is a Hash - if jit_guard_known_klass(jit, ctx, asm, comptime_recv.class, recv, comptime_recv, side_exit) == CantCompile - return CantCompile - end - - # We might allocate or raise - jit_prepare_routine_call(jit, ctx, asm) - - # Call rb_hash_aset - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - val = ctx.stack_opnd(0) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], key) - asm.mov(C_ARGS[2], val) - asm.call(C.rb_hash_aset) - - # Push the return value onto the stack - ctx.stack_pop(3) - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # opt_aset_with - # opt_aref_with - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_length(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_size(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_empty_p(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_succ(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_not(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_regexpmatch2(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # invokebuiltin - - def opt_invokebuiltin_delegate(jit, ctx, asm) - bf = C.rb_builtin_function.new(jit.operand(0)) - bf_argc = bf.argc - start_index = jit.operand(1) - - # ec, self, and arguments - if bf_argc + 2 > C_ARGS.size - return CantCompile - end - - # If the calls don't allocate, do they need up to date PC, SP? - jit_prepare_routine_call(jit, ctx, asm) - - # Call the builtin func (ec, recv, arg1, arg2, ...) - asm.comment('call builtin func') - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:self)]) - - # Copy arguments from locals - if bf_argc > 0 - # Load environment pointer EP from CFP - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)]) - - bf_argc.times do |i| - table_size = jit.iseq.body.local_table_size - offs = -table_size - C.VM_ENV_DATA_SIZE + 1 + start_index + i - asm.mov(C_ARGS[2 + i], [:rax, offs * C.VALUE.size]) - end - end - asm.call(bf.func_ptr) - - # Push the return value - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def opt_invokebuiltin_delegate_leave(jit, ctx, asm) - opt_invokebuiltin_delegate(jit, ctx, asm) - # opt_invokebuiltin_delegate is always followed by leave insn - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def getlocal_WC_0(jit, ctx, asm) - # Get operands - idx = jit.operand(0) - level = 0 - - # Get EP - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)]) - - # Get a local variable - asm.mov(:rax, [:rax, -idx * C.VALUE.size]) - - # Push it to the stack - stack_top = ctx.stack_push - asm.mov(stack_top, :rax) - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def getlocal_WC_1(jit, ctx, asm) - idx = jit.operand(0) - jit_getlocal_generic(jit, ctx, asm, idx:, level: 1) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def setlocal_WC_0(jit, ctx, asm) - slot_idx = jit.operand(0) - local_idx = slot_to_local_idx(jit.iseq, slot_idx) - - # Load environment pointer EP (level 0) from CFP - ep_reg = :rax - jit_get_ep(asm, 0, reg: ep_reg) - - # Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers - # only affect heap objects being written. If we know an immediate value is being written we - # can skip this check. - - # flags & VM_ENV_FLAG_WB_REQUIRED - flags_opnd = [ep_reg, C.VALUE.size * C.VM_ENV_DATA_INDEX_FLAGS] - asm.test(flags_opnd, C.VM_ENV_FLAG_WB_REQUIRED) - - # Create a side-exit to fall back to the interpreter - side_exit = side_exit(jit, ctx) - - # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 - asm.jnz(side_exit) - - # Pop the value to write from the stack - stack_top = ctx.stack_pop(1) - - # Write the value at the environment pointer - asm.mov(:rcx, stack_top) - asm.mov([ep_reg, -8 * slot_idx], :rcx) - - KeepCompiling - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def setlocal_WC_1(jit, ctx, asm) - idx = jit.operand(0) - jit_setlocal_generic(jit, ctx, asm, idx:, level: 1) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def putobject_INT2FIX_0_(jit, ctx, asm) - putobject(jit, ctx, asm, val: C.to_value(0)) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def putobject_INT2FIX_1_(jit, ctx, asm) - putobject(jit, ctx, asm, val: C.to_value(1)) - end - - # - # Helpers - # - - def jit_getlocal_generic(jit, ctx, asm, idx:, level:) - # Load environment pointer EP at level - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Get a local variable - asm.mov(:rax, [ep_reg, -idx * C.VALUE.size]) - - # Push it to the stack - stack_top = ctx.stack_push - asm.mov(stack_top, :rax) - KeepCompiling - end - - def jit_setlocal_generic(jit, ctx, asm, idx:, level:) - # Load environment pointer EP at level - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers - # only affect heap objects being written. If we know an immediate value is being written we - # can skip this check. - - # flags & VM_ENV_FLAG_WB_REQUIRED - flags_opnd = [ep_reg, C.VALUE.size * C.VM_ENV_DATA_INDEX_FLAGS] - asm.test(flags_opnd, C.VM_ENV_FLAG_WB_REQUIRED) - - # Create a side-exit to fall back to the interpreter - side_exit = side_exit(jit, ctx) - - # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 - asm.jnz(side_exit) - - # Pop the value to write from the stack - stack_top = ctx.stack_pop(1) - - # Write the value at the environment pointer - asm.mov(:rcx, stack_top) - asm.mov([ep_reg, -(C.VALUE.size * idx)], :rcx) - - KeepCompiling - end - - # Compute the index of a local variable from its slot index - def slot_to_local_idx(iseq, slot_idx) - # Layout illustration - # This is an array of VALUE - # | VM_ENV_DATA_SIZE | - # v v - # low addr <+-------+-------+-------+-------+------------------+ - # |local 0|local 1| ... |local n| .... | - # +-------+-------+-------+-------+------------------+ - # ^ ^ ^ ^ - # +-------+---local_table_size----+ cfp->ep--+ - # | | - # +------------------slot_idx----------------+ - # - # See usages of local_var_name() from iseq.c for similar calculation. - - local_table_size = iseq.body.local_table_size - op = slot_idx - C.VM_ENV_DATA_SIZE - local_table_size - op - 1 - end - - # @param asm [RubyVM::MJIT::Assembler] - def guard_object_is_heap(asm, object_opnd, side_exit) - asm.comment('guard object is heap') - # Test that the object is not an immediate - asm.test(object_opnd, C.RUBY_IMMEDIATE_MASK) - asm.jnz(side_exit) - - # Test that the object is not false - asm.cmp(object_opnd, Qfalse) - asm.je(side_exit) - end - - # @param asm [RubyVM::MJIT::Assembler] - def guard_object_is_array(asm, object_reg, flags_reg, side_exit) - asm.comment('guard object is array') - # Pull out the type mask - asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) - asm.and(flags_reg, C.RUBY_T_MASK) - - # Compare the result with T_ARRAY - asm.cmp(flags_reg, C.RUBY_T_ARRAY) - asm.jne(side_exit) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_chain_guard(opcode, jit, ctx, asm, side_exit, limit: 10) - opcode => :je | :jne | :jnz | :jz - - if ctx.chain_depth < limit - deeper = ctx.dup - deeper.chain_depth += 1 - - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx: deeper, pc: jit.pc), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = proc do |branch_asm| - # Not using `asm.comment` here since it's usually put before cmp/test before this. - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.public_send(opcode, branch_stub.target0.address) - end - end - end - branch_stub.compile.call(asm) - else - asm.public_send(opcode, side_exit) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, comptime_obj, side_exit, limit: 5) - # Only memory operand is supported for now - assert_equal(true, obj_opnd.is_a?(Array)) - - if known_klass == NilClass - asm.comment('guard object is nil') - asm.cmp(obj_opnd, Qnil) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - elsif known_klass == TrueClass - asm.comment('guard object is true') - asm.cmp(obj_opnd, Qtrue) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - elsif known_klass == FalseClass - asm.comment('guard object is false') - asm.cmp(obj_opnd, Qfalse) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - elsif known_klass == Integer && fixnum?(comptime_obj) - asm.comment('guard object is fixnum') - asm.test(obj_opnd, C.RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit, limit:) - elsif known_klass == Symbol && static_symbol?(comptime_obj) - # We will guard STATIC vs DYNAMIC as though they were separate classes - # DYNAMIC symbols can be handled by the general else case below - asm.comment('guard object is static symbol') - assert_equal(8, C.RUBY_SPECIAL_SHIFT) - asm.cmp(BytePtr[*obj_opnd], C.RUBY_SYMBOL_FLAG) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - elsif known_klass == Float - asm.incr_counter(:send_guard_float) - return CantCompile - elsif known_klass.singleton_class? - asm.comment('guard known object with singleton class') - asm.mov(:rax, C.to_value(comptime_obj)) - asm.cmp(obj_opnd, :rax) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - else - # Load memory to a register - asm.mov(:rax, obj_opnd) - obj_opnd = :rax - - # Check that the receiver is a heap object - # Note: if we get here, the class doesn't have immediate instances. - asm.comment('guard not immediate') - asm.test(obj_opnd, C.RUBY_IMMEDIATE_MASK) - jit_chain_guard(:jnz, jit, ctx, asm, side_exit, limit:) - asm.cmp(obj_opnd, Qfalse) - jit_chain_guard(:je, jit, ctx, asm, side_exit, limit:) - - # Bail if receiver class is different from known_klass - klass_opnd = [obj_opnd, C.RBasic.offsetof(:klass)] - asm.comment("guard known class #{known_klass}") - asm.mov(:rcx, to_value(known_klass)) - asm.cmp(klass_opnd, :rcx) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - end - end - - # @param jit [RubyVM::MJIT::JITState] - def two_fixnums_on_stack?(jit) - comptime_recv = jit.peek_at_stack(1) - comptime_arg = jit.peek_at_stack(0) - return fixnum?(comptime_recv) && fixnum?(comptime_arg) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def guard_two_fixnums(jit, ctx, asm, side_exit) - # Get stack operands without popping them - arg1 = ctx.stack_opnd(0) - arg0 = ctx.stack_opnd(1) - - asm.comment('guard arg0 fixnum') - asm.test(arg0, C.RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit) - # TODO: upgrade type, and skip the check when possible - - asm.comment('guard arg1 fixnum') - asm.test(arg1, C.RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit) - # TODO: upgrade type, and skip the check when possible - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_fixnum_cmp(jit, ctx, asm, opcode:, bop:) - opcode => :cmovl | :cmovle | :cmovg | :cmovge - - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - # Generate a side exit before popping operands - side_exit = side_exit(jit, ctx) - - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, bop) - return CantCompile - end - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.comment('guard recv is fixnum') # TODO: skip this with type information - asm.test(recv_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.comment('guard obj is fixnum') # TODO: skip this with type information - asm.test(obj_opnd, C.RUBY_FIXNUM_FLAG) - asm.jz(side_exit) - - asm.mov(:rax, obj_opnd) - asm.cmp(recv_opnd, :rax) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.public_send(opcode, :rax, :rcx) - - dst_opnd = ctx.stack_push - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_equality_specialized(jit, ctx, asm) - # Create a side-exit to fall back to the interpreter - side_exit = side_exit(jit, ctx) - - a_opnd = ctx.stack_opnd(1) - b_opnd = ctx.stack_opnd(0) - - comptime_a = jit.peek_at_stack(1) - comptime_b = jit.peek_at_stack(0) - - if two_fixnums_on_stack?(jit) - unless Invariants.assume_bop_not_redefined(jit, C.INTEGER_REDEFINED_OP_FLAG, C.BOP_EQ) - return false - end - - guard_two_fixnums(jit, ctx, asm, side_exit) - - asm.comment('check fixnum equality') - asm.mov(:rax, a_opnd) - asm.mov(:rcx, b_opnd) - asm.cmp(:rax, :rcx) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmove(:rax, :rcx) - - # Push the output on the stack - ctx.stack_pop(2) - dst = ctx.stack_push - asm.mov(dst, :rax) - - true - elsif comptime_a.class == String && comptime_b.class == String - unless Invariants.assume_bop_not_redefined(jit, C.STRING_REDEFINED_OP_FLAG, C.BOP_EQ) - # if overridden, emit the generic version - return false - end - - # Guard that a is a String - if jit_guard_known_klass(jit, ctx, asm, comptime_a.class, a_opnd, comptime_a, side_exit) == CantCompile - return false - end - - equal_label = asm.new_label(:equal) - ret_label = asm.new_label(:ret) - - # If they are equal by identity, return true - asm.mov(:rax, a_opnd) - asm.mov(:rcx, b_opnd) - asm.cmp(:rax, :rcx) - asm.je(equal_label) - - # Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard) - # Note: any T_STRING is valid here, but we check for a ::String for simplicity - # To pass a mutable static variable (rb_cString) requires an unsafe block - if jit_guard_known_klass(jit, ctx, asm, comptime_b.class, b_opnd, comptime_b, side_exit) == CantCompile - return false - end - - asm.comment('call rb_str_eql_internal') - asm.mov(C_ARGS[0], a_opnd) - asm.mov(C_ARGS[1], b_opnd) - asm.call(C.rb_str_eql_internal) - - # Push the output on the stack - ctx.stack_pop(2) - dst = ctx.stack_push - asm.mov(dst, C_RET) - asm.jmp(ret_label) - - asm.write_label(equal_label) - asm.mov(dst, Qtrue) - - asm.write_label(ret_label) - - true - else - false - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_prepare_routine_call(jit, ctx, asm) - jit.record_boundary_patch_point = true - jit_save_pc(jit, asm) - jit_save_sp(jit, ctx, asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param asm [RubyVM::MJIT::Assembler] - def jit_save_pc(jit, asm, comment: 'save PC to CFP') - next_pc = jit.pc + jit.insn.len * C.VALUE.size # Use the next one for backtrace and side exits - asm.comment(comment) - asm.mov(:rax, next_pc) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_save_sp(jit, ctx, asm) - if ctx.sp_offset != 0 - asm.comment('save SP to CFP') - asm.lea(SP, ctx.sp_opnd) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) - ctx.sp_offset = 0 - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jump_to_next_insn(jit, ctx, asm) - reset_depth = ctx.dup - reset_depth.chain_depth = 0 - - next_pc = jit.pc + jit.insn.len * C.VALUE.size - - # We are at the end of the current instruction. Record the boundary. - if jit.record_boundary_patch_point - exit_pos = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_side_exit(next_pc, ctx, ocb_asm) - @ocb.write(ocb_asm) - end - Invariants.record_global_inval_patch(asm, exit_pos) - jit.record_boundary_patch_point = false - end - - stub_next_block(jit.iseq, next_pc, reset_depth, asm, comment: 'jump_to_next_insn') - end - - # rb_vm_check_ints - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_check_ints(jit, ctx, asm) - asm.comment('RUBY_VM_CHECK_INTS(ec)') - asm.mov(:eax, [EC, C.rb_execution_context_t.offsetof(:interrupt_flag)]) - asm.test(:eax, :eax) - asm.jnz(side_exit(jit, ctx)) - end - - # vm_get_ep - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_get_ep(asm, level, reg:) - asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)]) - level.times do - # GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03 - asm.mov(reg, [reg, C.VALUE.size * C.VM_ENV_DATA_INDEX_SPECVAL]) - asm.and(reg, ~0x03) - end - end - - # vm_getivar - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd = nil) - side_exit = side_exit(jit, ctx) - starting_ctx = ctx.dup # copy for jit_chain_guard - - # Guard not special const - if C.SPECIAL_CONST_P(comptime_obj) - asm.incr_counter(:getivar_special_const) - return CantCompile - end - if obj_opnd.nil? # getivar - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) - else # attr_reader - asm.mov(:rax, obj_opnd) - end - guard_object_is_heap(asm, :rax, counted_exit(side_exit, :getivar_not_heap)) - - case C.BUILTIN_TYPE(comptime_obj) - when C.T_OBJECT - # This is the only supported case for now (ROBJECT_IVPTR) - else - asm.incr_counter(:getivar_not_t_object) - return CantCompile - end - - shape_id = C.rb_shape_get_shape_id(comptime_obj) - if shape_id == C.OBJ_TOO_COMPLEX_SHAPE_ID - asm.incr_counter(:getivar_too_complex) - return CantCompile - end - - asm.comment('guard shape') - asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) - jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic)) - - index = C.rb_shape_get_iv_index(shape_id, ivar_id) - if index - asm.comment('ROBJECT_IVPTR') - if C.FL_TEST_RAW(comptime_obj, C.ROBJECT_EMBED) - # Access embedded array - asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)]) - else - # Pull out an ivar table on heap - asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)]) - # Read the table - asm.mov(:rax, [:rax, index * C.VALUE.size]) - end - val_opnd = :rax - else - val_opnd = Qnil - end - - if obj_opnd - ctx.stack_pop # pop receiver for attr_reader - end - stack_opnd = ctx.stack_push - asm.mov(stack_opnd, val_opnd) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # vm_call_general (vm_sendish -> vm_call_general) - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_general(jit, ctx, asm, cd) - ci = cd.ci - mid = C.vm_ci_mid(ci) - argc = C.vm_ci_argc(ci) - flags = C.vm_ci_flag(ci) - jit_call_method(jit, ctx, asm, mid, argc, flags) - end - - # vm_call_method - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - # @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND - def jit_call_method(jit, ctx, asm, mid, argc, flags, send_shift: 0) - # Specialize on a compile-time receiver, and split a block for chain guards - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - # Generate a side exit - side_exit = side_exit(jit, ctx) - - # kw_splat is not supported yet - if flags & C.VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_kw_splat) - return CantCompile - end - - # Get a compile-time receiver and its class - recv_idx = argc + (flags & C.VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) - recv_idx += send_shift - comptime_recv = jit.peek_at_stack(recv_idx) - comptime_recv_klass = C.rb_class_of(comptime_recv) - - # Guard the receiver class (part of vm_search_method_fastpath) - recv_opnd = ctx.stack_opnd(recv_idx) - megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic) - if jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, comptime_recv, megamorphic_exit) == CantCompile - return CantCompile - end - - # Do method lookup (vm_cc_cme(cc) != NULL) - cme = C.rb_callable_method_entry(comptime_recv_klass, mid) - if cme.nil? - asm.incr_counter(:send_missing_cme) - return CantCompile # We don't support vm_call_method_name - end - - # The main check of vm_call_method before vm_call_method_each_type - case C.METHOD_ENTRY_VISI(cme) - when C.METHOD_VISI_PUBLIC - # You can always call public methods - when C.METHOD_VISI_PRIVATE - # Allow only callsites without a receiver - if flags & C.VM_CALL_FCALL == 0 - asm.incr_counter(:send_private) - return CantCompile - end - when C.METHOD_VISI_PROTECTED - asm.incr_counter(:send_protected) - return CantCompile # TODO: support this - else - # TODO: Change them to a constant and use case-in instead - raise 'unreachable' - end - - # Invalidate on redefinition (part of vm_search_method_fastpath) - Invariants.assume_method_lookup_stable(jit, cme) - - jit_call_method_each_type(jit, ctx, asm, argc, flags, cme, comptime_recv, recv_opnd, send_shift:) - end - - # vm_call_method_each_type - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_method_each_type(jit, ctx, asm, argc, flags, cme, comptime_recv, recv_opnd, send_shift:) - case cme.def.type - when C.VM_METHOD_TYPE_ISEQ - jit_call_iseq_setup(jit, ctx, asm, cme, flags, argc, send_shift:) - when C.VM_METHOD_TYPE_NOTIMPLEMENTED - asm.incr_counter(:send_notimplemented) - return CantCompile - when C.VM_METHOD_TYPE_CFUNC - jit_call_cfunc(jit, ctx, asm, cme, flags, argc, send_shift:) - when C.VM_METHOD_TYPE_ATTRSET - asm.incr_counter(:send_attrset) - return CantCompile - when C.VM_METHOD_TYPE_IVAR - jit_call_ivar(jit, ctx, asm, cme, flags, argc, comptime_recv, recv_opnd, send_shift:) - when C.VM_METHOD_TYPE_MISSING - asm.incr_counter(:send_missing) - return CantCompile - when C.VM_METHOD_TYPE_BMETHOD - asm.incr_counter(:send_bmethod) - return CantCompile - when C.VM_METHOD_TYPE_ALIAS - asm.incr_counter(:send_alias) - return CantCompile - when C.VM_METHOD_TYPE_OPTIMIZED - jit_call_optimized(jit, ctx, asm, cme, flags, argc, send_shift:) - when C.VM_METHOD_TYPE_UNDEF - asm.incr_counter(:send_undef) - return CantCompile - when C.VM_METHOD_TYPE_ZSUPER - asm.incr_counter(:send_zsuper) - return CantCompile - when C.VM_METHOD_TYPE_REFINED - asm.incr_counter(:send_refined) - return CantCompile - else - asm.incr_counter(:send_unknown_type) - return CantCompile - end - end - - # vm_call_iseq_setup - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_iseq_setup(jit, ctx, asm, cme, flags, argc, send_shift:) - iseq = def_iseq_ptr(cme.def) - opt_pc = jit_callee_setup_arg(jit, ctx, asm, flags, argc, iseq) - if opt_pc == CantCompile - # We hit some unsupported path of vm_callee_setup_arg - return CantCompile - end - - if flags & C.VM_CALL_TAILCALL != 0 - # We don't support vm_call_iseq_setup_tailcall - asm.incr_counter(:send_tailcall) - return CantCompile - end - jit_call_iseq_setup_normal(jit, ctx, asm, cme, flags, argc, iseq, send_shift:) - end - - # vm_call_iseq_setup_normal (vm_call_iseq_setup_2 -> vm_call_iseq_setup_normal) - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_iseq_setup_normal(jit, ctx, asm, cme, flags, argc, iseq, send_shift:) - # We will not have side exits from here. Adjust the stack. - if flags & C.VM_CALL_OPT_SEND != 0 - jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:) - end - - # Save caller SP and PC before pushing a callee frame for backtrace and side exits - asm.comment('save SP to caller CFP') - recv_idx = argc + (flags & C.VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) - # Skip setting this to SP register. This cfp->sp will be copied to SP on leave insn. - asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * -(1 + recv_idx))) # Pop receiver and arguments to prepare for side exits - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], :rax) - jit_save_pc(jit, asm, comment: 'save PC to caller CFP') - - frame_type = C.VM_FRAME_MAGIC_METHOD | C.VM_ENV_FLAG_LOCAL - jit_push_frame( - jit, ctx, asm, cme, flags, argc, frame_type, - iseq: iseq, - local_size: iseq.body.local_table_size - iseq.body.param.size, - stack_max: iseq.body.stack_max, - ) - - # Jump to a stub for the callee ISEQ - callee_ctx = Context.new - stub_next_block(iseq, iseq.body.iseq_encoded.to_i, callee_ctx, asm) - - EndBlock - end - - # vm_call_cfunc - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_cfunc(jit, ctx, asm, cme, flags, argc, send_shift:) - if jit_caller_setup_arg(jit, ctx, asm, flags) == CantCompile - return CantCompile - end - if jit_caller_remove_empty_kw_splat(jit, ctx, asm, flags) == CantCompile - return CantCompile - end - - jit_call_cfunc_with_frame(jit, ctx, asm, cme, flags, argc, send_shift:) - end - - # jit_call_cfunc_with_frame - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_cfunc_with_frame(jit, ctx, asm, cme, flags, argc, send_shift:) - cfunc = cme.def.body.cfunc - - if argc + 1 > 6 - asm.incr_counter(:send_cfunc_too_many_args) - return CantCompile - end - - frame_type = C.VM_FRAME_MAGIC_CFUNC | C.VM_FRAME_FLAG_CFRAME | C.VM_ENV_FLAG_LOCAL - if flags & C.VM_CALL_KW_SPLAT != 0 - frame_type |= C.VM_FRAME_FLAG_CFRAME_KW - end - - # EXEC_EVENT_HOOK: RUBY_EVENT_C_CALL and RUBY_EVENT_C_RETURN - if C.rb_mjit_global_events & (C.RUBY_EVENT_C_CALL | C.RUBY_EVENT_C_RETURN) != 0 - asm.incr_counter(:send_c_tracing) - return CantCompile - end - - # rb_check_arity - if cfunc.argc >= 0 && argc != cfunc.argc - asm.incr_counter(:send_arity) - return CantCompile - end - if cfunc.argc == -2 - asm.incr_counter(:send_cfunc_ruby_array_varg) - return CantCompile - end - - # We will not have side exits from here. Adjust the stack. - if flags & C.VM_CALL_OPT_SEND != 0 - jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:) - end - - # Check interrupts before SP motion to safely side-exit with the original SP. - jit_check_ints(jit, ctx, asm) - - # Save caller SP and PC before pushing a callee frame for backtrace and side exits - asm.comment('save SP to caller CFP') - sp_index = -(1 + argc) # Pop receiver and arguments for side exits # TODO: subtract one more for VM_CALL_ARGS_BLOCKARG - asm.lea(SP, ctx.sp_opnd(C.VALUE.size * sp_index)) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) - ctx.sp_offset = -sp_index - jit_save_pc(jit, asm, comment: 'save PC to caller CFP') - - # Push a callee frame. SP register and ctx are not modified inside this. - jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type) - - asm.comment('call C function') - case cfunc.argc - in (0..) # Non-variadic method - # Push receiver and args - (1 + argc).times do |i| - asm.mov(C_ARGS[i], ctx.stack_opnd(argc - i)) # TODO: +1 for VM_CALL_ARGS_BLOCKARG - end - in -1 # Variadic method: rb_f_puts(int argc, VALUE *argv, VALUE recv) - asm.mov(C_ARGS[0], argc) - asm.lea(C_ARGS[1], ctx.stack_opnd(argc - 1)) # argv - asm.mov(C_ARGS[2], ctx.stack_opnd(argc)) # recv - end - asm.mov(:rax, cfunc.func) - asm.call(:rax) # TODO: use rel32 if close enough - ctx.stack_pop(1 + argc) - - Invariants.record_global_inval_patch(asm, @full_cfunc_return) - - asm.comment('push the return value') - stack_ret = ctx.stack_push - asm.mov(stack_ret, C_RET) - - asm.comment('pop the stack frame') - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], CFP) - - # Let guard chains share the same successor (ctx.sp_offset == 1) - assert_equal(1, ctx.sp_offset) - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # vm_call_ivar (+ part of vm_call_method_each_type) - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_ivar(jit, ctx, asm, cme, flags, argc, comptime_recv, recv_opnd, send_shift:) - if flags & C.VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_ivar_splat) - return CantCompile - end - - if argc != 0 - asm.incr_counter(:send_arity) - return CantCompile - end - - # We don't support jit_call_opt_send_shift_stack for this yet. - if flags & C.VM_CALL_OPT_SEND != 0 - asm.incr_counter(:send_ivar_opt_send) - return CantCompile - end - - ivar_id = cme.def.body.attr.id - - if flags & C.VM_CALL_ARGS_BLOCKARG != 0 - asm.incr_counter(:send_ivar_blockarg) - return CantCompile - end - - jit_getivar(jit, ctx, asm, comptime_recv, ivar_id, recv_opnd) - end - - # vm_call_optimized - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_optimized(jit, ctx, asm, cme, flags, argc, send_shift:) - case cme.def.body.optimized.type - when C.OPTIMIZED_METHOD_TYPE_SEND - jit_call_opt_send(jit, ctx, asm, cme, flags, argc, send_shift:) - when C.OPTIMIZED_METHOD_TYPE_CALL - asm.incr_counter(:send_optimized_call) - return CantCompile - when C.OPTIMIZED_METHOD_TYPE_BLOCK_CALL - asm.incr_counter(:send_optimized_block_call) - return CantCompile - when C.OPTIMIZED_METHOD_TYPE_STRUCT_AREF - asm.incr_counter(:send_optimized_struct_aref) - return CantCompile - when C.OPTIMIZED_METHOD_TYPE_STRUCT_ASET - asm.incr_counter(:send_optimized_struct_aset) - return CantCompile - else - asm.incr_counter(:send_optimized_unknown_type) - return CantCompile - end - end - - # vm_call_opt_send - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_opt_send(jit, ctx, asm, cme, flags, argc, send_shift:) - if jit_caller_setup_arg(jit, ctx, asm, flags) == CantCompile - return CantCompile - end - - if argc == 0 - asm.incr_counter(:send_optimized_send_no_args) - return CantCompile - end - - argc -= 1 - # We aren't handling `send(:send, ...)` yet. This might work, but not tested yet. - if send_shift > 0 - asm.incr_counter(:send_optimized_send_send) - return CantCompile - end - # Ideally, we want to shift the stack here, but it's not safe until you reach the point - # where you never exit. `send_shift` signals to lazily shift the stack by this amount. - send_shift += 1 - - kw_splat = flags & C.VM_CALL_KW_SPLAT != 0 - jit_call_symbol(jit, ctx, asm, cme, C.VM_CALL_FCALL, argc, kw_splat, send_shift:) - end - - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:) - # We don't support `send(:send, ...)` for now. - assert_equal(1, send_shift) - - asm.comment('shift stack') - (0...argc).reverse_each do |i| - opnd = ctx.stack_opnd(i) - opnd2 = ctx.stack_opnd(i + 1) - asm.mov(:rax, opnd) - asm.mov(opnd2, :rax) - end - - ctx.stack_pop(1) - end - - # vm_call_symbol - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_call_symbol(jit, ctx, asm, cme, flags, argc, kw_splat, send_shift:) - flags |= C.VM_CALL_OPT_SEND | (kw_splat ? C.VM_CALL_KW_SPLAT : 0) - - comptime_symbol = jit.peek_at_stack(argc) - if comptime_symbol.class != String && !static_symbol?(comptime_symbol) - asm.incr_counter(:send_optimized_send_not_sym_or_str) - return CantCompile - end - - mid = C.get_symbol_id(comptime_symbol) - if mid == 0 - asm.incr_counter(:send_optimized_send_null_mid) - return CantCompile - end - - asm.comment("Guard #{comptime_symbol.inspect} is on stack") - mid_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_changed) - if jit_guard_known_klass(jit, ctx, asm, comptime_symbol.class, ctx.stack_opnd(argc), comptime_symbol, mid_changed_exit) == CantCompile - return CantCompile - end - asm.mov(C_ARGS[0], ctx.stack_opnd(argc)) - asm.call(C.rb_get_symbol_id) - asm.cmp(C_RET, mid) - jit_chain_guard(:jne, jit, ctx, asm, mid_changed_exit) - - if flags & C.VM_CALL_FCALL != 0 - return jit_call_method(jit, ctx, asm, mid, argc, flags, send_shift:) - end - - raise NotImplementedError # unreachable for now - end - - # vm_push_frame - # - # Frame structure: - # | args | locals | cme/cref | block_handler/prev EP | frame type (EP here) | stack bottom (SP here) - # - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, iseq: nil, local_size: 0, stack_max: 0) - # CHECK_VM_STACK_OVERFLOW0: next_cfp <= sp + (local_size + stack_max) - asm.comment('stack overflow check') - asm.lea(:rax, ctx.sp_opnd(C.rb_control_frame_t.size + C.VALUE.size * (local_size + stack_max))) - asm.cmp(CFP, :rax) - asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) - - local_size.times do |i| - asm.comment('set local variables') if i == 0 - local_index = ctx.sp_offset + i - asm.mov([SP, C.VALUE.size * local_index], Qnil) - end - - asm.comment('set up EP with managing data') - ep_offset = ctx.sp_offset + local_size + 2 - asm.mov(:rax, cme.to_i) - asm.mov([SP, C.VALUE.size * (ep_offset - 2)], :rax) - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], C.VM_BLOCK_HANDLER_NONE) - asm.mov([SP, C.VALUE.size * (ep_offset - 0)], frame_type) - - asm.comment('set up new frame') - cfp_offset = -C.rb_control_frame_t.size # callee CFP - # For ISEQ, JIT code will set it as needed. However, C func needs 0 there for svar frame detection. - if iseq.nil? - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:pc)], 0) - end - asm.mov(:rax, iseq.to_i) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:iseq)], :rax) - self_index = ctx.sp_offset - (1 + argc) # TODO: +1 for VM_CALL_ARGS_BLOCKARG - asm.mov(:rax, [SP, C.VALUE.size * self_index]) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:self)], :rax) - asm.lea(:rax, [SP, C.VALUE.size * ep_offset]) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:ep)], :rax) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:block_code)], 0) - # Update SP register only for ISEQ calls. SP-relative operations should be done above this. - sp_reg = iseq ? SP : :rax - asm.lea(sp_reg, [SP, C.VALUE.size * (ctx.sp_offset + local_size + 3)]) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:__bp__)], sp_reg) # TODO: get rid of this!! - - # cfp->jit_return is used only for ISEQs - if iseq - # Stub cfp->jit_return - return_ctx = ctx.dup - return_ctx.stack_size -= argc # Pop args # TODO: subtract 1 more for VM_CALL_ARGS_BLOCKARG - return_ctx.sp_offset = 1 # SP is in the position after popping a receiver and arguments - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx: return_ctx, pc: jit.pc + jit.insn.len * C.VALUE.size), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = proc do |branch_asm| - branch_asm.comment('set jit_return to callee CFP') - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.mov(:rax, branch_stub.target0.address) - branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) - end - end - end - branch_stub.compile.call(asm) - end - - asm.comment('switch to callee CFP') - # Update CFP register only for ISEQ calls - cfp_reg = iseq ? CFP : :rax - asm.lea(cfp_reg, [CFP, cfp_offset]) - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg) - end - - # vm_callee_setup_arg: Set up args and return opt_pc (or CantCompile) - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_callee_setup_arg(jit, ctx, asm, flags, argc, iseq) - if flags & C.VM_CALL_KW_SPLAT == 0 - if C.rb_simple_iseq_p(iseq) - if jit_caller_setup_arg(jit, ctx, asm, flags) == CantCompile - return CantCompile - end - if jit_caller_remove_empty_kw_splat(jit, ctx, asm, flags) == CantCompile - return CantCompile - end - - if argc != iseq.body.param.lead_num - # argument_arity_error - return CantCompile - end - - return 0 - else - # We don't support the remaining `else if`s yet. - asm.incr_counter(:send_iseq_not_simple) - return CantCompile - end - end - - # We don't support setup_parameters_complex - asm.incr_counter(:send_iseq_kw_splat) - return CantCompile - end - - # CALLER_SETUP_ARG: Return CantCompile if not supported - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_caller_setup_arg(jit, ctx, asm, flags) - if flags & C.VM_CALL_ARGS_SPLAT != 0 - # We don't support vm_caller_setup_arg_splat - asm.incr_counter(:send_args_splat) - return CantCompile - end - if flags & (C.VM_CALL_KWARG | C.VM_CALL_KW_SPLAT) != 0 - # We don't support keyword args either - asm.incr_counter(:send_kwarg) - return CantCompile - end - end - - # CALLER_REMOVE_EMPTY_KW_SPLAT: Return CantCompile if not supported - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def jit_caller_remove_empty_kw_splat(jit, ctx, asm, flags) - if (flags & C.VM_CALL_KW_SPLAT) > 0 - # We don't support removing the last Hash argument - asm.incr_counter(:send_kw_splat) - return CantCompile - end - end - - # Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access, - # and use Opnd::Mem to save registers. - def jit_array_len(asm, array_reg, len_reg) - asm.comment('get array length for embedded or heap') - - # Pull out the embed flag to check if it's an embedded array. - asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)]) - - # Get the length of the array - asm.and(len_reg, C.RARRAY_EMBED_LEN_MASK) - asm.sar(len_reg, C.RARRAY_EMBED_LEN_SHIFT) - - # Conditionally move the length of the heap array - asm.test([array_reg, C.RBasic.offsetof(:flags)], C.RARRAY_EMBED_FLAG) - - # Select the array length value - asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)]) - end - - def assert_equal(left, right) - if left != right - raise "'#{left.inspect}' was not '#{right.inspect}'" - end - end - - def fixnum?(obj) - (C.to_value(obj) & C.RUBY_FIXNUM_FLAG) == C.RUBY_FIXNUM_FLAG - end - - def static_symbol?(obj) - (C.to_value(obj) & 0xff) == C.RUBY_SYMBOL_FLAG - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - # @param asm [RubyVM::MJIT::Assembler] - def defer_compilation(jit, ctx, asm) - # Make a stub to compile the current insn - stub_next_block(jit.iseq, jit.pc, ctx, asm, comment: 'defer_compilation') - end - - def stub_next_block(iseq, pc, ctx, asm, comment: 'stub_next_block') - branch_stub = BranchStub.new( - iseq:, - shape: Default, - target0: BranchTarget.new(ctx:, pc:), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = proc do |branch_asm| - branch_asm.comment(comment) - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jmp(branch_stub.target0.address) - in Next0 - # Just write the block without a jump - end - end - end - branch_stub.compile.call(asm) - end - - # @param jit [RubyVM::MJIT::JITState] - # @param ctx [RubyVM::MJIT::Context] - def side_exit(jit, ctx) - if side_exit = jit.side_exits[jit.pc] - return side_exit - end - asm = Assembler.new - @exit_compiler.compile_side_exit(jit.pc, ctx, asm) - jit.side_exits[jit.pc] = @ocb.write(asm) - end - - def counted_exit(side_exit, name) - asm = Assembler.new - asm.incr_counter(name) - asm.jmp(side_exit) - @ocb.write(asm) - end - - def def_iseq_ptr(cme_def) - C.rb_iseq_check(cme_def.body.iseq.iseqptr) - end - - def to_value(obj) - @gc_refs << obj - C.to_value(obj) - end - end -end diff --git a/lib/ruby_vm/mjit/invariants.rb b/lib/ruby_vm/mjit/invariants.rb deleted file mode 100644 index e68a6e5503..0000000000 --- a/lib/ruby_vm/mjit/invariants.rb +++ /dev/null @@ -1,116 +0,0 @@ -require 'set' - -module RubyVM::MJIT - class Invariants - class << self - # Called by RubyVM::MJIT::Compiler to lazily initialize this - # @param cb [CodeBlock] - # @param ocb [CodeBlock] - # @param compiler [RubyVM::MJIT::Compiler] - # @param exit_compiler [RubyVM::MJIT::ExitCompiler] - def initialize(cb, ocb, compiler, exit_compiler) - @cb = cb - @ocb = ocb - @compiler = compiler - @exit_compiler = exit_compiler - @bop_blocks = Set.new # TODO: actually invalidate this - @cme_blocks = Hash.new { |h, k| h[k] = Set.new } - @patches = {} - - # freeze # workaround a binding.irb issue. TODO: resurrect this - end - - # @param jit [RubyVM::MJIT::JITState] - # @param klass [Integer] - # @param op [Integer] - def assume_bop_not_redefined(jit, klass, op) - return false unless C.BASIC_OP_UNREDEFINED_P(klass, op) - - ensure_block_entry_exit(jit, cause: 'assume_bop_not_redefined') - @bop_blocks << jit.block - true - end - - # @param jit [RubyVM::MJIT::JITState] - def assume_method_lookup_stable(jit, cme) - ensure_block_entry_exit(jit, cause: 'assume_method_lookup_stable') - @cme_blocks[cme.to_i] << jit.block - end - - # @param asm [RubyVM::MJIT::Assembler] - def record_global_inval_patch(asm, target) - asm.pos_marker do |address| - if @patches.key?(address) - raise 'multiple patches in the same address' - end - @patches[address] = target - end - end - - def on_cme_invalidate(cme) - @cme_blocks.fetch(cme.to_i, []).each do |block| - @cb.with_write_addr(block.start_addr) do - asm = Assembler.new - asm.comment('on_cme_invalidate') - asm.jmp(block.entry_exit) - @cb.write(asm) - end - # TODO: re-generate branches that refer to this block - end - end - - def on_constant_ic_update(iseq, ic, insn_idx) - # TODO: check multi ractor as well - if ic.entry.ic_cref - # No need to recompile the slowpath - return - end - - pc = iseq.body.iseq_encoded + insn_idx - insn_name = Compiler.decode_insn(pc.*).name - if insn_name != :opt_getconstant_path && insn_name != :trace_opt_getconstant_path - raise 'insn_idx was not at opt_getconstant_path' - end - if ic.to_i != pc[1] - raise 'insn_idx + 1 was not at the updated IC' - end - @compiler.invalidate_blocks(iseq, pc.to_i) - end - - def on_tracing_invalidate_all - # On-Stack Replacement - @patches.each do |address, target| - # TODO: assert patches don't overlap each other - @cb.with_write_addr(address) do - asm = Assembler.new - asm.comment('on_tracing_invalidate_all') - asm.jmp(target) - @cb.write(asm) - end - end - - # Avoid reusing past code - Compiler.reset_blocks - - C.mjit_for_each_iseq do |iseq| - # Disable entering past code - iseq.body.jit_func = 0 - # Compile this again if not converted to trace_* insns - iseq.body.total_calls = 0 - end - end - - # @param jit [RubyVM::MJIT::JITState] - # @param block [RubyVM::MJIT::Block] - def ensure_block_entry_exit(jit, cause:) - block = jit.block - if block.entry_exit.nil? - block.entry_exit = Assembler.new.then do |asm| - @exit_compiler.compile_entry_exit(block.pc, block.ctx, asm, cause:) - @ocb.write(asm) - end - end - end - end - end -end diff --git a/lib/ruby_vm/mjit/jit_state.rb b/lib/ruby_vm/mjit/jit_state.rb deleted file mode 100644 index 3e5fe996fa..0000000000 --- a/lib/ruby_vm/mjit/jit_state.rb +++ /dev/null @@ -1,37 +0,0 @@ -module RubyVM::MJIT - class JITState < Struct.new( - :iseq, # @param `RubyVM::MJIT::CPointer::Struct_rb_iseq_t` - :pc, # @param [Integer] The JIT target PC - :cfp, # @param `RubyVM::MJIT::CPointer::Struct_rb_control_frame_t` The JIT source CFP (before MJIT is called) - :block, # @param [RubyVM::MJIT::Block] - :side_exits, # @param [Hash{ Integer => Integer }] { PC => address } - :record_boundary_patch_point, # @param [TrueClass,FalseClass] - ) - def initialize(side_exits: {}, record_boundary_patch_point: false, **) = super - - def insn - Compiler.decode_insn(C.VALUE.new(pc).*) - end - - def operand(index, signed: false) - addr = pc + (index + 1) * Fiddle::SIZEOF_VOIDP - Fiddle::Pointer.new(addr)[0, Fiddle::SIZEOF_VOIDP].unpack(signed ? 'q' : 'Q')[0] - end - - def at_current_insn? - pc == cfp.pc.to_i - end - - def peek_at_stack(depth_from_top) - raise 'not at current insn' unless at_current_insn? - offset = -(1 + depth_from_top) - # rb_mjit_branch_stub_hit updates SP, so you don't need to worry about sp_offset - value = (cfp.sp + offset).* - C.to_ruby(value) - end - - def peek_at_self - C.to_ruby(cfp.self) - end - end -end diff --git a/lib/ruby_vm/mjit/stats.rb b/lib/ruby_vm/mjit/stats.rb deleted file mode 100644 index 22be01fe34..0000000000 --- a/lib/ruby_vm/mjit/stats.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true -module RubyVM::MJIT - def self.runtime_stats - stats = {} - - # Insn exits - INSNS.each_value do |insn| - exits = C.mjit_insn_exits[insn.bin] - if exits > 0 - stats[:"exit_#{insn.name}"] = exits - end - end - - # Runtime stats - C.rb_mjit_runtime_counters.members.each do |member| - stats[member] = C.rb_mjit_counters.public_send(member) - end - - # Other stats are calculated here - stats[:side_exit_count] = stats.select { |name, _count| name.start_with?('exit_') }.sum(&:last) - if stats[:vm_insns_count] > 0 - retired_in_mjit = stats[:mjit_insns_count] - stats[:side_exit_count] - stats[:total_insns_count] = retired_in_mjit + stats[:vm_insns_count] - stats[:ratio_in_mjit] = 100.0 * retired_in_mjit / stats[:total_insns_count] - end - - stats - end - - class << self - private - - def print_stats - stats = runtime_stats - $stderr.puts("***MJIT: Printing MJIT statistics on exit***") - - print_counters(stats, prefix: 'send_', prompt: 'method call exit reasons') - print_counters(stats, prefix: 'getivar_', prompt: 'getinstancevariable exit reasons') - print_counters(stats, prefix: 'optaref_', prompt: 'opt_aref exit reasons') - print_counters(stats, prefix: 'optgetconst_', prompt: 'opt_getconstant_path exit reasons') - print_counters(stats, prefix: 'expandarray_', prompt: 'expandarray exit reasons') - - $stderr.puts "compiled_block_count: #{format_number(13, stats[:compiled_block_count])}" - $stderr.puts "side_exit_count: #{format_number(13, stats[:side_exit_count])}" - $stderr.puts "total_insns_count: #{format_number(13, stats[:total_insns_count])}" if stats.key?(:total_insns_count) - $stderr.puts "vm_insns_count: #{format_number(13, stats[:vm_insns_count])}" if stats.key?(:vm_insns_count) - $stderr.puts "mjit_insns_count: #{format_number(13, stats[:mjit_insns_count])}" - $stderr.puts "ratio_in_mjit: #{format('%12.1f', stats[:ratio_in_mjit])}%" if stats.key?(:ratio_in_mjit) - - print_exit_counts(stats) - end - - def print_counters(stats, prefix:, prompt:) - $stderr.puts("#{prompt}: ") - counters = stats.filter { |key, _| key.start_with?(prefix) } - counters.filter! { |_, value| value != 0 } - counters.transform_keys! { |key| key.to_s.delete_prefix(prefix) } - - if counters.empty? - $stderr.puts(" (all relevant counters are zero)") - return - end - - counters = counters.to_a - counters.sort_by! { |(_, counter_value)| counter_value } - longest_name_length = counters.max_by { |(name, _)| name.length }.first.length - total = counters.sum { |(_, counter_value)| counter_value } - - counters.reverse_each do |(name, value)| - percentage = value.fdiv(total) * 100 - $stderr.printf(" %*s %s (%4.1f%%)\n", longest_name_length, name, format_number(10, value), percentage) - end - end - - def print_exit_counts(stats, how_many: 20, padding: 2) - exits = stats.filter_map { |name, count| [name.to_s.delete_prefix('exit_'), count] if name.start_with?('exit_') }.to_h - return if exits.empty? - - top_exits = exits.sort_by { |_name, count| -count }.first(how_many).to_h - total_exits = exits.values.sum - $stderr.puts "Top-#{top_exits.size} most frequent exit ops (#{format("%.1f", 100.0 * top_exits.values.sum / total_exits)}% of exits):" - - name_width = top_exits.map { |name, _count| name.length }.max + padding - count_width = top_exits.map { |_name, count| format_number(10, count).length }.max + padding - top_exits.each do |name, count| - ratio = 100.0 * count / total_exits - $stderr.puts "#{format("%#{name_width}s", name)}: #{format_number(count_width, count)} (#{format('%4.1f', ratio)}%)" - end - end - - # Format large numbers with comma separators for readability - def format_number(pad, number) - integer, decimal = number.to_s.split('.') - d_groups = integer.chars.reverse.each_slice(3) - with_commas = d_groups.map(&:join).join(',').reverse - [with_commas, decimal].compact.join('.').rjust(pad, ' ') - end - end -end |