diff options
| author | Kevin Newton <kddnewton@gmail.com> | 2022-08-18 15:39:18 -0400 |
|---|---|---|
| committer | Takashi Kokubun <takashikkbn@gmail.com> | 2022-08-29 09:09:41 -0700 |
| commit | f883aabc13d334771da926e632dca5758bb506c8 (patch) | |
| tree | fbf667cc6ab86170583e4702c86c0b11effb9413 /yjit | |
| parent | ea9ee31744a905d7bafdd064ed97f68b5b1f21fa (diff) | |
Instruction enum (https://github.com/Shopify/ruby/pull/423)
* Remove references to explicit instruction parts
Previously we would reference individual instruction fields
manually. We can't do that with instructions that are enums, so
this commit removes those references. As a side effect, we can
remove the push_insn_parts() function from the assembler because we
now explicitly push instruction structs every time.
* Switch instructions to enum
Instructions are now no longer a large struct with a bunch of
optional fields. Instead they are an enum with individual shapes
for the variants.
In terms of size, the instruction struct was 120 bytes while the
new instruction enum is 106 bytes. The bigger win however is that
we're not allocating any vectors for instruction operands (except
for CCall), which should help cut down on memory usage.
Adding new instructions will be a little more complicated going
forward, but every mission-critical function that needs to be
touched will have an exhaustive match, so the compiler should guide
any additions.
Notes
Notes:
Merged: https://github.com/ruby/ruby/pull/6289
Diffstat (limited to 'yjit')
| -rw-r--r-- | yjit/src/backend/arm64/mod.rs | 426 | ||||
| -rw-r--r-- | yjit/src/backend/ir.rs | 888 | ||||
| -rw-r--r-- | yjit/src/backend/tests.rs | 8 | ||||
| -rw-r--r-- | yjit/src/backend/x86_64/mod.rs | 431 |
4 files changed, 960 insertions, 793 deletions
diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index a32be6a6b2..60cdf2b9d1 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -59,6 +59,13 @@ impl From<Opnd> for A64Opnd { } } +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for A64Opnd { + fn from(opnd: &Opnd) -> Self { + A64Opnd::from(*opnd) + } +} + impl Assembler { // A special scratch register for intermediate processing. @@ -182,6 +189,41 @@ impl Assembler } } + /// Returns the operands that should be used for a boolean logic + /// instruction. + fn split_boolean_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + match (opnd0, opnd1) { + (Opnd::Reg(_), Opnd::Reg(_)) => { + (opnd0, opnd1) + }, + (reg_opnd @ Opnd::Reg(_), other_opnd) | + (other_opnd, reg_opnd @ Opnd::Reg(_)) => { + let opnd1 = split_bitmask_immediate(asm, other_opnd); + (reg_opnd, opnd1) + }, + _ => { + let opnd0 = split_load_operand(asm, opnd0); + let opnd1 = split_bitmask_immediate(asm, opnd1); + (opnd0, opnd1) + } + } + } + + /// Returns the operands that should be used for a csel instruction. + fn split_csel_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + let opnd0 = match opnd0 { + Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd0, + _ => split_load_operand(asm, opnd0) + }; + + let opnd1 = match opnd1 { + Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd1, + _ => split_load_operand(asm, opnd1) + }; + + (opnd0, opnd1) + } + let mut asm_local = Assembler::new_with_label_names(std::mem::take(&mut self.label_names)); let asm = &mut asm_local; let mut iterator = self.into_draining_iter(); @@ -192,7 +234,7 @@ impl Assembler // such that only the Op::Load instruction needs to handle that // case. If the values aren't heap objects then we'll treat them as // if they were just unsigned integer. - let skip_load = matches!(insn, Insn { op: Op::Load, .. }); + let skip_load = matches!(insn, Insn::Load { .. }); let mut opnd_iter = insn.opnd_iter_mut(); while let Some(opnd) = opnd_iter.next() { @@ -209,10 +251,10 @@ impl Assembler } match insn { - Insn { op: Op::Add, opnds, .. } => { - match (opnds[0], opnds[1]) { + Insn::Add { left, right, .. } => { + match (left, right) { (Opnd::Reg(_) | Opnd::InsnOut { .. }, Opnd::Reg(_) | Opnd::InsnOut { .. }) => { - asm.add(opnds[0], opnds[1]); + asm.add(left, right); }, (reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. }), other_opnd) | (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. })) => { @@ -220,30 +262,25 @@ impl Assembler asm.add(reg_opnd, opnd1); }, _ => { - let opnd0 = split_load_operand(asm, opnds[0]); - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd0 = split_load_operand(asm, left); + let opnd1 = split_shifted_immediate(asm, right); asm.add(opnd0, opnd1); } } }, - Insn { op: Op::And | Op::Or | Op::Xor, opnds, target, text, pos_marker, .. } => { - match (opnds[0], opnds[1]) { - (Opnd::Reg(_), Opnd::Reg(_)) => { - asm.push_insn_parts(insn.op, vec![opnds[0], opnds[1]], target, text, pos_marker); - }, - (reg_opnd @ Opnd::Reg(_), other_opnd) | - (other_opnd, reg_opnd @ Opnd::Reg(_)) => { - let opnd1 = split_bitmask_immediate(asm, other_opnd); - asm.push_insn_parts(insn.op, vec![reg_opnd, opnd1], target, text, pos_marker); - }, - _ => { - let opnd0 = split_load_operand(asm, opnds[0]); - let opnd1 = split_bitmask_immediate(asm, opnds[1]); - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); - } - } + Insn::And { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.and(opnd0, opnd1); + }, + Insn::Or { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.or(opnd0, opnd1); }, - Insn { op: Op::CCall, opnds, target, .. } => { + Insn::Xor { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.xor(opnd0, opnd1); + }, + Insn::CCall { opnds, target, .. } => { assert!(opnds.len() <= C_ARG_OPNDS.len()); // For each of the operands we're going to first load them @@ -258,60 +295,82 @@ impl Assembler // Now we push the CCall without any arguments so that it // just performs the call. - asm.ccall(target.unwrap().unwrap_fun_ptr(), vec![]); + asm.ccall(target.unwrap_fun_ptr(), vec![]); }, - Insn { op: Op::Cmp, opnds, .. } => { - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + Insn::Cmp { left, right } => { + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd1 = split_shifted_immediate(asm, right); asm.cmp(opnd0, opnd1); }, - Insn { op: Op::CRet, opnds, .. } => { - if opnds[0] != Opnd::Reg(C_RET_REG) { - let value = split_load_operand(asm, opnds[0]); + Insn::CRet(opnd) => { + if opnd != Opnd::Reg(C_RET_REG) { + let value = split_load_operand(asm, opnd); asm.mov(C_RET_OPND, value); } asm.cret(C_RET_OPND); }, - Insn { op: Op::CSelZ | Op::CSelNZ | Op::CSelE | Op::CSelNE | Op::CSelL | Op::CSelLE | Op::CSelG | Op::CSelGE, opnds, target, text, pos_marker, .. } => { - let new_opnds = opnds.into_iter().map(|opnd| { - match opnd { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd, - _ => split_load_operand(asm, opnd) - } - }).collect(); - - asm.push_insn_parts(insn.op, new_opnds, target, text, pos_marker); + Insn::CSelZ { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_z(opnd0, opnd1); }, - Insn { op: Op::IncrCounter, opnds, .. } => { + Insn::CSelNZ { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_nz(opnd0, opnd1); + }, + Insn::CSelE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_e(opnd0, opnd1); + }, + Insn::CSelNE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_ne(opnd0, opnd1); + }, + Insn::CSelL { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_l(opnd0, opnd1); + }, + Insn::CSelLE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_le(opnd0, opnd1); + }, + Insn::CSelG { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_g(opnd0, opnd1); + }, + Insn::CSelGE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_ge(opnd0, opnd1); + }, + Insn::IncrCounter { mem, value } => { // We'll use LDADD later which only works with registers // ... Load pointer into register - let counter_addr = split_lea_operand(asm, opnds[0]); + let counter_addr = split_lea_operand(asm, mem); // Load immediates into a register - let addend = match opnds[1] { + let addend = match value { opnd @ Opnd::Imm(_) | opnd @ Opnd::UImm(_) => asm.load(opnd), opnd => opnd, }; asm.incr_counter(counter_addr, addend); }, - Insn { op: Op::JmpOpnd, opnds, .. } => { - if let Opnd::Mem(_) = opnds[0] { - let opnd0 = split_load_operand(asm, opnds[0]); + Insn::JmpOpnd(opnd) => { + if let Opnd::Mem(_) = opnd { + let opnd0 = split_load_operand(asm, opnd); asm.jmp_opnd(opnd0); } else { - asm.jmp_opnd(opnds[0]); + asm.jmp_opnd(opnd); } }, - Insn { op: Op::Load, opnds, .. } => { - split_load_operand(asm, opnds[0]); + Insn::Load { opnd, .. } => { + split_load_operand(asm, opnd); }, - Insn { op: Op::LoadSExt, opnds, .. } => { - match opnds[0] { + Insn::LoadSExt { opnd, .. } => { + match opnd { // We only want to sign extend if the operand is a // register, instruction output, or memory address that // is 32 bits. Otherwise we'll just load the value @@ -319,87 +378,87 @@ impl Assembler Opnd::Reg(Reg { num_bits: 32, .. }) | Opnd::InsnOut { num_bits: 32, .. } | Opnd::Mem(Mem { num_bits: 32, .. }) => { - asm.load_sext(opnds[0]); + asm.load_sext(opnd); }, _ => { - asm.load(opnds[0]); + asm.load(opnd); } }; }, - Insn { op: Op::Mov, opnds, .. } => { - let value = match (opnds[0], opnds[1]) { + Insn::Mov { dest, src } => { + let value = match (dest, src) { // If the first operand is a memory operand, we're going // to transform this into a store instruction, so we'll // need to load this anyway. - (Opnd::Mem(_), Opnd::UImm(_)) => asm.load(opnds[1]), + (Opnd::Mem(_), Opnd::UImm(_)) => asm.load(src), // The value that is being moved must be either a // register or an immediate that can be encoded as a // bitmask immediate. Otherwise, we'll need to split the // move into multiple instructions. - _ => split_bitmask_immediate(asm, opnds[1]) + _ => split_bitmask_immediate(asm, src) }; // If we're attempting to load into a memory operand, then // we'll switch over to the store instruction. Otherwise // we'll use the normal mov instruction. - match opnds[0] { + match dest { Opnd::Mem(_) => { - let opnd0 = split_memory_address(asm, opnds[0]); + let opnd0 = split_memory_address(asm, dest); asm.store(opnd0, value); }, Opnd::Reg(_) => { - asm.mov(opnds[0], value); + asm.mov(dest, value); }, _ => unreachable!() }; }, - Insn { op: Op::Not, opnds, .. } => { + Insn::Not { opnd, .. } => { // The value that is being negated must be in a register, so // if we get anything else we need to load it first. - let opnd0 = match opnds[0] { - Opnd::Mem(_) => split_load_operand(asm, opnds[0]), - _ => opnds[0] + let opnd0 = match opnd { + Opnd::Mem(_) => split_load_operand(asm, opnd), + _ => opnd }; asm.not(opnd0); }, - Insn { op: Op::Store, opnds, .. } => { + Insn::Store { dest, src } => { // The displacement for the STUR instruction can't be more // than 9 bits long. If it's longer, we need to load the // memory address into a register first. - let opnd0 = split_memory_address(asm, opnds[0]); + let opnd0 = split_memory_address(asm, dest); // The value being stored must be in a register, so if it's // not already one we'll load it first. - let opnd1 = match opnds[1] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[1], - _ => split_load_operand(asm, opnds[1]) + let opnd1 = match src { + Opnd::Reg(_) | Opnd::InsnOut { .. } => src, + _ => split_load_operand(asm, src) }; asm.store(opnd0, opnd1); }, - Insn { op: Op::Sub, opnds, .. } => { - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + Insn::Sub { left, right, .. } => { + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd1 = split_shifted_immediate(asm, right); asm.sub(opnd0, opnd1); }, - Insn { op: Op::Test, opnds, .. } => { + Insn::Test { left, right } => { // The value being tested must be in a register, so if it's // not already one we'll load it first. - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; // The second value must be either a register or an // unsigned immediate that can be encoded as a bitmask // immediate. If it's not one of those, we'll need to load // it first. - let opnd1 = split_bitmask_immediate(asm, opnds[1]); + let opnd1 = split_bitmask_immediate(asm, right); asm.test(opnd0, opnd1); }, _ => { @@ -589,23 +648,20 @@ impl Assembler let start_write_pos = cb.get_write_pos(); for insn in &self.insns { match insn { - Insn { op: Op::Comment, text, .. } => { + Insn::Comment(text) => { if cfg!(feature = "asm_comments") { - cb.add_comment(text.as_ref().unwrap()); + cb.add_comment(text); } }, - Insn { op: Op::Label, target, .. } => { - cb.write_label(target.unwrap().unwrap_label_idx()); + Insn::Label(target) => { + cb.write_label(target.unwrap_label_idx()); }, // Report back the current position in the generated code - Insn { op: Op::PosMarker, pos_marker, .. } => { - let pos = cb.get_write_ptr(); - let pos_marker_fn = pos_marker.as_ref().unwrap(); - pos_marker_fn(pos); + Insn::PosMarker(pos_marker) => { + pos_marker(cb.get_write_ptr()); } - Insn { op: Op::BakeString, text, .. } => { - let str = text.as_ref().unwrap(); - for byte in str.as_bytes() { + Insn::BakeString(text) => { + for byte in text.as_bytes() { cb.write_byte(*byte); } @@ -615,69 +671,69 @@ impl Assembler // Pad out the string to the next 4-byte boundary so that // it's easy to jump past. - for _ in 0..(4 - ((str.len() + 1) % 4)) { + for _ in 0..(4 - ((text.len() + 1) % 4)) { cb.write_byte(0); } }, - Insn { op: Op::Add, opnds, out, .. } => { - adds(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Add { left, right, out } => { + adds(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::FrameSetup, .. } => { + Insn::FrameSetup => { stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); // X29 (frame_pointer) = SP mov(cb, X29, C_SP_REG); }, - Insn { op: Op::FrameTeardown, .. } => { + Insn::FrameTeardown => { // SP = X29 (frame pointer) mov(cb, C_SP_REG, X29); ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); }, - Insn { op: Op::Sub, opnds, out, .. } => { - subs(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Sub { left, right, out } => { + subs(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::And, opnds, out, .. } => { - and(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::And { left, right, out } => { + and(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Or, opnds, out, .. } => { - orr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Or { left, right, out } => { + orr(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Xor, opnds, out, .. } => { - eor(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Xor { left, right, out } => { + eor(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Not, opnds, out, .. } => { - mvn(cb, (*out).into(), opnds[0].into()); + Insn::Not { opnd, out } => { + mvn(cb, out.into(), opnd.into()); }, - Insn { op: Op::RShift, opnds, out, .. } => { - asr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::RShift { opnd, shift, out } => { + asr(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::URShift, opnds, out, .. } => { - lsr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::URShift { opnd, shift, out } => { + lsr(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::LShift, opnds, out, .. } => { - lsl(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::LShift { opnd, shift, out } => { + lsl(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::Store, opnds, .. } => { + Insn::Store { dest, src } => { // This order may be surprising but it is correct. The way // the Arm64 assembler works, the register that is going to // be stored is first and the address is second. However in // our IR we have the address first and the register second. - stur(cb, opnds[1].into(), opnds[0].into()); + stur(cb, src.into(), dest.into()); }, - Insn { op: Op::Load, opnds, out, .. } => { - match opnds[0] { + Insn::Load { opnd, out } => { + match *opnd { Opnd::Reg(_) | Opnd::InsnOut { .. } => { - mov(cb, (*out).into(), opnds[0].into()); + mov(cb, out.into(), opnd.into()); }, Opnd::UImm(uimm) => { - emit_load_value(cb, (*out).into(), uimm); + emit_load_value(cb, out.into(), uimm); }, Opnd::Imm(imm) => { - emit_load_value(cb, (*out).into(), imm as u64); + emit_load_value(cb, out.into(), imm as u64); }, Opnd::Mem(_) => { - ldur(cb, (*out).into(), opnds[0].into()); + ldur(cb, out.into(), opnd.into()); }, Opnd::Value(value) => { // We dont need to check if it's a special const @@ -689,7 +745,7 @@ impl Assembler // references to GC'd Value operands. If the value // being loaded is a heap object, we'll report that // back out to the gc_offsets list. - ldr_literal(cb, (*out).into(), 2); + ldr_literal(cb, out.into(), 2); b(cb, A64Opnd::new_imm(1 + (SIZEOF_VALUE as i64) / 4)); cb.write_bytes(&value.as_u64().to_le_bytes()); @@ -701,29 +757,29 @@ impl Assembler } }; }, - Insn { op: Op::LoadSExt, opnds, out, .. } => { - match opnds[0] { + Insn::LoadSExt { opnd, out } => { + match *opnd { Opnd::Reg(Reg { num_bits: 32, .. }) | Opnd::InsnOut { num_bits: 32, .. } => { - sxtw(cb, (*out).into(), opnds[0].into()); + sxtw(cb, out.into(), opnd.into()); }, Opnd::Mem(Mem { num_bits: 32, .. }) => { - ldursw(cb, (*out).into(), opnds[0].into()); + ldursw(cb, out.into(), opnd.into()); }, _ => unreachable!() }; }, - Insn { op: Op::Mov, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); }, - Insn { op: Op::Lea, opnds, out, .. } => { - let opnd: A64Opnd = opnds[0].into(); + Insn::Lea { opnd, out } => { + let opnd: A64Opnd = opnd.into(); match opnd { A64Opnd::Mem(mem) => { add( cb, - (*out).into(), + out.into(), A64Opnd::Reg(A64Reg { reg_no: mem.base_reg_no, num_bits: 64 }), A64Opnd::new_imm(mem.disp.into()) ); @@ -733,25 +789,25 @@ impl Assembler } }; }, - Insn { op: Op::LeaLabel, out, target, .. } => { - let label_idx = target.unwrap().unwrap_label_idx(); + Insn::LeaLabel { out, target, .. } => { + let label_idx = target.unwrap_label_idx(); cb.label_ref(label_idx, 4, |cb, end_addr, dst_addr| { adr(cb, Self::SCRATCH0, A64Opnd::new_imm(dst_addr - (end_addr - 4))); }); - mov(cb, (*out).into(), Self::SCRATCH0); + mov(cb, out.into(), Self::SCRATCH0); }, - Insn { op: Op::CPush, opnds, .. } => { - emit_push(cb, opnds[0].into()); + Insn::CPush(opnd) => { + emit_push(cb, opnd.into()); }, - Insn { op: Op::CPop, out, .. } => { - emit_pop(cb, (*out).into()); + Insn::CPop { out } => { + emit_pop(cb, out.into()); }, - Insn { op: Op::CPopInto, opnds, .. } => { - emit_pop(cb, opnds[0].into()); + Insn::CPopInto(opnd) => { + emit_pop(cb, opnd.into()); }, - Insn { op: Op::CPushAll, .. } => { + Insn::CPushAll => { let regs = Assembler::get_caller_save_regs(); for reg in regs { @@ -762,7 +818,7 @@ impl Assembler mrs(cb, Self::SCRATCH0, SystemRegister::NZCV); emit_push(cb, Self::SCRATCH0); }, - Insn { op: Op::CPopAll, .. } => { + Insn::CPopAll => { let regs = Assembler::get_caller_save_regs(); // Pop the state/flags register @@ -773,10 +829,10 @@ impl Assembler emit_pop(cb, A64Opnd::Reg(reg)); } }, - Insn { op: Op::CCall, target, .. } => { + Insn::CCall { target, .. } => { // The offset to the call target in bytes let src_addr = cb.get_write_ptr().into_i64(); - let dst_addr = target.unwrap().unwrap_fun_ptr() as i64; + let dst_addr = target.unwrap_fun_ptr() as i64; let offset = dst_addr - src_addr; // The offset in instruction count for BL's immediate let offset = offset / 4; @@ -790,20 +846,20 @@ impl Assembler blr(cb, Self::SCRATCH0); } }, - Insn { op: Op::CRet, .. } => { + Insn::CRet { .. } => { ret(cb, A64Opnd::None); }, - Insn { op: Op::Cmp, opnds, .. } => { - cmp(cb, opnds[0].into(), opnds[1].into()); + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); }, - Insn { op: Op::Test, opnds, .. } => { - tst(cb, opnds[0].into(), opnds[1].into()); + Insn::Test { left, right } => { + tst(cb, left.into(), right.into()); }, - Insn { op: Op::JmpOpnd, opnds, .. } => { - br(cb, opnds[0].into()); + Insn::JmpOpnd(opnd) => { + br(cb, opnd.into()); }, - Insn { op: Op::Jmp, target, .. } => { - match target.unwrap() { + Insn::Jmp(target) => { + match target { Target::CodePtr(dst_ptr) => { let src_addr = cb.get_write_ptr().into_i64(); let dst_addr = dst_ptr.into_i64(); @@ -831,60 +887,62 @@ impl Assembler // instruction once we know the offset. We're going // to assume we can fit into a single b instruction. // It will panic otherwise. - cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { + cb.label_ref(*label_idx, 4, |cb, src_addr, dst_addr| { b(cb, A64Opnd::new_imm((dst_addr - (src_addr - 4)) / 4)); }); }, _ => unreachable!() }; }, - Insn { op: Op::Je, target, .. } => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.unwrap()); + Insn::Je(target) => { + emit_conditional_jump::<{Condition::EQ}>(cb, *target); }, - Insn { op: Op::Jne, target, .. } => { - emit_conditional_jump::<{Condition::NE}>(cb, target.unwrap()); + Insn::Jne(target) => { + emit_conditional_jump::<{Condition::NE}>(cb, *target); }, - Insn { op: Op::Jl, target, .. } => { - emit_conditional_jump::<{Condition::LT}>(cb, target.unwrap()); + Insn::Jl(target) => { + emit_conditional_jump::<{Condition::LT}>(cb, *target); }, - Insn { op: Op::Jbe, target, .. } => { - emit_conditional_jump::<{Condition::LS}>(cb, target.unwrap()); + Insn::Jbe(target) => { + emit_conditional_jump::<{Condition::LS}>(cb, *target); }, - Insn { op: Op::Jz, target, .. } => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.unwrap()); + Insn::Jz(target) => { + emit_conditional_jump::<{Condition::EQ}>(cb, *target); }, - Insn { op: Op::Jnz, target, .. } => { - emit_conditional_jump::<{Condition::NE}>(cb, target.unwrap()); + Insn::Jnz(target) => { + emit_conditional_jump::<{Condition::NE}>(cb, *target); }, - Insn { op: Op::Jo, target, .. } => { - emit_conditional_jump::<{Condition::VS}>(cb, target.unwrap()); + Insn::Jo(target) => { + emit_conditional_jump::<{Condition::VS}>(cb, *target); }, - Insn { op: Op::IncrCounter, opnds, .. } => { - ldaddal(cb, opnds[1].into(), opnds[1].into(), opnds[0].into()); + Insn::IncrCounter { mem, value } => { + ldaddal(cb, value.into(), value.into(), mem.into()); }, - Insn { op: Op::Breakpoint, .. } => { + Insn::Breakpoint => { brk(cb, A64Opnd::None); }, - Insn { op: Op::CSelZ | Op::CSelE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::EQ); + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::EQ); }, - Insn { op: Op::CSelNZ | Op::CSelNE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::NE); + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::NE); }, - Insn { op: Op::CSelL, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::LT); + Insn::CSelL { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LT); }, - Insn { op: Op::CSelLE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::LE); + Insn::CSelLE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LE); }, - Insn { op: Op::CSelG, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::GT); + Insn::CSelG { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GT); }, - Insn { op: Op::CSelGE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::GE); + Insn::CSelGE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); } - Insn { op: Op::LiveReg, .. } => (), // just a reg alloc signal, no code - Insn { op: Op::PadEntryExit, .. } => { + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + Insn::PadEntryExit => { let jmp_len = 5 * 4; // Op::Jmp may emit 5 instructions while (cb.get_write_pos() - start_write_pos) < jmp_len { nop(cb); diff --git a/yjit/src/backend/ir.rs b/yjit/src/backend/ir.rs index cea8dfb227..fe525cf31d 100644 --- a/yjit/src/backend/ir.rs +++ b/yjit/src/backend/ir.rs @@ -24,155 +24,6 @@ pub const SP: Opnd = _SP; pub const C_ARG_OPNDS: [Opnd; 6] = _C_ARG_OPNDS; pub const C_RET_OPND: Opnd = _C_RET_OPND; -/// Instruction opcodes -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum Op -{ - // Add a comment into the IR at the point that this instruction is added. - // It won't have any impact on that actual compiled code. - Comment, - - // Add a label into the IR at the point that this instruction is added. - Label, - - // Mark a position in the generated code - PosMarker, - - // Bake a string directly into the instruction stream. - BakeString, - - // Add two operands together, and return the result as a new operand. This - // operand can then be used as the operand on another instruction. It - // accepts two operands, which can be of any type - // - // Under the hood when allocating registers, the IR will determine the most - // efficient way to get these values into memory. For example, if both - // operands are immediates, then it will load the first one into a register - // first with a mov instruction and then add them together. If one of them - // is a register, however, it will just perform a single add instruction. - Add, - - // This is the same as the OP_ADD instruction, except for subtraction. - Sub, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary AND operation. - And, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary OR operation. - Or, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary XOR operation. - Xor, - - // Perform the NOT operation on an individual operand, and return the result - // as a new operand. This operand can then be used as the operand on another - // instruction. - Not, - - /// Shift a value right by a certain amount (signed). - RShift, - - /// Shift a value right by a certain amount (unsigned). - URShift, - - /// Shift a value left by a certain amount. - LShift, - - // - // Low-level instructions - // - - // A low-level instruction that loads a value into a register. - Load, - - // A low-level instruction that loads a value into a register and - // sign-extends it to a 64-bit value. - LoadSExt, - - // Low-level instruction to store a value to memory. - Store, - - // Load effective address - Lea, - - // Load effective address relative to the current instruction pointer. It - // accepts a single signed immediate operand. - LeaLabel, - - // A low-level mov instruction. It accepts two operands. - Mov, - - // Bitwise AND test instruction - Test, - - // Compare two operands - Cmp, - - // Unconditional jump to a branch target - Jmp, - - // Unconditional jump which takes a reg/mem address operand - JmpOpnd, - - // Low-level conditional jump instructions - Jl, - Jbe, - Je, - Jne, - Jz, - Jnz, - Jo, - - // Conditional select instructions - CSelZ, - CSelNZ, - CSelE, - CSelNE, - CSelL, - CSelLE, - CSelG, - CSelGE, - - // Push and pop registers to/from the C stack - CPush, - CPop, - CPopInto, - - // Push and pop all of the caller-save registers and the flags to/from the C - // stack - CPushAll, - CPopAll, - - // C function call with N arguments (variadic) - CCall, - - // C function return - CRet, - - // Atomically increment a counter - // Input: memory operand, increment value - // Produces no output - IncrCounter, - - // Trigger a debugger breakpoint - Breakpoint, - - /// Set up the frame stack as necessary per the architecture. - FrameSetup, - - /// Tear down the frame stack as necessary per the architecture. - FrameTeardown, - - /// Take a specific register. Signal the register allocator to not use it. - LiveReg, - - /// Pad nop instructions to accomodate Op::Jmp in case the block is invalidated. - PadEntryExit, -} - // Memory operand base #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum MemBase @@ -429,26 +280,170 @@ impl From<CodePtr> for Target { type PosMarkerFn = Box<dyn Fn(CodePtr)>; /// YJIT IR instruction -pub struct Insn -{ - // Opcode for the instruction - pub(super) op: Op, +pub enum Insn { + /// Add two operands together, and return the result as a new operand. + Add { left: Opnd, right: Opnd, out: Opnd }, + + /// This is the same as the OP_ADD instruction, except that it performs the + /// binary AND operation. + And { left: Opnd, right: Opnd, out: Opnd }, + + /// Bake a string directly into the instruction stream. + BakeString(String), + + // Trigger a debugger breakpoint + Breakpoint, + + /// Add a comment into the IR at the point that this instruction is added. + /// It won't have any impact on that actual compiled code. + Comment(String), + + /// Compare two operands + Cmp { left: Opnd, right: Opnd }, + + /// Pop a register from the C stack + CPop { out: Opnd }, + + /// Pop all of the caller-save registers and the flags from the C stack + CPopAll, + + /// Pop a register from the C stack and store it into another register + CPopInto(Opnd), + + /// Push a register onto the C stack + CPush(Opnd), + + /// Push all of the caller-save registers and the flags to the C stack + CPushAll, + + // C function call with N arguments (variadic) + CCall { opnds: Vec<Opnd>, target: Target, out: Opnd }, + + // C function return + CRet(Opnd), + + /// Conditionally select if equal + CSelE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater + CSelG { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater or equal + CSelGE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Optional string for comments and labels - pub(super) text: Option<String>, + /// Conditionally select if less + CSelL { truthy: Opnd, falsy: Opnd, out: Opnd }, - // List of input operands/values - pub(super) opnds: Vec<Opnd>, + /// Conditionally select if less or equal + CSelLE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Output operand for this instruction - pub(super) out: Opnd, + /// Conditionally select if not equal + CSelNE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // List of branch targets (branch instructions only) - pub(super) target: Option<Target>, + /// Conditionally select if not zero + CSelNZ { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Callback to mark the position of this instruction - // in the generated code - pub(super) pos_marker: Option<PosMarkerFn>, + /// Conditionally select if zero + CSelZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Set up the frame stack as necessary per the architecture. + FrameSetup, + + /// Tear down the frame stack as necessary per the architecture. + FrameTeardown, + + // Atomically increment a counter + // Input: memory operand, increment value + // Produces no output + IncrCounter { mem: Opnd, value: Opnd }, + + /// Jump if below or equal + Jbe(Target), + + /// Jump if equal + Je(Target), + + /// Jump if lower + Jl(Target), + + // Unconditional jump to a branch target + Jmp(Target), + + // Unconditional jump which takes a reg/mem address operand + JmpOpnd(Opnd), + + /// Jump if not equal + Jne(Target), + + /// Jump if not zero + Jnz(Target), + + /// Jump if overflow + Jo(Target), + + /// Jump if zero + Jz(Target), + + // Add a label into the IR at the point that this instruction is added. + Label(Target), + + // Load effective address relative to the current instruction pointer. It + // accepts a single signed immediate operand. + LeaLabel { target: Target, out: Opnd }, + + // Load effective address + Lea { opnd: Opnd, out: Opnd }, + + /// Take a specific register. Signal the register allocator to not use it. + LiveReg { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register. + Load { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register and + // sign-extends it to a 64-bit value. + LoadSExt { opnd: Opnd, out: Opnd }, + + /// Shift a value left by a certain amount. + LShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // A low-level mov instruction. It accepts two operands. + Mov { dest: Opnd, src: Opnd }, + + // Perform the NOT operation on an individual operand, and return the result + // as a new operand. This operand can then be used as the operand on another + // instruction. + Not { opnd: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary OR operation. + Or { left: Opnd, right: Opnd, out: Opnd }, + + /// Pad nop instructions to accomodate Op::Jmp in case the block is + /// invalidated. + PadEntryExit, + + // Mark a position in the generated code + PosMarker(PosMarkerFn), + + /// Shift a value right by a certain amount (signed). + RShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // Low-level instruction to store a value to memory. + Store { dest: Opnd, src: Opnd }, + + // This is the same as the OP_ADD instruction, except for subtraction. + Sub { left: Opnd, right: Opnd, out: Opnd }, + + // Bitwise AND test instruction + Test { left: Opnd, right: Opnd }, + + /// Shift a value right by a certain amount (unsigned). + URShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary XOR operation. + Xor { left: Opnd, right: Opnd, out: Opnd } } impl Insn { @@ -464,34 +459,92 @@ impl Insn { InsnOpndMutIterator::new(self) } + /// Returns a string that describes which operation this instruction is + /// performing. This is used for debugging. + fn op(&self) -> &'static str { + match self { + Insn::Add { .. } => "Add", + Insn::And { .. } => "And", + Insn::BakeString(_) => "BakeString", + Insn::Breakpoint => "Breakpoint", + Insn::Comment(_) => "Comment", + Insn::Cmp { .. } => "Cmp", + Insn::CPop { .. } => "CPop", + Insn::CPopAll => "CPopAll", + Insn::CPopInto(_) => "CPopInto", + Insn::CPush(_) => "CPush", + Insn::CPushAll => "CPushAll", + Insn::CCall { .. } => "CCall", + Insn::CRet(_) => "CRet", + Insn::CSelE { .. } => "CSelE", + Insn::CSelG { .. } => "CSelG", + Insn::CSelGE { .. } => "CSelGE", + Insn::CSelL { .. } => "CSelL", + Insn::CSelLE { .. } => "CSelLE", + Insn::CSelNE { .. } => "CSelNE", + Insn::CSelNZ { .. } => "CSelNZ", + Insn::CSelZ { .. } => "CSelZ", + Insn::FrameSetup => "FrameSetup", + Insn::FrameTeardown => "FrameTeardown", + Insn::IncrCounter { .. } => "IncrCounter", + Insn::Jbe(_) => "Jbe", + Insn::Je(_) => "Je", + Insn::Jl(_) => "Jl", + Insn::Jmp(_) => "Jmp", + Insn::JmpOpnd(_) => "JmpOpnd", + Insn::Jne(_) => "Jne", + Insn::Jnz(_) => "Jnz", + Insn::Jo(_) => "Jo", + Insn::Jz(_) => "Jz", + Insn::Label(_) => "Label", + Insn::LeaLabel { .. } => "LeaLabel", + Insn::Lea { .. } => "Lea", + Insn::LiveReg { .. } => "LiveReg", + Insn::Load { .. } => "Load", + Insn::LoadSExt { .. } => "LoadSExt", + Insn::LShift { .. } => "LShift", + Insn::Mov { .. } => "Mov", + Insn::Not { .. } => "Not", + Insn::Or { .. } => "Or", + Insn::PadEntryExit => "PadEntryExit", + Insn::PosMarker(_) => "PosMarker", + Insn::RShift { .. } => "RShift", + Insn::Store { .. } => "Store", + Insn::Sub { .. } => "Sub", + Insn::Test { .. } => "Test", + Insn::URShift { .. } => "URShift", + Insn::Xor { .. } => "Xor" + } + } + /// Return a non-mutable reference to the out operand for this instruction /// if it has one. pub fn out_opnd(&self) -> Option<&Opnd> { match self { - Insn { op: Op::Add, out, .. } | - Insn { op: Op::And, out, .. } | - Insn { op: Op::CCall, out, .. } | - Insn { op: Op::CPop, out, .. } | - Insn { op: Op::CSelE, out, .. } | - Insn { op: Op::CSelG, out, .. } | - Insn { op: Op::CSelGE, out, .. } | - Insn { op: Op::CSelL, out, .. } | - Insn { op: Op::CSelLE, out, .. } | - Insn { op: Op::CSelNE, out, .. } | - Insn { op: Op::CSelNZ, out, .. } | - Insn { op: Op::CSelZ, out, .. } | - Insn { op: Op::Lea, out, .. } | - Insn { op: Op::LeaLabel, out, .. } | - Insn { op: Op::LiveReg, out, .. } | - Insn { op: Op::Load, out, .. } | - Insn { op: Op::LoadSExt, out, .. } | - Insn { op: Op::LShift, out, .. } | - Insn { op: Op::Not, out, .. } | - Insn { op: Op::Or, out, .. } | - Insn { op: Op::RShift, out, .. } | - Insn { op: Op::Sub, out, .. } | - Insn { op: Op::URShift, out, .. } | - Insn { op: Op::Xor, out, .. } => Some(out), + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaLabel { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), _ => None } } @@ -500,30 +553,55 @@ impl Insn { /// has one. pub fn out_opnd_mut(&mut self) -> Option<&mut Opnd> { match self { - Insn { op: Op::Add, out, .. } | - Insn { op: Op::And, out, .. } | - Insn { op: Op::CCall, out, .. } | - Insn { op: Op::CPop, out, .. } | - Insn { op: Op::CSelE, out, .. } | - Insn { op: Op::CSelG, out, .. } | - Insn { op: Op::CSelGE, out, .. } | - Insn { op: Op::CSelL, out, .. } | - Insn { op: Op::CSelLE, out, .. } | - Insn { op: Op::CSelNE, out, .. } | - Insn { op: Op::CSelNZ, out, .. } | - Insn { op: Op::CSelZ, out, .. } | - Insn { op: Op::Lea, out, .. } | - Insn { op: Op::LeaLabel, out, .. } | - Insn { op: Op::LiveReg, out, .. } | - Insn { op: Op::Load, out, .. } | - Insn { op: Op::LoadSExt, out, .. } | - Insn { op: Op::LShift, out, .. } | - Insn { op: Op::Not, out, .. } | - Insn { op: Op::Or, out, .. } | - Insn { op: Op::RShift, out, .. } | - Insn { op: Op::Sub, out, .. } | - Insn { op: Op::URShift, out, .. } | - Insn { op: Op::Xor, out, .. } => Some(out), + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaLabel { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Returns the target for this instruction if there is one. + pub fn target(&self) -> Option<&Target> { + match self { + Insn::Jbe(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::Jz(target) | + Insn::LeaLabel { target, .. } => Some(target), + _ => None + } + } + + /// Returns the text associated with this instruction if there is some. + pub fn text(&self) -> Option<&String> { + match self { + Insn::BakeString(text) | + Insn::Comment(text) => Some(text), _ => None } } @@ -547,77 +625,77 @@ impl<'a> Iterator for InsnOpndIterator<'a> { fn next(&mut self) -> Option<Self::Item> { match self.insn { - Insn { op: Op::BakeString, .. } | - Insn { op: Op::Breakpoint, .. } | - Insn { op: Op::Comment, .. } | - Insn { op: Op::CPop, .. } | - Insn { op: Op::CPopAll, .. } | - Insn { op: Op::CPushAll, .. } | - Insn { op: Op::FrameSetup, .. } | - Insn { op: Op::FrameTeardown, .. } | - Insn { op: Op::Jbe, .. } | - Insn { op: Op::Je, .. } | - Insn { op: Op::Jl, .. } | - Insn { op: Op::Jmp, .. } | - Insn { op: Op::Jne, .. } | - Insn { op: Op::Jnz, .. } | - Insn { op: Op::Jo, .. } | - Insn { op: Op::Jz, .. } | - Insn { op: Op::Label, .. } | - Insn { op: Op::LeaLabel, .. } | - Insn { op: Op::PadEntryExit, .. } | - Insn { op: Op::PosMarker, .. } => None, - Insn { op: Op::CPopInto, opnds, .. } | - Insn { op: Op::CPush, opnds, .. } | - Insn { op: Op::CRet, opnds, .. } | - Insn { op: Op::JmpOpnd, opnds, .. } | - Insn { op: Op::Lea, opnds, .. } | - Insn { op: Op::LiveReg, opnds, .. } | - Insn { op: Op::Load, opnds, .. } | - Insn { op: Op::LoadSExt, opnds, .. } | - Insn { op: Op::Not, opnds, .. } => { + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::CPopAll | + Insn::CPushAll | + Insn::FrameSetup | + Insn::FrameTeardown | + Insn::Jbe(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jmp(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::Jz(_) | + Insn::Label(_) | + Insn::LeaLabel { .. } | + Insn::PadEntryExit | + Insn::PosMarker(_) => None, + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&opnds[0]) + Some(&opnd) }, _ => None } }, - Insn { op: Op::Add, opnds, .. } | - Insn { op: Op::And, opnds, .. } | - Insn { op: Op::Cmp, opnds, .. } | - Insn { op: Op::CSelE, opnds, .. } | - Insn { op: Op::CSelG, opnds, .. } | - Insn { op: Op::CSelGE, opnds, .. } | - Insn { op: Op::CSelL, opnds, .. } | - Insn { op: Op::CSelLE, opnds, .. } | - Insn { op: Op::CSelNE, opnds, .. } | - Insn { op: Op::CSelNZ, opnds, .. } | - Insn { op: Op::CSelZ, opnds, .. } | - Insn { op: Op::IncrCounter, opnds, .. } | - Insn { op: Op::LShift, opnds, .. } | - Insn { op: Op::Mov, opnds, .. } | - Insn { op: Op::Or, opnds, .. } | - Insn { op: Op::RShift, opnds, .. } | - Insn { op: Op::Store, opnds, .. } | - Insn { op: Op::Sub, opnds, .. } | - Insn { op: Op::Test, opnds, .. } | - Insn { op: Op::URShift, opnds, .. } | - Insn { op: Op::Xor, opnds, .. } => { + Insn::Add { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::And { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Cmp { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::CSelE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelG { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelGE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelL { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelLE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::IncrCounter { mem: opnd0 @ _, value: opnd1 @ _, .. } | + Insn::LShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Mov { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Or { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::RShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Store { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Sub { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Test { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::URShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Xor { left: opnd0 @ _, right: opnd1 @ _, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&opnds[0]) + Some(&opnd0) } 1 => { self.idx += 1; - Some(&opnds[1]) + Some(&opnd1) } _ => None } }, - Insn { op: Op::CCall, opnds, .. } => { + Insn::CCall { opnds, .. } => { if self.idx < opnds.len() { let opnd = &opnds[self.idx]; self.idx += 1; @@ -643,77 +721,77 @@ impl<'a> InsnOpndMutIterator<'a> { pub(super) fn next(&mut self) -> Option<&mut Opnd> { match self.insn { - Insn { op: Op::BakeString, .. } | - Insn { op: Op::Breakpoint, .. } | - Insn { op: Op::Comment, .. } | - Insn { op: Op::CPop, .. } | - Insn { op: Op::CPopAll, .. } | - Insn { op: Op::CPushAll, .. } | - Insn { op: Op::FrameSetup, .. } | - Insn { op: Op::FrameTeardown, .. } | - Insn { op: Op::Jbe, .. } | - Insn { op: Op::Je, .. } | - Insn { op: Op::Jl, .. } | - Insn { op: Op::Jmp, .. } | - Insn { op: Op::Jne, .. } | - Insn { op: Op::Jnz, .. } | - Insn { op: Op::Jo, .. } | - Insn { op: Op::Jz, .. } | - Insn { op: Op::Label, .. } | - Insn { op: Op::LeaLabel, .. } | - Insn { op: Op::PadEntryExit, .. } | - Insn { op: Op::PosMarker, .. } => None, - Insn { op: Op::CPopInto, opnds, .. } | - Insn { op: Op::CPush, opnds, .. } | - Insn { op: Op::CRet, opnds, .. } | - Insn { op: Op::JmpOpnd, opnds, .. } | - Insn { op: Op::Lea, opnds, .. } | - Insn { op: Op::LiveReg, opnds, .. } | - Insn { op: Op::Load, opnds, .. } | - Insn { op: Op::LoadSExt, opnds, .. } | - Insn { op: Op::Not, opnds, .. } => { + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::CPopAll | + Insn::CPushAll | + Insn::FrameSetup | + Insn::FrameTeardown | + Insn::Jbe(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jmp(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::Jz(_) | + Insn::Label(_) | + Insn::LeaLabel { .. } | + Insn::PadEntryExit | + Insn::PosMarker(_) => None, + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&mut opnds[0]) + Some(opnd) }, _ => None } }, - Insn { op: Op::Add, opnds, .. } | - Insn { op: Op::And, opnds, .. } | - Insn { op: Op::Cmp, opnds, .. } | - Insn { op: Op::CSelE, opnds, .. } | - Insn { op: Op::CSelG, opnds, .. } | - Insn { op: Op::CSelGE, opnds, .. } | - Insn { op: Op::CSelL, opnds, .. } | - Insn { op: Op::CSelLE, opnds, .. } | - Insn { op: Op::CSelNE, opnds, .. } | - Insn { op: Op::CSelNZ, opnds, .. } | - Insn { op: Op::CSelZ, opnds, .. } | - Insn { op: Op::IncrCounter, opnds, .. } | - Insn { op: Op::LShift, opnds, .. } | - Insn { op: Op::Mov, opnds, .. } | - Insn { op: Op::Or, opnds, .. } | - Insn { op: Op::RShift, opnds, .. } | - Insn { op: Op::Store, opnds, .. } | - Insn { op: Op::Sub, opnds, .. } | - Insn { op: Op::Test, opnds, .. } | - Insn { op: Op::URShift, opnds, .. } | - Insn { op: Op::Xor, opnds, .. } => { + Insn::Add { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::And { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Cmp { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::CSelE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelG { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelGE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelL { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelLE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::IncrCounter { mem: opnd0 @ _, value: opnd1 @ _, .. } | + Insn::LShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Mov { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Or { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::RShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Store { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Sub { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Test { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::URShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Xor { left: opnd0 @ _, right: opnd1 @ _, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&mut opnds[0]) + Some(opnd0) } 1 => { self.idx += 1; - Some(&mut opnds[1]) + Some(opnd1) } _ => None } }, - Insn { op: Op::CCall, opnds, .. } => { + Insn::CCall { opnds, .. } => { if self.idx < opnds.len() { let opnd = &mut opnds[self.idx]; self.idx += 1; @@ -728,7 +806,7 @@ impl<'a> InsnOpndMutIterator<'a> { impl fmt::Debug for Insn { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{:?}(", self.op)?; + write!(fmt, "{}(", self.op())?; // Print list of operands let mut opnd_iter = self.opnd_iter(); @@ -741,10 +819,10 @@ impl fmt::Debug for Insn { write!(fmt, ")")?; // Print text, target, and pos if they are present - if let Some(text) = &self.text { + if let Some(text) = self.text() { write!(fmt, " {text:?}")? } - if let Some(target) = self.target { + if let Some(target) = self.target() { write!(fmt, " target={target:?}")?; } @@ -814,23 +892,6 @@ impl Assembler self.live_ranges.push(insn_idx); } - /// Append an instruction to the list by creating a new instruction from the - /// component parts given to this function. This will also create a new - /// output operand from the given operands for the new instruction. - pub(super) fn push_insn_parts( - &mut self, - op: Op, - opnds: Vec<Opnd>, - target: Option<Target>, - text: Option<String>, - pos_marker: Option<PosMarkerFn> - ) -> Opnd - { - let out = self.next_opnd_out(Opnd::match_num_bits(&opnds)); - self.push_insn(Insn { op, text, opnds, out, target, pos_marker }); - out - } - /// Create a new label instance that we can jump to pub fn new_label(&mut self, name: &str) -> Target { @@ -841,23 +902,6 @@ impl Assembler Target::Label(label_idx) } - /// Add a label at the current position - pub fn write_label(&mut self, label: Target) - { - assert!(label.unwrap_label_idx() < self.label_names.len()); - - let insn = Insn { - op: Op::Label, - text: None, - opnds: vec![], - out: Opnd::None, - target: Some(label), - pos_marker: None, - }; - self.insns.push(insn); - self.live_ranges.push(self.insns.len()); - } - /// Sets the out field on the various instructions that require allocated /// registers because their output is used as the operand on a subsequent /// instruction. This is our implementation of the linear scan algorithm. @@ -928,7 +972,7 @@ impl Assembler if let Some(Opnd::Reg(reg)) = asm.insns[start_index].out_opnd() { dealloc_reg(&mut pool, ®s, reg); } else { - unreachable!("no register allocated for insn {:?}", insn.op); + unreachable!("no register allocated for insn {:?}", insn); } } } @@ -937,7 +981,7 @@ impl Assembler } // C return values need to be mapped to the C return register - if insn.op == Op::CCall { + if matches!(insn, Insn::CCall { .. }) { assert_eq!(pool, 0, "register lives past C function call"); } @@ -958,7 +1002,7 @@ impl Assembler let mut out_reg: Option<Reg> = None; // C return values need to be mapped to the C return register - if insn.op == Op::CCall { + if matches!(insn, Insn::CCall { .. }) { out_reg = Some(take_reg(&mut pool, ®s, &C_RET_REG)); } @@ -983,9 +1027,9 @@ impl Assembler // already allocated. if out_reg.is_none() { out_reg = match &insn { - Insn { op: Op::LiveReg, opnds, .. } => { + Insn::LiveReg { opnd, .. } => { // Allocate a specific register - let reg = opnds[0].unwrap_reg(); + let reg = opnd.unwrap_reg(); Some(take_reg(&mut pool, ®s, ®)) }, _ => { @@ -1090,9 +1134,13 @@ impl AssemblerDrainingIterator { /// Returns the next instruction in the list with the indices corresponding /// to the next list of instructions. pub fn next_mapped(&mut self) -> Option<(usize, Insn)> { - self.next_unmapped().map(|(index, insn)| { - let opnds = insn.opnd_iter().map(|opnd| opnd.map_index(&self.indices)).collect(); - (index, Insn { opnds, ..insn }) + self.next_unmapped().map(|(index, mut insn)| { + let mut opnd_iter = insn.opnd_iter_mut(); + while let Some(opnd) = opnd_iter.next() { + *opnd = opnd.map_index(&self.indices); + } + + (index, insn) }) } @@ -1167,273 +1215,279 @@ impl Assembler { #[must_use] pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Add, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Add { left, right, out }); out } #[must_use] pub fn and(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::And, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::And { left, right, out }); out } pub fn bake_string(&mut self, text: &str) { - self.push_insn(Insn { op: Op::BakeString, opnds: vec![], out: Opnd::None, text: Some(text.to_string()), target: None, pos_marker: None }); + self.push_insn(Insn::BakeString(text.to_string())); } pub fn breakpoint(&mut self) { - self.push_insn(Insn { op: Op::Breakpoint, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Breakpoint); } #[must_use] pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&opnds)); - self.push_insn(Insn { op: Op::CCall, opnds, out, text: None, target: Some(Target::FunPtr(fptr)), pos_marker: None }); + self.push_insn(Insn::CCall { target: Target::FunPtr(fptr), opnds, out }); out } pub fn cmp(&mut self, left: Opnd, right: Opnd) { - self.push_insn(Insn { op: Op::Cmp, opnds: vec![left, right], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Cmp { left, right }); } pub fn comment(&mut self, text: &str) { - self.push_insn(Insn { op: Op::Comment, opnds: vec![], out: Opnd::None, text: Some(text.to_string()), target: None, pos_marker: None }); + self.push_insn(Insn::Comment(text.to_string())); } #[must_use] pub fn cpop(&mut self) -> Opnd { let out = self.next_opnd_out(Opnd::DEFAULT_NUM_BITS); - self.push_insn(Insn { op: Op::CPop, opnds: vec![], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPop { out }); out } pub fn cpop_all(&mut self) { - self.push_insn(Insn { op: Op::CPopAll, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPopAll); } pub fn cpop_into(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CPopInto, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPopInto(opnd)); } pub fn cpush(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CPush, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPush(opnd)); } pub fn cpush_all(&mut self) { - self.push_insn(Insn { op: Op::CPushAll, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPushAll); } pub fn cret(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CRet, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CRet(opnd)); } #[must_use] pub fn csel_e(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelE { truthy, falsy, out }); out } #[must_use] pub fn csel_g(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelG, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelG { truthy, falsy, out }); out } #[must_use] pub fn csel_ge(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelGE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelGE { truthy, falsy, out }); out } #[must_use] pub fn csel_l(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelL, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelL { truthy, falsy, out }); out } #[must_use] pub fn csel_le(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelLE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelLE { truthy, falsy, out }); out } #[must_use] pub fn csel_ne(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelNE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelNE { truthy, falsy, out }); out } #[must_use] pub fn csel_nz(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelNZ, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelNZ { truthy, falsy, out }); out } #[must_use] pub fn csel_z(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelZ, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelZ { truthy, falsy, out }); out } pub fn frame_setup(&mut self) { - self.push_insn(Insn { op: Op::FrameSetup, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::FrameSetup); } pub fn frame_teardown(&mut self) { - self.push_insn(Insn { op: Op::FrameTeardown, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::FrameTeardown); } pub fn incr_counter(&mut self, mem: Opnd, value: Opnd) { - self.push_insn(Insn { op: Op::IncrCounter, opnds: vec![mem, value], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::IncrCounter { mem, value }); } pub fn jbe(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jbe, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jbe(target)); } pub fn je(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Je, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Je(target)); } pub fn jl(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jl, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jl(target)); } pub fn jmp(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jmp, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jmp(target)); } pub fn jmp_opnd(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::JmpOpnd, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::JmpOpnd(opnd)); } pub fn jne(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jne, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jne(target)); } pub fn jnz(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jnz, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jnz(target)); } pub fn jo(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jo, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jo(target)); } pub fn jz(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jz, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jz(target)); } #[must_use] pub fn lea(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Lea, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Lea { opnd, out }); out } #[must_use] pub fn lea_label(&mut self, target: Target) -> Opnd { let out = self.next_opnd_out(Opnd::DEFAULT_NUM_BITS); - self.push_insn(Insn { op: Op::LeaLabel, opnds: vec![], out, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::LeaLabel { target, out }); out } #[must_use] pub fn live_reg_opnd(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::LiveReg, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LiveReg { opnd, out }); out } #[must_use] pub fn load(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Load, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Load { opnd, out }); out } #[must_use] pub fn load_sext(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::LoadSExt, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LoadSExt { opnd, out }); out } #[must_use] pub fn lshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::LShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LShift { opnd, shift, out }); out } pub fn mov(&mut self, dest: Opnd, src: Opnd) { - self.push_insn(Insn { op: Op::Mov, opnds: vec![dest, src], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Mov { dest, src }); } #[must_use] pub fn not(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Not, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Not { opnd, out }); out } #[must_use] pub fn or(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Or, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Or { left, right, out }); out } pub fn pad_entry_exit(&mut self) { - self.push_insn(Insn { op: Op::PadEntryExit, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::PadEntryExit); } //pub fn pos_marker<F: FnMut(CodePtr)>(&mut self, marker_fn: F) pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr) + 'static) { - self.push_insn(Insn { op: Op::PosMarker, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: Some(Box::new(marker_fn)) }); + self.push_insn(Insn::PosMarker(Box::new(marker_fn))); } #[must_use] pub fn rshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::RShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::RShift { opnd, shift, out }); out } pub fn store(&mut self, dest: Opnd, src: Opnd) { - self.push_insn(Insn { op: Op::Store, opnds: vec![dest, src], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Store { dest, src }); } #[must_use] pub fn sub(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Sub, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Sub { left, right, out }); out } pub fn test(&mut self, left: Opnd, right: Opnd) { - self.push_insn(Insn { op: Op::Test, opnds: vec![left, right], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Test { left, right }); } #[must_use] pub fn urshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::URShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::URShift { opnd, shift, out }); out } + /// Add a label at the current position + pub fn write_label(&mut self, target: Target) { + assert!(target.unwrap_label_idx() < self.label_names.len()); + self.push_insn(Insn::Label(target)); + } + #[must_use] pub fn xor(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Xor, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Xor { left, right, out }); out } } @@ -1444,7 +1498,7 @@ mod tests { #[test] fn test_opnd_iter() { - let insn = Insn { op: Op::Add, opnds: vec![Opnd::None, Opnd::None], out: Opnd::None, text: None, target: None, pos_marker: None }; + let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; let mut opnd_iter = insn.opnd_iter(); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); @@ -1455,7 +1509,7 @@ mod tests { #[test] fn test_opnd_iter_mut() { - let mut insn = Insn { op: Op::Add, opnds: vec![Opnd::None, Opnd::None], out: Opnd::None, text: None, target: None, pos_marker: None }; + let mut insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; let mut opnd_iter = insn.opnd_iter_mut(); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); diff --git a/yjit/src/backend/tests.rs b/yjit/src/backend/tests.rs index b89b7eb648..08e8849b4d 100644 --- a/yjit/src/backend/tests.rs +++ b/yjit/src/backend/tests.rs @@ -315,9 +315,9 @@ fn test_draining_iterator() { while let Some((index, insn)) = iter.next_unmapped() { match index { - 0 => assert_eq!(insn.op, Op::Load), - 1 => assert_eq!(insn.op, Op::Store), - 2 => assert_eq!(insn.op, Op::Add), + 0 => assert!(matches!(insn, Insn::Load { .. })), + 1 => assert!(matches!(insn, Insn::Store { .. })), + 2 => assert!(matches!(insn, Insn::Add { .. })), _ => panic!("Unexpected instruction index"), }; } @@ -337,7 +337,7 @@ fn test_lookback_iterator() { if index > 0 { let opnd_iter = iter.get_previous().unwrap().opnd_iter(); assert_eq!(opnd_iter.take(1).next(), Some(&Opnd::None)); - assert_eq!(insn.op, Op::Store); + assert!(matches!(insn, Insn::Store { .. })); } } } diff --git a/yjit/src/backend/x86_64/mod.rs b/yjit/src/backend/x86_64/mod.rs index 0c994144d0..bda7dc4c06 100644 --- a/yjit/src/backend/x86_64/mod.rs +++ b/yjit/src/backend/x86_64/mod.rs @@ -69,6 +69,13 @@ impl From<Opnd> for X86Opnd { } } +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for X86Opnd { + fn from(opnd: &Opnd) -> Self { + X86Opnd::from(*opnd) + } +} + impl Assembler { // A special scratch register for intermediate processing. @@ -96,11 +103,47 @@ impl Assembler /// Split IR instructions for the x86 platform fn x86_split(mut self) -> Assembler { + fn split_arithmetic_opnds(asm: &mut Assembler, live_ranges: &Vec<usize>, index: usize, unmapped_opnds: &Vec<Opnd>, left: &Opnd, right: &Opnd) -> (Opnd, Opnd) { + match (unmapped_opnds[0], unmapped_opnds[1]) { + (Opnd::Mem(_), Opnd::Mem(_)) => { + (asm.load(*left), asm.load(*right)) + }, + (Opnd::Mem(_), Opnd::UImm(value)) => { + // 32-bit values will be sign-extended + if imm_num_bits(value as i64) > 32 { + (asm.load(*left), asm.load(*right)) + } else { + (asm.load(*left), *right) + } + }, + (Opnd::Mem(_), Opnd::Imm(value)) => { + if imm_num_bits(value) > 32 { + (asm.load(*left), asm.load(*right)) + } else { + (asm.load(*left), *right) + } + }, + // Instruction output whose live range spans beyond this instruction + (Opnd::InsnOut { idx, .. }, _) => { + if live_ranges[idx] > index { + (asm.load(*left), *right) + } else { + (*left, *right) + } + }, + // We have to load memory operands to avoid corrupting them + (Opnd::Mem(_) | Opnd::Reg(_), _) => { + (asm.load(*left), *right) + }, + _ => (*left, *right) + } + } + let live_ranges: Vec<usize> = take(&mut self.live_ranges); let mut asm = Assembler::new_with_label_names(take(&mut self.label_names)); let mut iterator = self.into_draining_iter(); - while let Some((index, insn)) = iterator.next_unmapped() { + while let Some((index, mut insn)) = iterator.next_unmapped() { // When we're iterating through the instructions with x86_split, we // need to know the previous live ranges in order to tell if a // register lasts beyond the current instruction. So instead of @@ -122,8 +165,15 @@ impl Assembler // - Most instructions can't be encoded with 64-bit immediates. // - We look for Op::Load specifically when emiting to keep GC'ed // VALUEs alive. This is a sort of canonicalization. - let mapped_opnds: Vec<Opnd> = insn.opnd_iter().map(|opnd| { - if insn.op == Op::Load { + let mut unmapped_opnds: Vec<Opnd> = vec![]; + + let is_load = matches!(insn, Insn::Load { .. }); + let mut opnd_iter = insn.opnd_iter_mut(); + + while let Some(opnd) = opnd_iter.next() { + unmapped_opnds.push(*opnd); + + *opnd = if is_load { iterator.map_opnd(*opnd) } else if let Opnd::Value(value) = opnd { // Since mov(mem64, imm32) sign extends, as_i64() makes sure @@ -136,130 +186,137 @@ impl Assembler } else { iterator.map_opnd(*opnd) } - }).collect(); + } - match insn { - Insn { op: Op::Add | Op::Sub | Op::And | Op::Cmp | Op::Or | Op::Test | Op::Xor, opnds, target, text, pos_marker, .. } => { - let (opnd0, opnd1) = match (opnds[0], opnds[1]) { - (Opnd::Mem(_), Opnd::Mem(_)) => { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - }, - (Opnd::Mem(_), Opnd::UImm(value)) => { - // 32-bit values will be sign-extended - if imm_num_bits(value as i64) > 32 { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - } else { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } - }, - (Opnd::Mem(_), Opnd::Imm(value)) => { - if imm_num_bits(value) > 32 { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - } else { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } - }, - // Instruction output whose live range spans beyond this instruction - (Opnd::InsnOut { idx, .. }, _) => { - if live_ranges[idx] > index { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } else { - (mapped_opnds[0], mapped_opnds[1]) - } - }, - // We have to load memory operands to avoid corrupting them - (Opnd::Mem(_) | Opnd::Reg(_), _) => { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - }, - _ => (mapped_opnds[0], mapped_opnds[1]) - }; + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + let (split_left, split_right) = split_arithmetic_opnds(&mut asm, &live_ranges, index, &unmapped_opnds, left, right); + + *left = split_left; + *right = split_right; + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*left, *right])); - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); + asm.push_insn(insn); + }, + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + let (split_left, split_right) = split_arithmetic_opnds(&mut asm, &live_ranges, index, &unmapped_opnds, left, right); + + *left = split_left; + *right = split_right; + + asm.push_insn(insn); }, // These instructions modify their input operand in-place, so we // may need to load the input value to preserve it - Insn { op: Op::LShift | Op::RShift | Op::URShift, opnds, target, text, pos_marker, .. } => { - let (opnd0, opnd1) = match (opnds[0], opnds[1]) { + Insn::LShift { opnd, shift, out } | + Insn::RShift { opnd, shift, out } | + Insn::URShift { opnd, shift, out } => { + match (&unmapped_opnds[0], &unmapped_opnds[1]) { // Instruction output whose live range spans beyond this instruction (Opnd::InsnOut { idx, .. }, _) => { - if live_ranges[idx] > index { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } else { - (mapped_opnds[0], mapped_opnds[1]) + if live_ranges[*idx] > index { + *opnd = asm.load(*opnd); } }, // We have to load memory operands to avoid corrupting them (Opnd::Mem(_) | Opnd::Reg(_), _) => { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) + *opnd = asm.load(*opnd); }, - _ => (mapped_opnds[0], mapped_opnds[1]) + _ => {} }; - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); - }, - Insn { op: Op::CSelZ | Op::CSelNZ | Op::CSelE | Op::CSelNE | Op::CSelL | Op::CSelLE | Op::CSelG | Op::CSelGE, target, text, pos_marker, .. } => { - let new_opnds = mapped_opnds.into_iter().map(|opnd| { - match opnd { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd, - _ => asm.load(opnd) + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*opnd, *shift])); + asm.push_insn(insn); + }, + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } | + Insn::CSelL { truthy, falsy, out } | + Insn::CSelLE { truthy, falsy, out } | + Insn::CSelG { truthy, falsy, out } | + Insn::CSelGE { truthy, falsy, out } => { + match truthy { + Opnd::Reg(_) | Opnd::InsnOut { .. } => {}, + _ => { + *truthy = asm.load(*truthy); } - }).collect(); + }; - asm.push_insn_parts(insn.op, new_opnds, target, text, pos_marker); + match falsy { + Opnd::Reg(_) | Opnd::InsnOut { .. } => {}, + _ => { + *falsy = asm.load(*falsy); + } + }; + + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*truthy, *falsy])); + asm.push_insn(insn); }, - Insn { op: Op::Mov, .. } => { - match (mapped_opnds[0], mapped_opnds[1]) { + Insn::Mov { dest, src } => { + match (&dest, &src) { (Opnd::Mem(_), Opnd::Mem(_)) => { // We load opnd1 because for mov, opnd0 is the output - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); }, (Opnd::Mem(_), Opnd::UImm(value)) => { // 32-bit values will be sign-extended - if imm_num_bits(value as i64) > 32 { - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + if imm_num_bits(*value as i64) > 32 { + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); } else { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } }, (Opnd::Mem(_), Opnd::Imm(value)) => { - if imm_num_bits(value) > 32 { - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + if imm_num_bits(*value) > 32 { + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); } else { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } }, _ => { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } } }, - Insn { op: Op::Not, opnds, .. } => { - let opnd0 = match opnds[0] { + Insn::Not { opnd, .. } => { + let opnd0 = match unmapped_opnds[0] { // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. Opnd::InsnOut { idx, .. } => { if live_ranges[idx] > index { - asm.load(mapped_opnds[0]) + asm.load(*opnd) } else { - mapped_opnds[0] + *opnd } }, // We have to load memory and register operands to avoid // corrupting them. Opnd::Mem(_) | Opnd::Reg(_) => { - asm.load(mapped_opnds[0]) + asm.load(*opnd) }, // Otherwise we can just reuse the existing operand. - _ => mapped_opnds[0] + _ => *opnd }; asm.not(opnd0); }, _ => { - asm.push_insn_parts(insn.op, mapped_opnds, insn.target, insn.text, insn.pos_marker); + if insn.out_opnd().is_some() { + let out_num_bits = Opnd::match_num_bits_iter(insn.opnd_iter()); + let out = insn.out_opnd_mut().unwrap(); + *out = asm.next_opnd_out(out_num_bits); + } + + asm.push_insn(insn); } }; @@ -281,26 +338,24 @@ impl Assembler let start_write_pos = cb.get_write_pos(); for insn in &self.insns { match insn { - Insn { op: Op::Comment, text, .. } => { + Insn::Comment(text) => { if cfg!(feature = "asm_comments") { - cb.add_comment(text.as_ref().unwrap()); + cb.add_comment(text); } }, // Write the label at the current position - Insn { op: Op::Label, target, .. } => { - cb.write_label(target.unwrap().unwrap_label_idx()); + Insn::Label(target) => { + cb.write_label(target.unwrap_label_idx()); }, // Report back the current position in the generated code - Insn { op: Op::PosMarker, pos_marker, .. } => { - let pos = cb.get_write_ptr(); - let pos_marker_fn = pos_marker.as_ref().unwrap(); - pos_marker_fn(pos); + Insn::PosMarker(pos_marker) => { + pos_marker(cb.get_write_ptr()); }, - Insn { op: Op::BakeString, text, .. } => { - for byte in text.as_ref().unwrap().as_bytes() { + Insn::BakeString(text) => { + for byte in text.as_bytes() { cb.write_byte(*byte); } @@ -309,55 +364,55 @@ impl Assembler cb.write_byte(0); }, - Insn { op: Op::Add, opnds, .. } => { - add(cb, opnds[0].into(), opnds[1].into()) + Insn::Add { left, right, .. } => { + add(cb, left.into(), right.into()) }, - Insn { op: Op::FrameSetup, .. } => {}, - Insn { op: Op::FrameTeardown, .. } => {}, + Insn::FrameSetup => {}, + Insn::FrameTeardown => {}, - Insn { op: Op::Sub, opnds, .. } => { - sub(cb, opnds[0].into(), opnds[1].into()) + Insn::Sub { left, right, .. } => { + sub(cb, left.into(), right.into()) }, - Insn { op: Op::And, opnds, .. } => { - and(cb, opnds[0].into(), opnds[1].into()) + Insn::And { left, right, .. } => { + and(cb, left.into(), right.into()) }, - Insn { op: Op::Or, opnds, .. } => { - or(cb, opnds[0].into(), opnds[1].into()); + Insn::Or { left, right, .. } => { + or(cb, left.into(), right.into()); }, - Insn { op: Op::Xor, opnds, .. } => { - xor(cb, opnds[0].into(), opnds[1].into()); + Insn::Xor { left, right, .. } => { + xor(cb, left.into(), right.into()); }, - Insn { op: Op::Not, opnds, .. } => { - not(cb, opnds[0].into()); + Insn::Not { opnd, .. } => { + not(cb, opnd.into()); }, - Insn { op: Op::LShift, opnds, .. } => { - shl(cb, opnds[0].into(), opnds[1].into()) + Insn::LShift { opnd, shift , ..} => { + shl(cb, opnd.into(), shift.into()) }, - Insn { op: Op::RShift, opnds, .. } => { - sar(cb, opnds[0].into(), opnds[1].into()) + Insn::RShift { opnd, shift , ..} => { + sar(cb, opnd.into(), shift.into()) }, - Insn { op: Op::URShift, opnds, .. } => { - shr(cb, opnds[0].into(), opnds[1].into()) + Insn::URShift { opnd, shift, .. } => { + shr(cb, opnd.into(), shift.into()) }, - Insn { op: Op::Store, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Store { dest, src } => { + mov(cb, dest.into(), src.into()); }, // This assumes only load instructions can contain references to GC'd Value operands - Insn { op: Op::Load, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); + Insn::Load { opnd, out } => { + mov(cb, out.into(), opnd.into()); // If the value being loaded is a heap object - if let Opnd::Value(val) = opnds[0] { + if let Opnd::Value(val) = opnd { if !val.special_const_p() { // The pointer immediate is encoded as the last part of the mov written out let ptr_offset: u32 = (cb.get_write_pos() as u32) - (SIZEOF_VALUE as u32); @@ -366,45 +421,45 @@ impl Assembler } }, - Insn { op: Op::LoadSExt, opnds, out, .. } => { - movsx(cb, (*out).into(), opnds[0].into()); + Insn::LoadSExt { opnd, out } => { + movsx(cb, out.into(), opnd.into()); }, - Insn { op: Op::Mov, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); }, // Load effective address - Insn { op: Op::Lea, opnds, out, .. } => { - lea(cb, (*out).into(), opnds[0].into()); + Insn::Lea { opnd, out } => { + lea(cb, out.into(), opnd.into()); }, // Load relative address - Insn { op: Op::LeaLabel, out, target, .. } => { - let label_idx = target.unwrap().unwrap_label_idx(); + Insn::LeaLabel { target, out } => { + let label_idx = target.unwrap_label_idx(); cb.label_ref(label_idx, 7, |cb, src_addr, dst_addr| { let disp = dst_addr - src_addr; lea(cb, Self::SCRATCH0, mem_opnd(8, RIP, disp.try_into().unwrap())); }); - mov(cb, (*out).into(), Self::SCRATCH0); + mov(cb, out.into(), Self::SCRATCH0); }, // Push and pop to/from the C stack - Insn { op: Op::CPush, opnds, .. } => { - push(cb, opnds[0].into()); + Insn::CPush(opnd) => { + push(cb, opnd.into()); }, - Insn { op: Op::CPop, out, .. } => { - pop(cb, (*out).into()); + Insn::CPop { out } => { + pop(cb, out.into()); }, - Insn { op: Op::CPopInto, opnds, .. } => { - pop(cb, opnds[0].into()); + Insn::CPopInto(opnd) => { + pop(cb, opnd.into()); }, // Push and pop to the C stack all caller-save registers and the // flags - Insn { op: Op::CPushAll, .. } => { + Insn::CPushAll => { let regs = Assembler::get_caller_save_regs(); for reg in regs { @@ -412,7 +467,7 @@ impl Assembler } pushfq(cb); }, - Insn { op: Op::CPopAll, .. } => { + Insn::CPopAll => { let regs = Assembler::get_caller_save_regs(); popfq(cb); @@ -422,7 +477,7 @@ impl Assembler }, // C function call - Insn { op: Op::CCall, opnds, target, .. } => { + Insn::CCall { opnds, target, .. } => { // Temporary assert!(opnds.len() <= _C_ARG_OPNDS.len()); @@ -431,92 +486,92 @@ impl Assembler mov(cb, X86Opnd::Reg(_C_ARG_OPNDS[idx].unwrap_reg()), opnds[idx].into()); } - let ptr = target.unwrap().unwrap_fun_ptr(); + let ptr = target.unwrap_fun_ptr(); call_ptr(cb, RAX, ptr); }, - Insn { op: Op::CRet, opnds, .. } => { + Insn::CRet(opnd) => { // TODO: bias allocation towards return register - if opnds[0] != Opnd::Reg(C_RET_REG) { - mov(cb, RAX, opnds[0].into()); + if *opnd != Opnd::Reg(C_RET_REG) { + mov(cb, RAX, opnd.into()); } ret(cb); }, // Compare - Insn { op: Op::Cmp, opnds, .. } => { - cmp(cb, opnds[0].into(), opnds[1].into()); + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); } // Test and set flags - Insn { op: Op::Test, opnds, .. } => { - test(cb, opnds[0].into(), opnds[1].into()); + Insn::Test { left, right } => { + test(cb, left.into(), right.into()); } - Insn { op: Op::JmpOpnd, opnds, .. } => { - jmp_rm(cb, opnds[0].into()); + Insn::JmpOpnd(opnd) => { + jmp_rm(cb, opnd.into()); } // Conditional jump to a label - Insn { op: Op::Jmp, target, .. } => { - match target.unwrap() { + Insn::Jmp(target) => { + match *target { Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), Target::Label(label_idx) => jmp_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Je, target, .. } => { - match target.unwrap() { + Insn::Je(target) => { + match *target { Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), Target::Label(label_idx) => je_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jne, target, .. } => { - match target.unwrap() { + Insn::Jne(target) => { + match *target { Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), Target::Label(label_idx) => jne_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jl, target, .. } => { - match target.unwrap() { + Insn::Jl(target) => { + match *target { Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), Target::Label(label_idx) => jl_label(cb, label_idx), _ => unreachable!() } }, - Insn { op: Op::Jbe, target, .. } => { - match target.unwrap() { + Insn::Jbe(target) => { + match *target { Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), Target::Label(label_idx) => jbe_label(cb, label_idx), _ => unreachable!() } }, - Insn { op: Op::Jz, target, .. } => { - match target.unwrap() { + Insn::Jz(target) => { + match *target { Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), Target::Label(label_idx) => jz_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jnz, target, .. } => { - match target.unwrap() { + Insn::Jnz(target) => { + match *target { Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), Target::Label(label_idx) => jnz_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jo, target, .. } => { - match target.unwrap() { + Insn::Jo(target) => { + match *target { Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), Target::Label(label_idx) => jo_label(cb, label_idx), _ => unreachable!() @@ -524,49 +579,49 @@ impl Assembler } // Atomically increment a counter at a given memory location - Insn { op: Op::IncrCounter, opnds, .. } => { - assert!(matches!(opnds[0], Opnd::Mem(_))); - assert!(matches!(opnds[1], Opnd::UImm(_) | Opnd::Imm(_) ) ); + Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::Mem(_))); + assert!(matches!(value, Opnd::UImm(_) | Opnd::Imm(_) ) ); write_lock_prefix(cb); - add(cb, opnds[0].into(), opnds[1].into()); + add(cb, mem.into(), value.into()); }, - Insn { op: Op::Breakpoint, .. } => int3(cb), + Insn::Breakpoint => int3(cb), - Insn { op: Op::CSelZ, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovnz(cb, (*out).into(), opnds[1].into()); + Insn::CSelZ { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovnz(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelNZ, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovz(cb, (*out).into(), opnds[1].into()); + Insn::CSelNZ { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovz(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovne(cb, (*out).into(), opnds[1].into()); + Insn::CSelE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovne(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelNE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmove(cb, (*out).into(), opnds[1].into()); + Insn::CSelNE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmove(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelL, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovge(cb, (*out).into(), opnds[1].into()); + Insn::CSelL { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovge(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelLE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovg(cb, (*out).into(), opnds[1].into()); + Insn::CSelLE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovg(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelG, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovle(cb, (*out).into(), opnds[1].into()); + Insn::CSelG { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovle(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelGE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovl(cb, (*out).into(), opnds[1].into()); + Insn::CSelGE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovl(cb, out.into(), falsy.into()); } - Insn { op: Op::LiveReg, .. } => (), // just a reg alloc signal, no code - Insn { op: Op::PadEntryExit, .. } => { + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + Insn::PadEntryExit => { // We assume that our Op::Jmp usage that gets invalidated is <= 5 let code_size: u32 = (cb.get_write_pos() - start_write_pos).try_into().unwrap(); if code_size < 5 { @@ -578,7 +633,7 @@ impl Assembler // we feed to the backend could get lowered into other // instructions. So it's possible that some of our backend // instructions can never make it to the emit stage. - _ => panic!("unsupported instruction passed to x86 backend: {:?}", insn.op) + _ => panic!("unsupported instruction passed to x86 backend: {:?}", insn) }; } |