summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlan Wu <XrXr@users.noreply.github.com>2023-03-16 15:39:27 -0400
committerTakashi Kokubun <takashikkbn@gmail.com>2023-03-17 09:30:24 -0700
commit10e4fa3a0f05f72733d132794cedd08906b40196 (patch)
treeaaa4bb05559fc5216e76901044743d1cc59c29e0
parentc62cf60d187fbe74da97f79148789cdd1bfa0332 (diff)
YJIT: Use raw pointers and shared references over `Rc<RefCell<_>>`
`Rc` and `RefCell` both incur runtime space costs. In addition, `RefCell` has given us some headaches with the non obvious borrow panics it likes to throw out. The latest one started with 7fd53eeb46db261bbc20025cdab70096245a5cbe and is yet to be resolved. Since we already rely on the GC to properly reclaim memory for `Block` and `Branch`, we might as well stop paying the overhead of `Rc` and `RefCell`. The `RefCell` panics go away with this change, too. On 25 iterations of `railsbench` with a stats build I got `yjit_alloc_size: 8,386,129 => 7,348,637`, with the new memory size 87.6% of the status quo. This makes the metadata and machine code size roughly line up one-to-one. The general idea here is to use `&` shared references with [interior mutability][1] with `Cell`, which doesn't take any extra space. The `noalias` requirement that `&mut` imposes is way too hard to meet and verify. Imagine replacing places where we would've gotten `BorrowError` from `RefCell` with Rust/LLVM miscompiling us due to aliasing violations. With shared references, we don't have to think about subtle cases like the GC _sometimes_ calling the mark callback while codegen has an aliasing reference in a stack frame below. We mostly only need to worry about liveness, with which the GC already helps. There is now a clean split between blocks and branches that are not yet fully constructed and ones that are "in-service", so to speak. Working with `PendingBranch` and `JITState` don't really involve `unsafe` stuff. This change allows `Branch` and `Block` to not have as many optional fields as many of them are only optional during compilation. Fields that change post-compilation are wrapped in `Cell` to facilitate mutation through shared references. I do some `unsafe` dances here. I've included just a couple tests to run with Miri (`cargo +nightly miri test miri`). We can add more Miri tests if desired. [1]: https://doc.rust-lang.org/std/cell/struct.UnsafeCell.html
Notes
Notes: Merged: https://github.com/ruby/ruby/pull/7443
-rw-r--r--yjit/src/codegen.rs225
-rw-r--r--yjit/src/core.rs1228
-rw-r--r--yjit/src/disasm.rs26
-rw-r--r--yjit/src/invariants.rs119
-rw-r--r--yjit/src/stats.rs4
5 files changed, 965 insertions, 637 deletions
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 31582c26bc..e9c17cb537 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -12,6 +12,7 @@ use crate::utils::*;
use CodegenStatus::*;
use YARVOpnd::*;
+use std::cell::Cell;
use std::cmp;
use std::collections::HashMap;
use std::ffi::CStr;
@@ -38,63 +39,89 @@ type InsnGenFn = fn(
ocb: &mut OutlinedCb,
) -> CodegenStatus;
-/// Code generation state
-/// This struct only lives while code is being generated
+/// Ephemeral code generation state.
+/// Represents a [core::Block] while we build it.
pub struct JITState {
- // Block version being compiled
- block: BlockRef,
-
- // Instruction sequence this is associated with
+ /// Instruction sequence for the compiling block
iseq: IseqPtr,
- // Index of the current instruction being compiled
- insn_idx: u16,
+ /// The iseq index of the first instruction in the block
+ starting_insn_idx: IseqIdx,
+
+ /// The [Context] entering into the first instruction of the block
+ starting_ctx: Context,
+
+ /// The placement for the machine code of the [Block]
+ output_ptr: CodePtr,
- // Opcode for the instruction being compiled
+ /// Index of the current instruction being compiled
+ insn_idx: IseqIdx,
+
+ /// Opcode for the instruction being compiled
opcode: usize,
- // PC of the instruction being compiled
+ /// PC of the instruction being compiled
pc: *mut VALUE,
- // Side exit to the instruction being compiled. See :side-exit:.
+ /// Side exit to the instruction being compiled. See :side-exit:.
side_exit_for_pc: Option<CodePtr>,
- // Execution context when compilation started
- // This allows us to peek at run-time values
+ /// Execution context when compilation started
+ /// This allows us to peek at run-time values
ec: EcPtr,
- // Whether we need to record the code address at
- // the end of this bytecode instruction for global invalidation
- record_boundary_patch_point: bool,
+ /// The outgoing branches the block will have
+ pub pending_outgoing: Vec<PendingBranchRef>,
+
+ // --- Fields for block invalidation and invariants tracking below:
+ // Public mostly so into_block defined in the sibling module core
+ // can partially move out of Self.
- // The block's outgoing branches
- outgoing: Vec<BranchRef>,
+ /// Whether we need to record the code address at
+ /// the end of this bytecode instruction for global invalidation
+ pub record_boundary_patch_point: bool,
- // The block's CME dependencies
- cme_dependencies: Vec<CmePtr>,
+ /// Code for immediately exiting upon entry to the block.
+ /// Required for invalidation.
+ pub block_entry_exit: Option<CodePtr>,
+
+ /// A list of callable method entries that must be valid for the block to be valid.
+ pub method_lookup_assumptions: Vec<CmePtr>,
+
+ /// A list of basic operators that not be redefined for the block to be valid.
+ pub bop_assumptions: Vec<(RedefinitionFlag, ruby_basic_operators)>,
+
+ /// A list of constant expression path segments that must have
+ /// not been written to for the block to be valid.
+ pub stable_constant_names_assumption: Option<*const ID>,
+
+ /// When true, the block is valid only when there is a total of one ractor running
+ pub block_assumes_single_ractor: bool,
}
impl JITState {
- pub fn new(blockref: &BlockRef, ec: EcPtr) -> Self {
+ pub fn new(blockid: BlockId, starting_ctx: Context, output_ptr: CodePtr, ec: EcPtr) -> Self {
JITState {
- block: blockref.clone(),
- iseq: ptr::null(), // TODO: initialize this from the blockid
+ iseq: blockid.iseq,
+ starting_insn_idx: blockid.idx,
+ starting_ctx,
+ output_ptr,
insn_idx: 0,
opcode: 0,
pc: ptr::null_mut::<VALUE>(),
side_exit_for_pc: None,
+ pending_outgoing: vec![],
ec,
record_boundary_patch_point: false,
- outgoing: Vec::new(),
- cme_dependencies: Vec::new(),
+ block_entry_exit: None,
+ method_lookup_assumptions: vec![],
+ bop_assumptions: vec![],
+ stable_constant_names_assumption: None,
+ block_assumes_single_ractor: false,
}
}
- pub fn get_block(&self) -> BlockRef {
- self.block.clone()
- }
-
- pub fn get_insn_idx(&self) -> u16 {
+ pub fn get_insn_idx(&self) -> IseqIdx {
self.insn_idx
}
@@ -110,6 +137,18 @@ impl JITState {
self.pc
}
+ pub fn get_starting_insn_idx(&self) -> IseqIdx {
+ self.starting_insn_idx
+ }
+
+ pub fn get_block_entry_exit(&self) -> Option<CodePtr> {
+ self.block_entry_exit
+ }
+
+ pub fn get_starting_ctx(&self) -> Context {
+ self.starting_ctx.clone()
+ }
+
pub fn get_arg(&self, arg_idx: isize) -> VALUE {
// insn_len require non-test config
#[cfg(not(test))]
@@ -175,18 +214,22 @@ impl JITState {
}
}
+ pub fn assume_method_lookup_stable(&mut self, ocb: &mut OutlinedCb, cme: CmePtr) {
+ jit_ensure_block_entry_exit(self, ocb);
+ self.method_lookup_assumptions.push(cme);
+ }
+
fn get_cfp(&self) -> *mut rb_control_frame_struct {
unsafe { get_ec_cfp(self.ec) }
}
- // Push an outgoing branch ref
- pub fn push_outgoing(&mut self, branch: BranchRef) {
- self.outgoing.push(branch);
+ pub fn assume_stable_constant_names(&mut self, ocb: &mut OutlinedCb, id: *const ID) {
+ jit_ensure_block_entry_exit(self, ocb);
+ self.stable_constant_names_assumption = Some(id);
}
- // Push a CME dependency
- pub fn push_cme_dependency(&mut self, cme: CmePtr) {
- self.cme_dependencies.push(cme);
+ pub fn queue_outgoing_branch(&mut self, branch: PendingBranchRef) {
+ self.pending_outgoing.push(branch)
}
}
@@ -498,22 +541,20 @@ fn get_side_exit(jit: &mut JITState, ocb: &mut OutlinedCb, ctx: &Context) -> Tar
// Ensure that there is an exit for the start of the block being compiled.
// Block invalidation uses this exit.
pub fn jit_ensure_block_entry_exit(jit: &mut JITState, ocb: &mut OutlinedCb) {
- let blockref = jit.block.clone();
- let mut block = blockref.borrow_mut();
- let block_ctx = block.get_ctx();
- let blockid = block.get_blockid();
-
- if block.entry_exit.is_some() {
+ if jit.block_entry_exit.is_some() {
return;
}
+ let block_starting_context = &jit.get_starting_ctx();
+
// If we're compiling the first instruction in the block.
- if jit.insn_idx == blockid.idx {
+ if jit.insn_idx == jit.starting_insn_idx {
// Generate the exit with the cache in jitstate.
- block.entry_exit = Some(get_side_exit(jit, ocb, &block_ctx).unwrap_code_ptr());
+ let entry_exit = get_side_exit(jit, ocb, block_starting_context).unwrap_code_ptr();
+ jit.block_entry_exit = Some(entry_exit);
} else {
- let block_entry_pc = unsafe { rb_iseq_pc_at_idx(blockid.iseq, blockid.idx.into()) };
- block.entry_exit = Some(gen_outlined_exit(block_entry_pc, &block_ctx, ocb));
+ let block_entry_pc = unsafe { rb_iseq_pc_at_idx(jit.iseq, jit.starting_insn_idx.into()) };
+ jit.block_entry_exit = Some(gen_outlined_exit(block_entry_pc, block_starting_context, ocb));
}
}
@@ -725,18 +766,18 @@ pub fn gen_single_block(
verify_blockid(blockid);
assert!(!(blockid.idx == 0 && ctx.get_stack_size() > 0));
+ // Save machine code placement of the block. `cb` might page switch when we
+ // generate code in `ocb`.
+ let block_start_addr = cb.get_write_ptr();
+
// Instruction sequence to compile
let iseq = blockid.iseq;
let iseq_size = unsafe { get_iseq_encoded_size(iseq) };
let iseq_size: u16 = iseq_size.try_into().unwrap();
let mut insn_idx: u16 = blockid.idx;
- let starting_insn_idx = insn_idx;
-
- // Allocate the new block
- let blockref = Block::new(blockid, &ctx, cb.get_write_ptr());
// Initialize a JIT state object
- let mut jit = JITState::new(&blockref, ec);
+ let mut jit = JITState::new(blockid, ctx.clone(), cb.get_write_ptr(), ec);
jit.iseq = blockid.iseq;
// Create a backend assembler instance
@@ -762,7 +803,7 @@ pub fn gen_single_block(
// We need opt_getconstant_path to be in a block all on its own. Cut the block short
// if we run into it. This is necessary because we want to invalidate based on the
// instruction's index.
- if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > starting_insn_idx {
+ if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > jit.starting_insn_idx {
jump_to_next_insn(&mut jit, &ctx, &mut asm, ocb);
break;
}
@@ -814,17 +855,15 @@ pub fn gen_single_block(
println!("can't compile {}", insn_name(opcode));
}
- let mut block = jit.block.borrow_mut();
-
// TODO: if the codegen function makes changes to ctx and then return YJIT_CANT_COMPILE,
// the exit this generates would be wrong. We could save a copy of the entry context
// and assert that ctx is the same here.
gen_exit(jit.pc, &ctx, &mut asm);
- // If this is the first instruction in the block, then we can use
- // the exit for block->entry_exit.
- if insn_idx == block.get_blockid().idx {
- block.entry_exit = Some(block.get_start_addr());
+ // If this is the first instruction in the block, then
+ // the entry address is the address for block_entry_exit
+ if insn_idx == jit.starting_insn_idx {
+ jit.block_entry_exit = Some(jit.output_ptr);
}
break;
@@ -842,45 +881,28 @@ pub fn gen_single_block(
break;
}
}
+ let end_insn_idx = insn_idx;
- // Finish filling out the block
- {
- let mut block = jit.block.borrow_mut();
- if block.entry_exit.is_some() {
- asm.pad_inval_patch();
- }
-
- // Compile code into the code block
- let gc_offsets = asm.compile(cb);
-
- // Add the GC offsets to the block
- block.set_gc_obj_offsets(gc_offsets);
-
- // Set CME dependencies to the block
- block.set_cme_dependencies(jit.cme_dependencies);
-
- // Set outgoing branches to the block
- block.set_outgoing(jit.outgoing);
-
- // Mark the end position of the block
- block.set_end_addr(cb.get_write_ptr());
+ // We currently can't handle cases where the request is for a block that
+ // doesn't go to the next instruction in the same iseq.
+ assert!(!jit.record_boundary_patch_point);
- // Store the index of the last instruction in the block
- block.set_end_idx(insn_idx);
+ // Pad the block if it has the potential to be invalidated
+ if jit.block_entry_exit.is_some() {
+ asm.pad_inval_patch();
}
- // We currently can't handle cases where the request is for a block that
- // doesn't go to the next instruction.
- assert!(!jit.record_boundary_patch_point);
+ // Compile code into the code block
+ let gc_offsets = asm.compile(cb);
+ let end_addr = cb.get_write_ptr();
// If code for the block doesn't fit, fail
if cb.has_dropped_bytes() || ocb.unwrap().has_dropped_bytes() {
- free_block(&blockref);
return Err(());
}
// Block compiled successfully
- Ok(blockref)
+ Ok(jit.into_block(end_insn_idx, block_start_addr, end_addr, gc_offsets))
}
fn gen_nop(
@@ -3595,7 +3617,7 @@ fn gen_branchif(
ctx,
Some(next_block),
Some(ctx),
- BranchGenFn::BranchIf(BranchShape::Default),
+ BranchGenFn::BranchIf(Cell::new(BranchShape::Default)),
);
}
@@ -3652,7 +3674,7 @@ fn gen_branchunless(
ctx,
Some(next_block),
Some(ctx),
- BranchGenFn::BranchUnless(BranchShape::Default),
+ BranchGenFn::BranchUnless(Cell::new(BranchShape::Default)),
);
}
@@ -3706,7 +3728,7 @@ fn gen_branchnil(
ctx,
Some(next_block),
Some(ctx),
- BranchGenFn::BranchNil(BranchShape::Default),
+ BranchGenFn::BranchNil(Cell::new(BranchShape::Default)),
);
}
@@ -4533,7 +4555,7 @@ fn jit_obj_respond_to(
// Invalidate this block if method lookup changes for the method being queried. This works
// both for the case where a method does or does not exist, as for the latter we asked for a
// "negative CME" earlier.
- assume_method_lookup_stable(jit, ocb, target_cme);
+ jit.assume_method_lookup_stable(ocb, target_cme);
// Generate a side exit
let side_exit = get_side_exit(jit, ocb, ctx);
@@ -6308,7 +6330,7 @@ fn gen_send_general(
// Register block for invalidation
//assert!(cme->called_id == mid);
- assume_method_lookup_stable(jit, ocb, cme);
+ jit.assume_method_lookup_stable(ocb, cme);
// To handle the aliased method case (VM_METHOD_TYPE_ALIAS)
loop {
@@ -6478,7 +6500,7 @@ fn gen_send_general(
flags |= VM_CALL_FCALL | VM_CALL_OPT_SEND;
- assume_method_lookup_stable(jit, ocb, cme);
+ jit.assume_method_lookup_stable(ocb, cme);
let (known_class, type_mismatch_exit) = {
if compile_time_name.string_p() {
@@ -6973,8 +6995,8 @@ fn gen_invokesuper(
// We need to assume that both our current method entry and the super
// method entry we invoke remain stable
- assume_method_lookup_stable(jit, ocb, me);
- assume_method_lookup_stable(jit, ocb, cme);
+ jit.assume_method_lookup_stable(ocb, me);
+ jit.assume_method_lookup_stable(ocb, cme);
// Method calls may corrupt types
ctx.clear_local_types();
@@ -7428,14 +7450,13 @@ fn gen_opt_getconstant_path(
asm.store(stack_top, ic_entry_val);
} else {
// Optimize for single ractor mode.
- // FIXME: This leaks when st_insert raises NoMemoryError
if !assume_single_ractor_mode(jit, ocb) {
return CantCompile;
}
// Invalidate output code on any constant writes associated with
// constants referenced within the current block.
- assume_stable_constant_names(jit, ocb, idlist);
+ jit.assume_stable_constant_names(ocb, idlist);
jit_putobject(jit, ctx, asm, unsafe { (*ice).value });
}
@@ -8112,15 +8133,15 @@ mod tests {
use super::*;
fn setup_codegen() -> (JITState, Context, Assembler, CodeBlock, OutlinedCb) {
- let blockid = BlockId {
- iseq: ptr::null(),
- idx: 0,
- };
let cb = CodeBlock::new_dummy(256 * 1024);
- let block = Block::new(blockid, &Context::default(), cb.get_write_ptr());
return (
- JITState::new(&block, ptr::null()), // No execution context in tests. No peeking!
+ JITState::new(
+ BlockId { iseq: std::ptr::null(), idx: 0 },
+ Context::default(),
+ cb.get_write_ptr(),
+ ptr::null(), // No execution context in tests. No peeking!
+ ),
Context::default(),
Assembler::new(),
cb,
diff --git a/yjit/src/core.rs b/yjit/src/core.rs
index 5c8bf32e86..888f795279 100644
--- a/yjit/src/core.rs
+++ b/yjit/src/core.rs
@@ -1,3 +1,4 @@
+//! Code versioning, retained live control flow graph mutations, type tracking, etc.
use crate::asm::*;
use crate::backend::ir::*;
use crate::codegen::*;
@@ -11,12 +12,16 @@ use crate::disasm::*;
use core::ffi::c_void;
use std::cell::*;
use std::collections::HashSet;
-use std::hash::{Hash, Hasher};
+use std::fmt;
use std::mem;
-use std::rc::{Rc};
+use std::ops::Range;
+use std::rc::Rc;
+use mem::MaybeUninit;
+use std::ptr;
+use ptr::NonNull;
use YARVOpnd::*;
use TempMapping::*;
-use crate::invariants::block_assumptions_free;
+use crate::invariants::*;
// Maximum number of temp value types we keep track of
pub const MAX_TEMP_TYPES: usize = 8;
@@ -24,6 +29,10 @@ pub const MAX_TEMP_TYPES: usize = 8;
// Maximum number of local variable types we keep track of
const MAX_LOCAL_TYPES: usize = 8;
+/// An index into `ISEQ_BODY(iseq)->iseq_encoded`. Points
+/// to a YARV instruction or an instruction operand.
+pub type IseqIdx = u16;
+
// Represent the type of a value (local/stack/self) in YJIT
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Type {
@@ -410,12 +419,12 @@ pub enum BranchShape {
Default, // Neither target is next
}
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
pub enum BranchGenFn {
- BranchIf(BranchShape),
- BranchNil(BranchShape),
- BranchUnless(BranchShape),
- JumpToTarget0(BranchShape),
+ BranchIf(Cell<BranchShape>),
+ BranchNil(Cell<BranchShape>),
+ BranchUnless(Cell<BranchShape>),
+ JumpToTarget0(Cell<BranchShape>),
JNZToTarget0,
JZToTarget0,
JBEToTarget0,
@@ -423,10 +432,10 @@ pub enum BranchGenFn {
}
impl BranchGenFn {
- pub fn call(self, asm: &mut Assembler, target0: CodePtr, target1: Option<CodePtr>) {
+ pub fn call(&self, asm: &mut Assembler, target0: CodePtr, target1: Option<CodePtr>) {
match self {
BranchGenFn::BranchIf(shape) => {
- match shape {
+ match shape.get() {
BranchShape::Next0 => asm.jz(target1.unwrap().into()),
BranchShape::Next1 => asm.jnz(target0.into()),
BranchShape::Default => {
@@ -436,7 +445,7 @@ impl BranchGenFn {
}
}
BranchGenFn::BranchNil(shape) => {
- match shape {
+ match shape.get() {
BranchShape::Next0 => asm.jne(target1.unwrap().into()),
BranchShape::Next1 => asm.je(target0.into()),
BranchShape::Default => {
@@ -446,7 +455,7 @@ impl BranchGenFn {
}
}
BranchGenFn::BranchUnless(shape) => {
- match shape {
+ match shape.get() {
BranchShape::Next0 => asm.jnz(target1.unwrap().into()),
BranchShape::Next1 => asm.jz(target0.into()),
BranchShape::Default => {
@@ -456,10 +465,10 @@ impl BranchGenFn {
}
}
BranchGenFn::JumpToTarget0(shape) => {
- if shape == BranchShape::Next1 {
+ if shape.get() == BranchShape::Next1 {
panic!("Branch shape Next1 not allowed in JumpToTarget0!");
}
- if shape == BranchShape::Default {
+ if shape.get() == BranchShape::Default {
asm.jmp(target0.into());
}
}
@@ -479,12 +488,12 @@ impl BranchGenFn {
}
}
- pub fn get_shape(self) -> BranchShape {
+ pub fn get_shape(&self) -> BranchShape {
match self {
BranchGenFn::BranchIf(shape) |
BranchGenFn::BranchNil(shape) |
BranchGenFn::BranchUnless(shape) |
- BranchGenFn::JumpToTarget0(shape) => shape,
+ BranchGenFn::JumpToTarget0(shape) => shape.get(),
BranchGenFn::JNZToTarget0 |
BranchGenFn::JZToTarget0 |
BranchGenFn::JBEToTarget0 |
@@ -492,18 +501,18 @@ impl BranchGenFn {
}
}
- pub fn set_shape(&mut self, new_shape: BranchShape) {
+ pub fn set_shape(&self, new_shape: BranchShape) {
match self {
BranchGenFn::BranchIf(shape) |
BranchGenFn::BranchNil(shape) |
BranchGenFn::BranchUnless(shape) => {
- *shape = new_shape;
+ shape.set(new_shape);
}
BranchGenFn::JumpToTarget0(shape) => {
if new_shape == BranchShape::Next1 {
panic!("Branch shape Next1 not allowed in JumpToTarget0!");
}
- *shape = new_shape;
+ shape.set(new_shape);
}
BranchGenFn::JNZToTarget0 |
BranchGenFn::JZToTarget0 |
@@ -516,7 +525,7 @@ impl BranchGenFn {
}
/// A place that a branch could jump to
-#[derive(Debug)]
+#[derive(Debug, Clone)]
enum BranchTarget {
Stub(Box<BranchStub>), // Not compiled yet
Block(BlockRef), // Already compiled
@@ -526,79 +535,108 @@ impl BranchTarget {
fn get_address(&self) -> Option<CodePtr> {
match self {
BranchTarget::Stub(stub) => stub.address,
- BranchTarget::Block(blockref) => Some(blockref.borrow().start_addr),
+ BranchTarget::Block(blockref) => Some(unsafe { blockref.as_ref() }.start_addr),
}
}
fn get_blockid(&self) -> BlockId {
match self {
- BranchTarget::Stub(stub) => stub.id,
- BranchTarget::Block(blockref) => blockref.borrow().blockid,
+ BranchTarget::Stub(stub) => BlockId { iseq: stub.iseq.get(), idx: stub.iseq_idx },
+ BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.get_blockid(),
}
}
fn get_ctx(&self) -> Context {
match self {
BranchTarget::Stub(stub) => stub.ctx.clone(),
- BranchTarget::Block(blockref) => blockref.borrow().ctx.clone(),
+ BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.ctx.clone(),
}
}
fn get_block(&self) -> Option<BlockRef> {
match self {
BranchTarget::Stub(_) => None,
- BranchTarget::Block(blockref) => Some(blockref.clone()),
+ BranchTarget::Block(blockref) => Some(*blockref),
}
}
- fn set_iseq(&mut self, iseq: IseqPtr) {
+ fn set_iseq(&self, iseq: IseqPtr) {
match self {
- BranchTarget::Stub(stub) => stub.id.iseq = iseq,
- BranchTarget::Block(blockref) => blockref.borrow_mut().blockid.iseq = iseq,
+ BranchTarget::Stub(stub) => stub.iseq.set(iseq),
+ BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.iseq.set(iseq),
}
}
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
struct BranchStub {
address: Option<CodePtr>,
- id: BlockId,
+ iseq: Cell<IseqPtr>,
+ iseq_idx: IseqIdx,
ctx: Context,
}
/// Store info about an outgoing branch in a code segment
/// Note: care must be taken to minimize the size of branch objects
-#[derive(Debug)]
pub struct Branch {
// Block this is attached to
block: BlockRef,
// Positions where the generated code starts and ends
- start_addr: Option<CodePtr>,
- end_addr: Option<CodePtr>, // exclusive
+ start_addr: CodePtr,
+ end_addr: Cell<CodePtr>, // exclusive
// Branch target blocks and their contexts
- targets: [Option<Box<BranchTarget>>; 2],
+ targets: [Cell<Option<Box<BranchTarget>>>; 2],
// Branch code generation function
gen_fn: BranchGenFn,
}
+/// A [Branch] for a [Block] that is under construction.
+/// Fields correspond, but may be `None` during construction.
+pub struct PendingBranch {
+ /// Allocation holder for the address of the constructed branch
+ /// in error paths Box deallocates it.
+ uninit_branch: Box<MaybeUninit<Branch>>,
+
+ /// Branch code generation function
+ gen_fn: BranchGenFn,
+
+ /// Positions where the generated code starts and ends
+ start_addr: Cell<Option<CodePtr>>,
+ end_addr: Cell<Option<CodePtr>>, // exclusive
+
+ /// Branch target blocks and their contexts
+ targets: [Cell<Option<Box<BranchTarget>>>; 2],
+}
+
impl Branch {
// Compute the size of the branch code
fn code_size(&self) -> usize {
- (self.end_addr.unwrap().raw_ptr() as usize) - (self.start_addr.unwrap().raw_ptr() as usize)
+ (self.end_addr.get().raw_ptr() as usize) - (self.start_addr.raw_ptr() as usize)
}
/// Get the address of one of the branch destination
fn get_target_address(&self, target_idx: usize) -> Option<CodePtr> {
- self.targets[target_idx].as_ref().and_then(|target| target.get_address())
+ unsafe {
+ self.targets[target_idx]
+ .ref_unchecked()
+ .as_ref()
+ .and_then(|target| target.get_address())
+ }
}
fn get_stub_count(&self) -> usize {
let mut count = 0;
- for target in self.targets.iter().flatten() {
- if let BranchTarget::Stub(_) = target.as_ref() {
+ for target in self.targets.iter() {
+ if unsafe {
+ // SAFETY: no mutation
+ matches!(
+ target.ref_unchecked().as_ref().map(Box::as_ref),
+ Some(BranchTarget::Stub(_))
+ )
+ } {
count += 1;
}
}
@@ -606,19 +644,117 @@ impl Branch {
}
}
+impl std::fmt::Debug for Branch {
+ // Can't derive this because `targets: !Copy` due to Cell.
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let targets = unsafe {
+ // SAFETY:
+ // While the references are live for the result of this function,
+ // no mutation happens because we are only calling derived fmt::Debug functions.
+ [self.targets[0].as_ptr().as_ref().unwrap(), self.targets[1].as_ptr().as_ref().unwrap()]
+ };
+
+ formatter
+ .debug_struct("Branch")
+ .field("block", &self.block)
+ .field("start", &self.start_addr)
+ .field("end", &self.end_addr)
+ .field("targets", &targets)
+ .field("gen_fn", &self.gen_fn)
+ .finish()
+ }
+}
+
+impl PendingBranch {
+ /// Set up a branch target at `target_idx`. Find an existing block to branch to
+ /// or generate a stub for one.
+ fn set_target(
+ &self,
+ target_idx: u32,
+ target: BlockId,
+ ctx: &Context,
+ ocb: &mut OutlinedCb,
+ ) -> Option<CodePtr> {
+ // If the block already exists
+ if let Some(blockref) = find_block_version(target, ctx) {
+ let block = unsafe { blockref.as_ref() };
+
+ // Fill out the target with this block
+ self.targets[target_idx.as_usize()]
+ .set(Some(Box::new(BranchTarget::Block(blockref))));
+ return Some(block.start_addr);
+ }
+
+ // The branch struct is uninitialized right now but as a stable address.
+ // We make sure the stub runs after the branch is initialized.
+ let branch_struct_addr = self.uninit_branch.as_ptr() as usize;
+ let stub_addr = gen_call_branch_stub_hit(ocb, branch_struct_addr, target_idx);
+
+ if let Some(stub_addr) = stub_addr {
+ // Fill the branch target with a stub
+ self.targets[target_idx.as_usize()].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
+ address: Some(stub_addr),
+ iseq: Cell::new(target.iseq),
+ iseq_idx: target.idx,
+ ctx: ctx.clone(),
+ })))));
+ }
+
+ stub_addr
+ }
+
+ // Construct the branch and wire it up in the grpah
+ fn into_branch(mut self, uninit_block: BlockRef) -> BranchRef {
+ // Make the branch
+ let branch = Branch {
+ block: uninit_block,
+ start_addr: self.start_addr.get().unwrap(),
+ end_addr: Cell::new(self.end_addr.get().unwrap()),
+ targets: self.targets,
+ gen_fn: self.gen_fn,
+ };
+ // Move it to the designated place on
+ // the heap and unwrap MaybeUninit.
+ self.uninit_branch.write(branch);
+ let raw_branch: *mut MaybeUninit<Branch> = Box::into_raw(self.uninit_branch);
+ let branchref = NonNull::new(raw_branch as *mut Branch).expect("no null from Box");
+
+ // SAFETY: just allocated it
+ let branch = unsafe { branchref.as_ref() };
+ // For block branch targets, put the new branch in the
+ // appropriate incoming list.
+ for target in branch.targets.iter() {
+ // SAFETY: no mutation
+ let out_block: Option<BlockRef> = unsafe {
+ target.ref_unchecked().as_ref().and_then(|target| target.get_block())
+ };
+
+ if let Some(out_block) = out_block {
+ // SAFETY: These blockrefs come from set_target() which only puts blocks from
+ // ISeqs, which are all initialized. Note that uninit_block isn't in any ISeq
+ // payload yet.
+ unsafe { out_block.as_ref() }.incoming.push(branchref);
+ }
+ }
+
+ branchref
+ }
+}
+
// In case a block is invalidated, this helps to remove all pointers to the block.
pub type CmePtr = *const rb_callable_method_entry_t;
/// Basic block version
/// Represents a portion of an iseq compiled with a given context
/// Note: care must be taken to minimize the size of block_t objects
-#[derive(Clone, Debug)]
+#[derive(Debug)]
pub struct Block {
- // Bytecode sequence (iseq, idx) this is a version of
- blockid: BlockId,
+ // The byte code instruction sequence this is a version of.
+ // Can change due to moving GC.
+ iseq: Cell<IseqPtr>,
- // Index one past the last instruction for this block in the iseq
- end_idx: u16,
+ // Index range covered by this version in `ISEQ_BODY(iseq)->iseq_encoded`.
+ iseq_range: Range<IseqIdx>,
// Context at the start of the block
// This should never be mutated
@@ -626,11 +762,11 @@ pub struct Block {
// Positions where the generated code starts and ends
start_addr: CodePtr,
- end_addr: Option<CodePtr>,
+ end_addr: Cell<CodePtr>,
// List of incoming branches (from predecessors)
// These are reference counted (ownership shared between predecessor and successors)
- incoming: Vec<BranchRef>,
+ incoming: MutableBranchList,
// NOTE: we might actually be able to store the branches here without refcounting
// however, using a RefCell makes it easy to get a pointer to Branch objects
@@ -644,20 +780,38 @@ pub struct Block {
// CME dependencies of this block, to help to remove all pointers to this
// block in the system.
- cme_dependencies: Box<[CmePtr]>,
+ cme_dependencies: Box<[Cell<CmePtr>]>,
// Code address of an exit for `ctx` and `blockid`.
// Used for block invalidation.
- pub entry_exit: Option<CodePtr>,
-}
-
-/// Reference-counted pointer to a block that can be borrowed mutably.
-/// Wrapped so we could implement [Hash] and [Eq] for use with stdlib collections.
-#[derive(Debug)]
-pub struct BlockRef(Rc<RefCell<Block>>);
-
-/// Reference-counted pointer to a branch that can be borrowed mutably
-pub type BranchRef = Rc<RefCell<Branch>>;
+ entry_exit: Option<CodePtr>,
+}
+
+/// Pointer to a [Block].
+///
+/// # Safety
+///
+/// _Never_ derive a `&mut Block` from this and always use
+/// [std::ptr::NonNull::as_ref] to get a `&Block`. `&'a mut`
+/// in Rust asserts that there are no other references live
+/// over the lifetime `'a`. This uniqueness assertion does
+/// not hold in many situations for us, even when you ignore
+/// the fact that our control flow graph can have cycles.
+/// Here are just two examples where we have overlapping references:
+/// - Yielding to a different OS thread within the same
+/// ractor during compilation
+/// - The GC calling [rb_yjit_iseq_mark] during compilation
+///
+/// Technically, for soundness, we also need to ensure that
+/// the we have the VM lock while the result of `as_ref()`
+/// is live, so that no deallocation happens while the
+/// shared reference is live. The vast majority of our code run while
+/// holding the VM lock, though.
+pub type BlockRef = NonNull<Block>;
+
+/// Pointer to a [Branch]. See [BlockRef] for notes about
+/// proper usage.
+pub type BranchRef = NonNull<Branch>;
/// List of block versions for a given blockid
type VersionList = Vec<BlockRef>;
@@ -666,47 +820,33 @@ type VersionList = Vec<BlockRef>;
/// An instance of this is stored on each iseq
type VersionMap = Vec<VersionList>;
-impl BlockRef {
- /// Constructor
- pub fn new(rc: Rc<RefCell<Block>>) -> Self {
- Self(rc)
- }
-
- /// Borrow the block through [RefCell].
- pub fn borrow(&self) -> Ref<'_, Block> {
- self.0.borrow()
- }
-
- /// Borrow the block for mutation through [RefCell].
- pub fn borrow_mut(&self) -> RefMut<'_, Block> {
- self.0.borrow_mut()
- }
-}
+/// [Interior mutability][1] wrapper for a list of branches.
+/// O(n) insertion, but space efficient. We generally expect
+/// blocks to have only a few branches.
+///
+/// [1]: https://doc.rust-lang.org/std/cell/struct.UnsafeCell.html
+#[repr(transparent)]
+struct MutableBranchList(Cell<Box<[BranchRef]>>);
-impl Clone for BlockRef {
- /// Clone the [Rc]
- fn clone(&self) -> Self {
- Self(self.0.clone())
+impl MutableBranchList {
+ fn push(&self, branch: BranchRef) {
+ // Temporary move the boxed slice out of self.
+ // oom=abort is load bearing here...
+ let mut current_list = self.0.take().into_vec();
+ current_list.push(branch);
+ self.0.set(current_list.into_boxed_slice());
}
}
-impl Hash for BlockRef {
- /// Hash the reference by hashing the pointer
- fn hash<H: Hasher>(&self, state: &mut H) {
- let rc_ptr = Rc::as_ptr(&self.0);
- rc_ptr.hash(state);
- }
-}
+impl fmt::Debug for MutableBranchList {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ // SAFETY: the derived Clone for boxed slices does not mutate this Cell
+ let branches = unsafe { self.0.ref_unchecked().clone() };
-impl PartialEq for BlockRef {
- /// Equality defined by allocation identity
- fn eq(&self, other: &Self) -> bool {
- Rc::ptr_eq(&self.0, &other.0)
+ formatter.debug_list().entries(branches.into_iter()).finish()
}
}
-/// It's comparison by identity so all the requirements are satisfied
-impl Eq for BlockRef {}
/// This is all the data YJIT stores on an iseq
/// This will be dynamically allocated by C code
@@ -845,15 +985,21 @@ pub extern "C" fn rb_yjit_iseq_free(payload: *mut c_void) {
// SAFETY: We got the pointer from Box::into_raw().
let payload = unsafe { Box::from_raw(payload) };
- // Increment the freed iseq count
- incr_counter!(freed_iseq_count);
-
- // Free all blocks in the payload
+ // Free all blocks in version_map. The GC doesn't free running iseqs.
for versions in &payload.version_map {
for block in versions {
- free_block(block);
+ // SAFETY: blocks in the version_map are always well connected
+ unsafe { free_block(*block, true) };
}
}
+
+ // Free dead blocks
+ for block in payload.dead_blocks {
+ unsafe { free_block(block, false) };
+ }
+
+ // Increment the freed iseq count
+ incr_counter!(freed_iseq_count);
}
/// GC callback for marking GC objects in the the per-iseq payload.
@@ -880,20 +1026,26 @@ pub extern "C" fn rb_yjit_iseq_mark(payload: *mut c_void) {
for versions in &payload.version_map {
for block in versions {
- let block = block.borrow();
+ // SAFETY: all blocks inside version_map are initialized.
+ let block = unsafe { block.as_ref() };
- unsafe { rb_gc_mark_movable(block.blockid.iseq.into()) };
+ unsafe { rb_gc_mark_movable(block.iseq.get().into()) };
// Mark method entry dependencies
- for &cme_dep in block.cme_dependencies.iter() {
- unsafe { rb_gc_mark_movable(cme_dep.into()) };
+ for cme_dep in block.cme_dependencies.iter() {
+ unsafe { rb_gc_mark_movable(cme_dep.get().into()) };
}
// Mark outgoing branch entries
for branch in block.outgoing.iter() {
- let branch = branch.borrow();
- for target in branch.targets.iter().flatten() {
- unsafe { rb_gc_mark_movable(target.get_blockid().iseq.into()) };
+ let branch = unsafe { branch.as_ref() };
+ for target in branch.targets.iter() {
+ // SAFETY: no mutation inside unsafe
+ let target_iseq = unsafe { target.ref_unchecked().as_ref().map(|target| target.get_blockid().iseq) };
+
+ if let Some(target_iseq) = target_iseq {
+ unsafe { rb_gc_mark_movable(target_iseq.into()) };
+ }
}
}
@@ -941,13 +1093,16 @@ pub extern "C" fn rb_yjit_iseq_update_references(payload: *mut c_void) {
for versions in &payload.version_map {
for version in versions {
- let mut block = version.borrow_mut();
+ // SAFETY: all blocks inside version_map are initialized
+ let block = unsafe { version.as_ref() };
- block.blockid.iseq = unsafe { rb_gc_location(block.blockid.iseq.into()) }.as_iseq();
+ block.iseq.set(unsafe { rb_gc_location(block.iseq.get().into()) }.as_iseq());
// Update method entry dependencies
- for cme_dep in block.cme_dependencies.iter_mut() {
- *cme_dep = unsafe { rb_gc_location((*cme_dep).into()) }.as_cme();
+ for cme_dep in block.cme_dependencies.iter() {
+ let cur_cme: VALUE = cme_dep.get().into();
+ let new_cme = unsafe { rb_gc_location(cur_cme) }.as_cme();
+ cme_dep.set(new_cme);
}
// Walk over references to objects in generated code.
@@ -973,12 +1128,19 @@ pub extern "C" fn rb_yjit_iseq_update_references(payload: *mut c_void) {
}
// Update outgoing branch entries
- let outgoing_branches = block.outgoing.clone(); // clone to use after borrow
- mem::drop(block); // end mut borrow: target.set_iseq and target.get_blockid() might (mut) borrow it
- for branch in outgoing_branches.iter() {
- let mut branch = branch.borrow_mut();
- for target in branch.targets.iter_mut().flatten() {
- target.set_iseq(unsafe { rb_gc_location(target.get_blockid().iseq.into()) }.as_iseq());
+ for branch in block.outgoing.iter() {
+ let branch = unsafe { branch.as_ref() };
+ for target in branch.targets.iter() {
+ // SAFETY: no mutation inside unsafe
+ let current_iseq = unsafe { target.ref_unchecked().as_ref().map(|target| target.get_blockid().iseq) };
+
+ if let Some(current_iseq) = current_iseq {
+ let updated_iseq = unsafe { rb_gc_location(current_iseq.into()) }
+ .as_iseq();
+ // SAFETY: the Cell::set is not on the reference given out
+ // by ref_unchecked.
+ unsafe { target.ref_unchecked().as_ref().unwrap().set_iseq(updated_iseq) };
+ }
}
}
}
@@ -1058,7 +1220,7 @@ pub fn get_or_create_iseq_block_list(iseq: IseqPtr) -> Vec<BlockRef> {
// For each version at this instruction index
for version in version_list {
// Clone the block ref and add it to the list
- blocks.push(version.clone());
+ blocks.push(*version);
}
}
@@ -1078,13 +1240,14 @@ fn find_block_version(blockid: BlockId, ctx: &Context) -> Option<BlockRef> {
let mut best_diff = usize::MAX;
// For each version matching the blockid
- for blockref in versions.iter_mut() {
- let block = blockref.borrow();
+ for blockref in versions.iter() {
+ let block = unsafe { blockref.as_ref() };
+
// Note that we always prefer the first matching
// version found because of inline-cache chains
match ctx.diff(&block.ctx) {
TypeDiff::Compatible(diff) if diff < best_diff => {
- best_version = Some(blockref.clone());
+ best_version = Some(*blockref);
best_diff = diff;
}
_ => {}
@@ -1130,23 +1293,40 @@ pub fn limit_block_versions(blockid: BlockId, ctx: &Context) -> Context {
return ctx.clone();
}
-/// Keep track of a block version. Block should be fully constructed.
-/// Uses `cb` for running write barriers.
-fn add_block_version(blockref: &BlockRef, cb: &CodeBlock) {
- let block = blockref.borrow();
+/// Install a block version into its [IseqPayload], letting the GC track its
+/// lifetime, and allowing it to be considered for use for other
+/// blocks we might generate. Uses `cb` for running write barriers.
+///
+/// # Safety
+///
+/// The block must be fully initialized. Its incoming and outgoing edges,
+/// if there are any, must point to initialized blocks, too.
+///
+/// Note that the block might gain edges after this function returns,
+/// as can happen during [gen_block_series]. Initialized here doesn't mean
+/// ready to be consumed or that the machine code tracked by the block is
+/// ready to be run.
+///
+/// Due to this transient state where a block is tracked by the GC by
+/// being inside an [IseqPayload] but not ready to be executed, it's
+/// generally unsound to call any Ruby methods during codegen. That has
+/// the potential to run blocks which are not ready.
+unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) {
+ // SAFETY: caller ensures initialization
+ let block = unsafe { blockref.as_ref() };
// Function entry blocks must have stack size 0
- assert!(!(block.blockid.idx == 0 && block.ctx.stack_size > 0));
+ assert!(!(block.iseq_range.start == 0 && block.ctx.stack_size > 0));
- let version_list = get_or_create_version_list(block.blockid);
+ let version_list = get_or_create_version_list(block.get_blockid());
- version_list.push(blockref.clone());
+ version_list.push(blockref);
version_list.shrink_to_fit();
// By writing the new block to the iseq, the iseq now
// contains new references to Ruby objects. Run write barriers.
- let iseq: VALUE = block.blockid.iseq.into();
- for &dep in block.iter_cme_deps() {
+ let iseq: VALUE = block.iseq.get().into();
+ for dep in block.iter_cme_deps() {
obj_written!(iseq, dep.into());
}
@@ -1163,16 +1343,16 @@ fn add_block_version(blockref: &BlockRef, cb: &CodeBlock) {
incr_counter!(compiled_block_count);
// Mark code pages for code GC
- let iseq_payload = get_iseq_payload(block.blockid.iseq).unwrap();
- for page in cb.addrs_to_pages(block.start_addr, block.end_addr.unwrap()) {
+ let iseq_payload = get_iseq_payload(block.iseq.get()).unwrap();
+ for page in cb.addrs_to_pages(block.start_addr, block.end_addr.get()) {
iseq_payload.pages.insert(page);
}
}
/// Remove a block version from the version map of its parent ISEQ
fn remove_block_version(blockref: &BlockRef) {
- let block = blockref.borrow();
- let version_list = match get_version_list(block.blockid) {
+ let block = unsafe { blockref.as_ref() };
+ let version_list = match get_version_list(block.get_blockid()) {
Some(version_list) => version_list,
None => return,
};
@@ -1181,47 +1361,73 @@ fn remove_block_version(blockref: &BlockRef) {
version_list.retain(|other| blockref != other);
}
-//===========================================================================
-// I put the implementation of traits for core.rs types below
-// We can move these closer to the above structs later if we want.
-//===========================================================================
+impl JITState {
+ // Finish compiling and turn a jit state into a block
+ // note that the block is still not in shape.
+ pub fn into_block(self, end_insn_idx: IseqIdx, start_addr: CodePtr, end_addr: CodePtr, gc_obj_offsets: Vec<u32>) -> BlockRef {
+ // Allocate the block and get its pointer
+ let blockref: *mut MaybeUninit<Block> = Box::into_raw(Box::new(MaybeUninit::uninit()));
-impl Block {
- pub fn new(blockid: BlockId, ctx: &Context, start_addr: CodePtr) -> BlockRef {
- let block = Block {
- blockid,
- end_idx: 0,
- ctx: ctx.clone(),
+ incr_counter_by!(num_gc_obj_refs, gc_obj_offsets.len());
+
+ // Make the new block
+ let block = MaybeUninit::new(Block {
start_addr,
- end_addr: None,
- incoming: Vec::new(),
- outgoing: Box::new([]),
- gc_obj_offsets: Box::new([]),
- cme_dependencies: Box::new([]),
- entry_exit: None,
- };
+ iseq: Cell::new(self.get_iseq()),
+ iseq_range: self.get_starting_insn_idx()..end_insn_idx,
+ ctx: self.get_starting_ctx(),
+ end_addr: Cell::new(end_addr),
+ incoming: MutableBranchList(Cell::default()),
+ gc_obj_offsets: gc_obj_offsets.into_boxed_slice(),
+ entry_exit: self.get_block_entry_exit(),
+ cme_dependencies: self.method_lookup_assumptions.into_iter().map(Cell::new).collect(),
+ // Pending branches => actual branches
+ outgoing: self.pending_outgoing.into_iter().map(|pending_out| {
+ let pending_out = Rc::try_unwrap(pending_out)
+ .ok().expect("all PendingBranchRefs should be unique when ready to construct a Block");
+ pending_out.into_branch(NonNull::new(blockref as *mut Block).expect("no null from Box"))
+ }).collect()
+ });
+ // Initialize it on the heap
+ // SAFETY: allocated with Box above
+ unsafe { ptr::write(blockref, block) };
- // Wrap the block in a reference counted refcell
- // so that the block ownership can be shared
- BlockRef::new(Rc::new(RefCell::new(block)))
- }
+ // Block is initialized now. Note that MaybeUnint<T> has the same layout as T.
+ let blockref = NonNull::new(blockref as *mut Block).expect("no null from Box");
- pub fn get_blockid(&self) -> BlockId {
- self.blockid
+ // Track all the assumptions the block makes as invariants
+ if self.block_assumes_single_ractor {
+ track_single_ractor_assumption(blockref);
+ }
+ for bop in self.bop_assumptions {
+ track_bop_assumption(blockref, bop);
+ }
+ // SAFETY: just allocated it above
+ for cme in unsafe { blockref.as_ref() }.cme_dependencies.iter() {
+ track_method_lookup_stability_assumption(blockref, cme.get());
+ }
+ if let Some(idlist) = self.stable_constant_names_assumption {
+ track_stable_constant_names_assumption(blockref, idlist);
+ }
+
+ blockref
}
+}
- pub fn get_end_idx(&self) -> u16 {
- self.end_idx
+impl Block {
+ pub fn get_blockid(&self) -> BlockId {
+ BlockId { iseq: self.iseq.get(), idx: self.iseq_range.start }
}
- pub fn get_ctx(&self) -> Context {
- self.ctx.clone()
+ pub fn get_end_idx(&self) -> IseqIdx {
+ self.iseq_range.end
}
pub fn get_ctx_count(&self) -> usize {
let mut count = 1; // block.ctx
for branch in self.outgoing.iter() {
- count += branch.borrow().get_stub_count();
+ // SAFETY: &self implies it's initialized
+ count += unsafe { branch.as_ref() }.get_stub_count();
}
count
}
@@ -1232,57 +1438,23 @@ impl Block {
}
#[allow(unused)]
- pub fn get_end_addr(&self) -> Option<CodePtr> {
- self.end_addr
+ pub fn get_end_addr(&self) -> CodePtr {
+ self.end_addr.get()
}
/// Get an immutable iterator over cme dependencies
- pub fn iter_cme_deps(&self) -> std::slice::Iter<'_, CmePtr> {
- self.cme_dependencies.iter()
- }
-
- /// Set the end address in the generated for the block
- /// This can be done only once for a block
- pub fn set_end_addr(&mut self, addr: CodePtr) {
- // TODO: assert constraint that blocks can shrink but not grow in length
- self.end_addr = Some(addr);
- }
-
- /// Set the index of the last instruction in the block
- /// This can be done only once for a block
- pub fn set_end_idx(&mut self, end_idx: u16) {
- assert!(self.end_idx == 0);
- self.end_idx = end_idx;
- }
-
- pub fn set_gc_obj_offsets(self: &mut Block, gc_offsets: Vec<u32>) {
- assert_eq!(self.gc_obj_offsets.len(), 0);
- if !gc_offsets.is_empty() {
- incr_counter_by!(num_gc_obj_refs, gc_offsets.len());
- self.gc_obj_offsets = gc_offsets.into_boxed_slice();
- }
- }
-
- /// Instantiate a new CmeDependency struct and add it to the list of
- /// dependencies for this block.
- pub fn set_cme_dependencies(&mut self, cme_dependencies: Vec<CmePtr>) {
- self.cme_dependencies = cme_dependencies.into_boxed_slice();
+ pub fn iter_cme_deps(&self) -> impl Iterator<Item = CmePtr> + '_ {
+ self.cme_dependencies.iter().map(Cell::get)
}
// Push an incoming branch ref and shrink the vector
- fn push_incoming(&mut self, branch: BranchRef) {
+ fn push_incoming(&self, branch: BranchRef) {
self.incoming.push(branch);
- self.incoming.shrink_to_fit();
- }
-
- // Push an outgoing branch ref and shrink the vector
- pub fn set_outgoing(&mut self, outgoing: Vec<BranchRef>) {
- self.outgoing = outgoing.into_boxed_slice();
}
// Compute the size of the block code
pub fn code_size(&self) -> usize {
- (self.end_addr.unwrap().raw_ptr() as usize) - (self.start_addr.raw_ptr() as usize)
+ (self.end_addr.get().into_usize()) - (self.start_addr.into_usize())
}
}
@@ -1704,39 +1876,43 @@ fn gen_block_series_body(
// Generate code for the first block
let first_block = gen_single_block(blockid, start_ctx, ec, cb, ocb).ok()?;
- batch.push(first_block.clone()); // Keep track of this block version
+ batch.push(first_block); // Keep track of this block version
// Add the block version to the VersionMap for this ISEQ
- add_block_version(&first_block, cb);
+ unsafe { add_block_version(first_block, cb) };
// Loop variable
- let mut last_blockref = first_block.clone();
+ let mut last_blockref = first_block;
loop {
// Get the last outgoing branch from the previous block.
let last_branchref = {
- let last_block = last_blockref.borrow();
+ let last_block = unsafe { last_blockref.as_ref() };
match last_block.outgoing.last() {
- Some(branch) => branch.clone(),
+ Some(branch) => *branch,
None => {
break;
} // If last block has no branches, stop.
}
};
- let mut last_branch = last_branchref.borrow_mut();
+ let last_branch = unsafe { last_branchref.as_ref() };
+
+ incr_counter!(block_next_count);
// gen_direct_jump() can request a block to be placed immediately after by
// leaving a single target that has a `None` address.
- let last_target = match &mut last_branch.targets {
- [Some(last_target), None] if last_target.get_address().is_none() => last_target,
- _ => break
+ // SAFETY: no mutation inside the unsafe block
+ let (requested_blockid, requested_ctx) = unsafe {
+ match (last_branch.targets[0].ref_unchecked(), last_branch.targets[1].ref_unchecked()) {
+ (Some(last_target), None) if last_target.get_address().is_none() => {
+ (last_target.get_blockid(), last_target.get_ctx())
+ }
+ _ => {
+ // We're done when no fallthrough block is requested
+ break;
+ }
+ }
};
- incr_counter!(block_next_count);
-
- // Get id and context for the new block
- let requested_blockid = last_target.get_blockid();
- let requested_ctx = last_target.get_ctx();
-
// Generate new block using context from the last branch.
let result = gen_single_block(requested_blockid, &requested_ctx, ec, cb, ocb);
@@ -1744,10 +1920,10 @@ fn gen_block_series_body(
if result.is_err() {
// Remove previously compiled block
// versions from the version map
- mem::drop(last_branch); // end borrow
- for blockref in &batch {
- free_block(blockref);
- remove_block_version(blockref);
+ for blockref in batch {
+ remove_block_version(&blockref);
+ // SAFETY: block was well connected because it was in a version_map
+ unsafe { free_block(blockref, false) };
}
// Stop compiling
@@ -1757,16 +1933,14 @@ fn gen_block_series_body(
let new_blockref = result.unwrap();
// Add the block version to the VersionMap for this ISEQ
- add_block_version(&new_blockref, cb);
+ unsafe { add_block_version(new_blockref, cb) };
// Connect the last branch and the new block
- last_branch.targets[0] = Some(Box::new(BranchTarget::Block(new_blockref.clone())));
- new_blockref
- .borrow_mut()
- .push_incoming(last_branchref.clone());
+ last_branch.targets[0].set(Some(Box::new(BranchTarget::Block(new_blockref))));
+ unsafe { new_blockref.as_ref().incoming.push(last_branchref) };
// Track the block
- batch.push(new_blockref.clone());
+ batch.push(new_blockref);
// Repeat with newest block
last_blockref = new_blockref;
@@ -1777,12 +1951,12 @@ fn gen_block_series_body(
// If dump_iseq_disasm is active, see if this iseq's location matches the given substring.
// If so, we print the new blocks to the console.
if let Some(substr) = get_option_ref!(dump_iseq_disasm).as_ref() {
- let blockid_idx = blockid.idx;
- let iseq_location = iseq_get_location(blockid.iseq, blockid_idx);
+ let iseq_location = iseq_get_location(blockid.iseq, blockid.idx);
if iseq_location.contains(substr) {
- let last_block = last_blockref.borrow();
- println!("Compiling {} block(s) for {}, ISEQ offsets [{}, {})", batch.len(), iseq_location, blockid_idx, last_block.end_idx);
- print!("{}", disasm_iseq_insn_range(blockid.iseq, blockid.idx, last_block.end_idx));
+ let last_block = unsafe { last_blockref.as_ref() };
+ let iseq_range = &last_block.iseq_range;
+ println!("Compiling {} block(s) for {}, ISEQ offsets [{}, {})", batch.len(), iseq_location, iseq_range.start, iseq_range.end);
+ print!("{}", disasm_iseq_insn_range(blockid.iseq, iseq_range.start, iseq_range.end));
}
}
}
@@ -1829,8 +2003,8 @@ pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option<CodePtr> {
// If the block contains no Ruby instructions
Some(block) => {
- let block = block.borrow();
- if block.end_idx == insn_idx {
+ let block = unsafe { block.as_ref() };
+ if block.iseq_range.is_empty() {
return None;
}
}
@@ -1841,13 +2015,14 @@ pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option<CodePtr> {
}
/// Generate code for a branch, possibly rewriting and changing the size of it
-fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) {
+fn regenerate_branch(cb: &mut CodeBlock, branch: &Branch) {
// Remove old comments
- if let (Some(start_addr), Some(end_addr)) = (branch.start_addr, branch.end_addr) {
- cb.remove_comments(start_addr, end_addr)
- }
+ cb.remove_comments(branch.start_addr, branch.end_addr.get());
- let branch_terminates_block = branch.end_addr == branch.block.borrow().end_addr;
+ // SAFETY: having a &Branch implies branch.block is initialized.
+ let block = unsafe { branch.block.as_ref() };
+
+ let branch_terminates_block = branch.end_addr.get() == block.get_end_addr();
// Generate the branch
let mut asm = Assembler::new();
@@ -1861,17 +2036,17 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) {
// Rewrite the branch
let old_write_pos = cb.get_write_pos();
let old_dropped_bytes = cb.has_dropped_bytes();
- cb.set_write_ptr(branch.start_addr.unwrap());
+ cb.set_write_ptr(branch.start_addr);
cb.set_dropped_bytes(false);
asm.compile(cb);
+ let new_end_addr = cb.get_write_ptr();
- branch.end_addr = Some(cb.get_write_ptr());
+ branch.end_addr.set(new_end_addr);
// The block may have shrunk after the branch is rewritten
- let mut block = branch.block.borrow_mut();
if branch_terminates_block {
// Adjust block size
- block.end_addr = branch.end_addr;
+ block.end_addr.set(new_end_addr);
}
// cb.write_pos is both a write cursor and a marker for the end of
@@ -1891,34 +2066,29 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) {
}
}
-/// Create a new outgoing branch entry for a block
-fn make_branch_entry(jit: &mut JITState, block: &BlockRef, gen_fn: BranchGenFn) -> BranchRef {
- let branch = Branch {
- // Block this is attached to
- block: block.clone(),
-
- // Positions where the generated code starts and ends
- start_addr: None,
- end_addr: None,
+pub type PendingBranchRef = Rc<PendingBranch>;
- // Branch target blocks and their contexts
- targets: [None, None],
-
- // Branch code generation function
+/// Create a new outgoing branch entry for a block
+fn new_pending_branch(jit: &mut JITState, gen_fn: BranchGenFn) -> PendingBranchRef {
+ let branch = Rc::new(PendingBranch {
+ uninit_branch: Box::new(MaybeUninit::uninit()),
gen_fn,
- };
+ start_addr: Cell::new(None),
+ end_addr: Cell::new(None),
+ targets: [Cell::new(None), Cell::new(None)],
+ });
+
+ incr_counter!(compiled_branch_count); // TODO not true. count at finalize time
// Add to the list of outgoing branches for the block
- let branchref = Rc::new(RefCell::new(branch));
- jit.push_outgoing(branchref.clone());
- incr_counter!(compiled_branch_count);
+ jit.queue_outgoing_branch(branch.clone());
- return branchref;
+ branch
}
c_callable! {
/// Generated code calls this function with the SysV calling convention.
- /// See [set_branch_target].
+ /// See [gen_call_branch_stub_hit].
fn branch_stub_hit(
branch_ptr: *const c_void,
target_idx: u32,
@@ -1937,39 +2107,38 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) -
println!("branch_stub_hit");
}
- assert!(!branch_ptr.is_null());
-
- //branch_ptr is actually:
- //branch_ptr: *const RefCell<Branch>
- let branch_rc = unsafe { BranchRef::from_raw(branch_ptr as *const RefCell<Branch>) };
-
- // We increment the strong count because we want to keep the reference owned
- // by the branch stub alive. Return branch stubs can be hit multiple times.
- unsafe { Rc::increment_strong_count(branch_ptr) };
+ let branch_ref = NonNull::<Branch>::new(branch_ptr as *mut Branch)
+ .expect("Branches should not be null");
- let mut branch = branch_rc.borrow_mut();
+ // SAFETY: We have the VM lock, and the branch is initialized by the time generated
+ // code calls this function.
+ let branch = unsafe { branch_ref.as_ref() };
let branch_size_on_entry = branch.code_size();
+ let housing_block = unsafe { branch.block.as_ref() };
let target_idx: usize = target_idx.as_usize();
- let target = branch.targets[target_idx].as_ref().unwrap();
- let target_blockid = target.get_blockid();
- let target_ctx = target.get_ctx();
-
let target_branch_shape = match target_idx {
0 => BranchShape::Next0,
1 => BranchShape::Next1,
_ => unreachable!("target_idx < 2 must always hold"),
};
+ let (target_blockid, target_ctx): (BlockId, Context) = unsafe {
+ // SAFETY: no mutation of the target's Cell. Just reading out data.
+ let target = branch.targets[target_idx].ref_unchecked().as_ref().unwrap();
+
+ // If this branch has already been patched, return the dst address
+ // Note: recursion can cause the same stub to be hit multiple times
+ if let BranchTarget::Block(_) = target.as_ref() {
+ return target.get_address().unwrap().raw_ptr();
+ }
+
+ (target.get_blockid(), target.get_ctx())
+ };
+
let cb = CodegenGlobals::get_inline_cb();
let ocb = CodegenGlobals::get_outlined_cb();
- // If this branch has already been patched, return the dst address
- // Note: ractors can cause the same stub to be hit multiple times
- if let BranchTarget::Block(_) = target.as_ref() {
- return target.get_address().unwrap().raw_ptr();
- }
-
let (cfp, original_interp_sp) = unsafe {
let cfp = get_ec_cfp(ec);
let original_interp_sp = get_cfp_sp(cfp);
@@ -1996,78 +2165,70 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) -
// Try to find an existing compiled version of this block
let mut block = find_block_version(target_blockid, &target_ctx);
-
+ let mut branch_modified = false;
// If this block hasn't yet been compiled
if block.is_none() {
let branch_old_shape = branch.gen_fn.get_shape();
- let mut branch_modified = false;
+
// If the new block can be generated right after the branch (at cb->write_pos)
- if Some(cb.get_write_ptr()) == branch.end_addr {
+ if cb.get_write_ptr() == branch.end_addr.get() {
// This branch should be terminating its block
- assert!(branch.end_addr == branch.block.borrow().end_addr);
+ assert!(branch.end_addr == housing_block.end_addr);
// Change the branch shape to indicate the target block will be placed next
branch.gen_fn.set_shape(target_branch_shape);
// Rewrite the branch with the new, potentially more compact shape
- regenerate_branch(cb, &mut branch);
+ regenerate_branch(cb, branch);
branch_modified = true;
// Ensure that the branch terminates the codeblock just like
// before entering this if block. This drops bytes off the end
// in case we shrank the branch when regenerating.
- cb.set_write_ptr(branch.end_addr.unwrap());
+ cb.set_write_ptr(branch.end_addr.get());
}
// Compile the new block version
- drop(branch); // Stop mutable RefCell borrow since GC might borrow branch for marking
block = gen_block_series(target_blockid, &target_ctx, ec, cb, ocb);
- branch = branch_rc.borrow_mut();
if block.is_none() && branch_modified {
// We couldn't generate a new block for the branch, but we modified the branch.
// Restore the branch by regenerating it.
branch.gen_fn.set_shape(branch_old_shape);
- regenerate_branch(cb, &mut branch);
+ regenerate_branch(cb, branch);
}
}
// Finish building the new block
let dst_addr = match block {
- Some(block_rc) => {
- let mut block: RefMut<_> = block_rc.borrow_mut();
+ Some(new_block) => {
+ let new_block = unsafe { new_block.as_ref() };
// Branch shape should reflect layout
- assert!(!(branch.gen_fn.get_shape() == target_branch_shape && Some(block.start_addr) != branch.end_addr));
+ assert!(!(branch.gen_fn.get_shape() == target_branch_shape && new_block.start_addr != branch.end_addr.get()));
// Add this branch to the list of incoming branches for the target
- block.push_incoming(branch_rc.clone());
- mem::drop(block); // end mut borrow
+ new_block.push_incoming(branch_ref);
// Update the branch target address
- branch.targets[target_idx] = Some(Box::new(BranchTarget::Block(block_rc.clone())));
+ branch.targets[target_idx].set(Some(Box::new(BranchTarget::Block(new_block.into()))));
// Rewrite the branch with the new jump target address
- regenerate_branch(cb, &mut branch);
+ regenerate_branch(cb, branch);
// Restore interpreter sp, since the code hitting the stub expects the original.
unsafe { rb_set_cfp_sp(cfp, original_interp_sp) };
- block_rc.borrow().start_addr
+ new_block.start_addr
}
None => {
- // Code GC needs to borrow blocks for invalidation, so their mutable
- // borrows must be dropped first.
- drop(block);
- drop(branch);
// Trigger code GC. The whole ISEQ will be recompiled later.
// We shouldn't trigger it in the middle of compilation in branch_stub_hit
// because incomplete code could be used when cb.dropped_bytes is flipped
// by code GC. So this place, after all compilation, is the safest place
// to hook code GC on branch_stub_hit.
cb.code_gc();
- branch = branch_rc.borrow_mut();
// Failed to service the stub by generating a new block so now we
// need to exit to the interpreter at the stubbed location. We are
@@ -2087,55 +2248,35 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) -
assert!(
new_branch_size <= branch_size_on_entry,
"branch stubs should never enlarge branches (start_addr: {:?}, old_size: {}, new_size: {})",
- branch.start_addr.unwrap().raw_ptr(), branch_size_on_entry, new_branch_size,
+ branch.start_addr.raw_ptr(), branch_size_on_entry, new_branch_size,
);
// Return a pointer to the compiled block version
dst_addr.raw_ptr()
}
-/// Set up a branch target at an index with a block version or a stub
-fn set_branch_target(
- target_idx: u32,
- target: BlockId,
- ctx: &Context,
- branchref: &BranchRef,
- branch: &mut Branch,
+/// Generate a "stub", a piece of code that calls the compiler back when run.
+/// A piece of code that redeems for more code; a thunk for code.
+fn gen_call_branch_stub_hit(
ocb: &mut OutlinedCb,
-) {
- let maybe_block = find_block_version(target, ctx);
-
- // If the block already exists
- if let Some(blockref) = maybe_block {
- let mut block = blockref.borrow_mut();
-
- // Add an incoming branch into this block
- block.push_incoming(branchref.clone());
-
- // Fill out the target with this block
- branch.targets[target_idx.as_usize()] = Some(Box::new(BranchTarget::Block(blockref.clone())));
-
- return;
- }
-
+ branch_struct_address: usize,
+ target_idx: u32,
+) -> Option<CodePtr> {
let ocb = ocb.unwrap();
// Generate an outlined stub that will call branch_stub_hit()
let stub_addr = ocb.get_write_ptr();
- // Get a raw pointer to the branch. We clone and then decrement the strong count which overall
- // balances the strong count. We do this so that we're passing the result of [Rc::into_raw] to
- // [Rc::from_raw] as required.
- // We make sure the block housing the branch is still alive when branch_stub_hit() is running.
- let branch_ptr: *const RefCell<Branch> = BranchRef::into_raw(branchref.clone());
- unsafe { BranchRef::decrement_strong_count(branch_ptr) };
-
let mut asm = Assembler::new();
asm.comment("branch stub hit");
// Set up the arguments unique to this stub for:
- // branch_stub_hit(branch_ptr, target_idx, ec)
- asm.mov(C_ARG_OPNDS[0], Opnd::const_ptr(branch_ptr as *const u8));
+ //
+ // branch_stub_hit(branch_ptr, target_idx, ec)
+ //
+ // Bake pointer to Branch into output code.
+ // We make sure the block housing the branch is still alive when branch_stub_hit() is running.
+ asm.mov(C_ARG_OPNDS[0], branch_struct_address.into());
asm.mov(C_ARG_OPNDS[1], target_idx.into());
// Jump to trampoline to call branch_stub_hit()
@@ -2146,13 +2287,9 @@ fn set_branch_target(
if ocb.has_dropped_bytes() {
// No space
+ None
} else {
- // Fill the branch target with a stub
- branch.targets[target_idx.as_usize()] = Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
- address: Some(stub_addr),
- id: target,
- ctx: ctx.clone(),
- }))));
+ Some(stub_addr)
}
}
@@ -2188,28 +2325,26 @@ pub fn gen_branch_stub_hit_trampoline(ocb: &mut OutlinedCb) -> CodePtr {
impl Assembler
{
// Mark the start position of a patchable branch in the machine code
- fn mark_branch_start(&mut self, branchref: &BranchRef)
+ fn mark_branch_start(&mut self, branchref: &PendingBranchRef)
{
// We need to create our own branch rc object
// so that we can move the closure below
let branchref = branchref.clone();
self.pos_marker(move |code_ptr| {
- let mut branch = branchref.borrow_mut();
- branch.start_addr = Some(code_ptr);
+ branchref.start_addr.set(Some(code_ptr));
});
}
// Mark the end position of a patchable branch in the machine code
- fn mark_branch_end(&mut self, branchref: &BranchRef)
+ fn mark_branch_end(&mut self, branchref: &PendingBranchRef)
{
// We need to create our own branch rc object
// so that we can move the closure below
let branchref = branchref.clone();
self.pos_marker(move |code_ptr| {
- let mut branch = branchref.borrow_mut();
- branch.end_addr = Some(code_ptr);
+ branchref.end_addr.set(Some(code_ptr));
});
}
}
@@ -2224,66 +2359,63 @@ pub fn gen_branch(
ctx1: Option<&Context>,
gen_fn: BranchGenFn,
) {
- let branchref = make_branch_entry(jit, &jit.get_block(), gen_fn);
- let branch = &mut branchref.borrow_mut();
+ let branch = new_pending_branch(jit, gen_fn);
// Get the branch targets or stubs
- set_branch_target(0, target0, ctx0, &branchref, branch, ocb);
- if let Some(ctx) = ctx1 {
- set_branch_target(1, target1.unwrap(), ctx, &branchref, branch, ocb);
- if branch.targets[1].is_none() {
- return; // avoid unwrap() in gen_fn()
+ let target0_addr = branch.set_target(0, target0, ctx0, ocb);
+ let target1_addr = if let Some(ctx) = ctx1 {
+ let addr = branch.set_target(1, target1.unwrap(), ctx, ocb);
+ if addr.is_none() {
+ // target1 requested but we're out of memory.
+ // Avoid unwrap() in gen_fn()
+ return;
}
- }
+
+ addr
+ } else { None };
// Call the branch generation function
- asm.mark_branch_start(&branchref);
- if let Some(dst_addr) = branch.get_target_address(0) {
- gen_fn.call(asm, dst_addr, branch.get_target_address(1));
+ asm.mark_branch_start(&branch);
+ if let Some(dst_addr) = target0_addr {
+ branch.gen_fn.call(asm, dst_addr, target1_addr);
}
- asm.mark_branch_end(&branchref);
+ asm.mark_branch_end(&branch);
}
pub fn gen_direct_jump(jit: &mut JITState, ctx: &Context, target0: BlockId, asm: &mut Assembler) {
- let branchref = make_branch_entry(jit, &jit.get_block(), BranchGenFn::JumpToTarget0(BranchShape::Default));
- let mut branch = branchref.borrow_mut();
-
- let mut new_target = BranchTarget::Stub(Box::new(BranchStub {
- address: None,
- ctx: ctx.clone(),
- id: target0,
- }));
-
+ let branch = new_pending_branch(jit, BranchGenFn::JumpToTarget0(Cell::new(BranchShape::Default)));
let maybe_block = find_block_version(target0, ctx);
// If the block already exists
- if let Some(blockref) = maybe_block {
- let mut block = blockref.borrow_mut();
+ let new_target = if let Some(blockref) = maybe_block {
+ let block = unsafe { blockref.as_ref() };
let block_addr = block.start_addr;
- block.push_incoming(branchref.clone());
-
- new_target = BranchTarget::Block(blockref.clone());
-
- branch.gen_fn.set_shape(BranchShape::Default);
-
// Call the branch generation function
asm.comment("gen_direct_jmp: existing block");
- asm.mark_branch_start(&branchref);
+ asm.mark_branch_start(&branch);
branch.gen_fn.call(asm, block_addr, None);
- asm.mark_branch_end(&branchref);
- } else {
- // `None` in new_target.address signals gen_block_series() to compile the
- // target block right after this one (fallthrough).
- branch.gen_fn.set_shape(BranchShape::Next0);
+ asm.mark_branch_end(&branch);
+ BranchTarget::Block(blockref)
+ } else {
// The branch is effectively empty (a noop)
asm.comment("gen_direct_jmp: fallthrough");
- asm.mark_branch_start(&branchref);
- asm.mark_branch_end(&branchref);
- }
+ asm.mark_branch_start(&branch);
+ asm.mark_branch_end(&branch);
+ branch.gen_fn.set_shape(BranchShape::Next0);
+
+ // `None` in new_target.address signals gen_block_series() to
+ // compile the target block right after this one (fallthrough).
+ BranchTarget::Stub(Box::new(BranchStub {
+ address: None,
+ ctx: ctx.clone(),
+ iseq: Cell::new(target0.iseq),
+ iseq_idx: target0.idx,
+ }))
+ };
- branch.targets[0] = Some(Box::new(new_target));
+ branch.targets[0].set(Some(Box::new(new_target)));
}
/// Create a stub to force the code up to this point to be executed
@@ -2304,81 +2436,127 @@ pub fn defer_compilation(
}
next_ctx.chain_depth += 1;
- let block_rc = jit.get_block();
- let branch_rc = make_branch_entry(jit, &jit.get_block(), BranchGenFn::JumpToTarget0(BranchShape::Default));
- let mut branch = branch_rc.borrow_mut();
- let block = block_rc.borrow();
+ let branch = new_pending_branch(jit, BranchGenFn::JumpToTarget0(Cell::new(BranchShape::Default)));
let blockid = BlockId {
- iseq: block.blockid.iseq,
+ iseq: jit.get_iseq(),
idx: jit.get_insn_idx(),
};
- set_branch_target(0, blockid, &next_ctx, &branch_rc, &mut branch, ocb);
+
+ // Likely a stub due to the increased chain depth
+ let target0_address = branch.set_target(0, blockid, &next_ctx, ocb);
// Call the branch generation function
asm.comment("defer_compilation");
- asm.mark_branch_start(&branch_rc);
- if let Some(dst_addr) = branch.get_target_address(0) {
+ asm.mark_branch_start(&branch);
+ if let Some(dst_addr) = target0_address {
branch.gen_fn.call(asm, dst_addr, None);
}
- asm.mark_branch_end(&branch_rc);
+ asm.mark_branch_end(&branch);
// If the block we're deferring from is empty
- if jit.get_block().borrow().get_blockid().idx == jit.get_insn_idx() {
+ if jit.get_starting_insn_idx() == jit.get_insn_idx() {
incr_counter!(defer_empty_count);
}
incr_counter!(defer_count);
}
-fn remove_from_graph(blockref: &BlockRef) {
- let block = blockref.borrow();
+/// Remove a block from the live control flow graph.
+/// Block must be initialized and incoming/outgoing edges
+/// must also point to initialized blocks.
+unsafe fn remove_from_graph(blockref: BlockRef) {
+ let block = unsafe { blockref.as_ref() };
// Remove this block from the predecessor's targets
- for pred_branchref in &block.incoming {
+ for pred_branchref in block.incoming.0.take().iter() {
// Branch from the predecessor to us
- let mut pred_branch = pred_branchref.borrow_mut();
+ let pred_branch = unsafe { pred_branchref.as_ref() };
// If this is us, nullify the target block
- for target_idx in 0..=1 {
- if let Some(target) = pred_branch.targets[target_idx].as_ref() {
- if target.get_block().as_ref() == Some(blockref) {
- pred_branch.targets[target_idx] = None;
- }
+ for target_idx in 0..pred_branch.targets.len() {
+ // SAFETY: no mutation inside unsafe
+ let target_is_us = unsafe {
+ pred_branch.targets[target_idx]
+ .ref_unchecked()
+ .as_ref()
+ .and_then(|target| target.get_block())
+ .and_then(|target_block| (target_block == blockref).then(|| ()))
+ .is_some()
+ };
+
+ if target_is_us {
+ pred_branch.targets[target_idx].set(None);
}
}
}
// For each outgoing branch
for out_branchref in block.outgoing.iter() {
- let out_branch = out_branchref.borrow();
-
+ let out_branch = unsafe { out_branchref.as_ref() };
// For each successor block
- for out_target in out_branch.targets.iter().flatten() {
- if let Some(succ_blockref) = &out_target.get_block() {
+ for out_target in out_branch.targets.iter() {
+ // SAFETY: copying out an Option<BlockRef>. No mutation.
+ let succ_block: Option<BlockRef> = unsafe {
+ out_target.ref_unchecked().as_ref().and_then(|target| target.get_block())
+ };
+
+ if let Some(succ_block) = succ_block {
// Remove outgoing branch from the successor's incoming list
- let mut succ_block = succ_blockref.borrow_mut();
- succ_block
- .incoming
- .retain(|succ_incoming| !Rc::ptr_eq(succ_incoming, out_branchref));
+ // SAFETY: caller promises the block has valid outgoing edges.
+ let succ_block = unsafe { succ_block.as_ref() };
+ // Temporarily move out of succ_block.incoming.
+ let succ_incoming = succ_block.incoming.0.take();
+ let mut succ_incoming = succ_incoming.into_vec();
+ succ_incoming.retain(|branch| branch != out_branchref);
+ succ_block.incoming.0.set(succ_incoming.into_boxed_slice()); // allocs. Rely on oom=abort
}
}
}
}
-/// Remove most references to a block to deallocate it.
-/// Does not touch references from iseq payloads.
-pub fn free_block(blockref: &BlockRef) {
- block_assumptions_free(blockref);
+/// Tear down a block and deallocate it.
+/// Caller has to ensure that the code tracked by the block is not
+/// running, as running code may hit [branch_stub_hit] who exepcts
+/// [Branch] to be live.
+///
+/// We currently ensure this through the `jit_cont` system in cont.c
+/// and sometimes through the GC calling [rb_yjit_iseq_free]. The GC
+/// has proven that an ISeq is not running if it calls us to free it.
+///
+/// For delayed deallocation, since dead blocks don't keep
+/// blocks they refer alive, by the time we get here their outgoing
+/// edges may be dangling. Pass `graph_intact=false` such these cases.
+pub unsafe fn free_block(blockref: BlockRef, graph_intact: bool) {
+ // Careful with order here.
+ // First, remove all pointers to the referent block
+ unsafe {
+ block_assumptions_free(blockref);
+
+ if graph_intact {
+ remove_from_graph(blockref);
+ }
+ }
- remove_from_graph(blockref);
+ // SAFETY: we should now have a unique pointer to the block
+ unsafe { dealloc_block(blockref) }
+}
- // Branches have a Rc pointing at the block housing them.
- // Break the cycle.
- blockref.borrow_mut().incoming.clear();
- blockref.borrow_mut().outgoing = Box::new([]);
+/// Deallocate a block and its outgoing branches. Blocks own their outgoing branches.
+/// Caller must ensure that we have unique ownership for the referent block
+unsafe fn dealloc_block(blockref: BlockRef) {
+ unsafe {
+ for outgoing in blockref.as_ref().outgoing.iter() {
+ // this Box::from_raw matches the Box::into_raw from PendingBranch::into_branch
+ mem::drop(Box::from_raw(outgoing.as_ptr()));
+ }
+ }
- // No explicit deallocation here as blocks are ref-counted.
+ // Deallocate the referent Block
+ unsafe {
+ // this Box::from_raw matches the Box::into_raw from JITState::into_block
+ mem::drop(Box::from_raw(blockref.as_ptr()));
+ }
}
// Some runtime checks for integrity of a program location
@@ -2396,20 +2574,21 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
// TODO: want to assert that all other ractors are stopped here. Can't patch
// machine code that some other thread is running.
- let block = blockref.borrow();
+ let block = unsafe { (*blockref).as_ref() };
+ let id_being_invalidated = block.get_blockid();
let mut cb = CodegenGlobals::get_inline_cb();
let ocb = CodegenGlobals::get_outlined_cb();
- verify_blockid(block.blockid);
+ verify_blockid(id_being_invalidated);
#[cfg(feature = "disasm")]
{
// If dump_iseq_disasm is specified, print to console that blocks for matching ISEQ names were invalidated.
if let Some(substr) = get_option_ref!(dump_iseq_disasm).as_ref() {
- let blockid_idx = block.blockid.idx;
- let iseq_location = iseq_get_location(block.blockid.iseq, blockid_idx);
+ let iseq_range = &block.iseq_range;
+ let iseq_location = iseq_get_location(block.iseq.get(), iseq_range.start);
if iseq_location.contains(substr) {
- println!("Invalidating block from {}, ISEQ offsets [{}, {})", iseq_location, blockid_idx, block.end_idx);
+ println!("Invalidating block from {}, ISEQ offsets [{}, {})", iseq_location, iseq_range.start, iseq_range.end);
}
}
}
@@ -2431,9 +2610,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
.entry_exit
.expect("invalidation needs the entry_exit field");
{
- let block_end = block
- .end_addr
- .expect("invalidation needs constructed block");
+ let block_end = block.get_end_addr();
if block_start == block_entry_exit {
// Some blocks exit on entry. Patching a jump to the entry at the
@@ -2462,10 +2639,8 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
}
// For each incoming branch
- mem::drop(block); // end borrow: regenerate_branch might mut borrow this
- let block = blockref.borrow().clone();
- for branchref in &block.incoming {
- let mut branch = branchref.borrow_mut();
+ for branchref in block.incoming.0.take().iter() {
+ let branch = unsafe { branchref.as_ref() };
let target_idx = if branch.get_target_address(0) == Some(block_start) {
0
} else {
@@ -2473,30 +2648,35 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
};
// Assert that the incoming branch indeed points to the block being invalidated
- let incoming_target = branch.targets[target_idx].as_ref().unwrap();
- assert_eq!(Some(block_start), incoming_target.get_address());
- if let Some(incoming_block) = &incoming_target.get_block() {
- assert_eq!(blockref, incoming_block);
+ // SAFETY: no mutation.
+ unsafe {
+ let incoming_target = branch.targets[target_idx].ref_unchecked().as_ref().unwrap();
+ assert_eq!(Some(block_start), incoming_target.get_address());
+ if let Some(incoming_block) = &incoming_target.get_block() {
+ assert_eq!(blockref, incoming_block);
+ }
}
- // Create a stub for this branch target or rewire it to a valid block
- set_branch_target(target_idx as u32, block.blockid, &block.ctx, branchref, &mut branch, ocb);
+ // Create a stub for this branch target
+ let stub_addr = gen_call_branch_stub_hit(ocb, branchref.as_ptr() as usize, target_idx as u32);
- if branch.targets[target_idx].is_none() {
- // We were unable to generate a stub (e.g. OOM). Use the block's
- // exit instead of a stub for the block. It's important that we
- // still patch the branch in this situation so stubs are unique
- // to branches. Think about what could go wrong if we run out of
- // memory in the middle of this loop.
- branch.targets[target_idx] = Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
- address: block.entry_exit,
- id: block.blockid,
- ctx: block.ctx.clone(),
- }))));
- }
+ // In case we were unable to generate a stub (e.g. OOM). Use the block's
+ // exit instead of a stub for the block. It's important that we
+ // still patch the branch in this situation so stubs are unique
+ // to branches. Think about what could go wrong if we run out of
+ // memory in the middle of this loop.
+ let stub_addr = stub_addr.unwrap_or(block_entry_exit);
+
+ // Fill the branch target with a stub
+ branch.targets[target_idx].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
+ address: Some(stub_addr),
+ iseq: block.iseq.clone(),
+ iseq_idx: block.iseq_range.start,
+ ctx: block.ctx.clone(),
+ })))));
// Check if the invalidated block immediately follows
- let target_next = Some(block.start_addr) == branch.end_addr;
+ let target_next = block.start_addr == branch.end_addr.get();
if target_next {
// The new block will no longer be adjacent.
@@ -2507,7 +2687,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
// Rewrite the branch with the new jump target address
let old_branch_size = branch.code_size();
- regenerate_branch(cb, &mut branch);
+ regenerate_branch(cb, branch);
if target_next && branch.end_addr > block.end_addr {
panic!("yjit invalidate rewrote branch past end of invalidated block: {:?} (code_size: {})", branch, block.code_size());
@@ -2515,7 +2695,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
if !target_next && branch.code_size() > old_branch_size {
panic!(
"invalidated branch grew in size (start_addr: {:?}, old_size: {}, new_size: {})",
- branch.start_addr.unwrap().raw_ptr(), old_branch_size, branch.code_size()
+ branch.start_addr.raw_ptr(), old_branch_size, branch.code_size()
);
}
}
@@ -2528,17 +2708,21 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
// points will always have an instruction index of 0. We'll need to
// change this in the future when we support optional parameters because
// they enter the function with a non-zero PC
- if block.blockid.idx == 0 {
+ if block.iseq_range.start == 0 {
// TODO:
// We could reset the exec counter to zero in rb_iseq_reset_jit_func()
// so that we eventually compile a new entry point when useful
- unsafe { rb_iseq_reset_jit_func(block.blockid.iseq) };
+ unsafe { rb_iseq_reset_jit_func(block.iseq.get()) };
}
// FIXME:
// Call continuation addresses on the stack can also be atomically replaced by jumps going to the stub.
- delayed_deallocation(blockref);
+ // SAFETY: This block was in a version_map earlier
+ // in this function before we removed it, so it's well connected.
+ unsafe { remove_from_graph(*blockref) };
+
+ delayed_deallocation(*blockref);
ocb.unwrap().mark_all_executable();
cb.mark_all_executable();
@@ -2553,22 +2737,49 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
// invalidated branch pointers. Example:
// def foo(n)
// if n == 2
+// # 1.times{} to use a cfunc to avoid exiting from the
+// # frame which will use the retained return address
// return 1.times { Object.define_method(:foo) {} }
// end
//
// foo(n + 1)
// end
// p foo(1)
-pub fn delayed_deallocation(blockref: &BlockRef) {
+pub fn delayed_deallocation(blockref: BlockRef) {
block_assumptions_free(blockref);
- // We do this another time when we deem that it's safe
- // to deallocate in case there is another Ractor waiting to acquire the
- // VM lock inside branch_stub_hit().
- remove_from_graph(blockref);
+ let payload = get_iseq_payload(unsafe { blockref.as_ref() }.iseq.get()).unwrap();
+ payload.dead_blocks.push(blockref);
+}
+
+trait RefUnchecked {
+ type Contained;
+ unsafe fn ref_unchecked(&self) -> &Self::Contained;
+}
+
+impl<T> RefUnchecked for Cell<T> {
+ type Contained = T;
- let payload = get_iseq_payload(blockref.borrow().blockid.iseq).unwrap();
- payload.dead_blocks.push(blockref.clone());
+ /// Gives a reference to the contents of a [Cell].
+ /// Dangerous; please include a SAFETY note.
+ ///
+ /// An easy way to use this without triggering Undefined Behavior is to
+ /// 1. ensure there is transitively no Cell/UnsafeCell mutation in the `unsafe` block
+ /// 2. ensure the `unsafe` block does not return any references, so our
+ /// analysis is lexically confined. This is trivially true if the block
+ /// returns a `bool`, for example. Aggregates that store references have
+ /// explicit lifetime parameters that look like `<'a>`.
+ ///
+ /// There are other subtler situations that don't follow these rules yet
+ /// are still sound.
+ /// See `test_miri_ref_unchecked()` for examples. You can play with it
+ /// with `cargo +nightly miri test miri`.
+ unsafe fn ref_unchecked(&self) -> &Self::Contained {
+ // SAFETY: pointer is dereferenceable because it's from a &Cell.
+ // It's up to the caller to follow aliasing rules with the output
+ // reference.
+ unsafe { self.as_ptr().as_ref().unwrap() }
+ }
}
#[cfg(test)]
@@ -2603,4 +2814,97 @@ mod tests {
// TODO: write more tests for Context type diff
}
+
+ #[test]
+ fn test_miri_ref_unchecked() {
+ let blockid = BlockId {
+ iseq: ptr::null(),
+ idx: 0,
+ };
+ let cb = CodeBlock::new_dummy(1024);
+ let dumm_addr = cb.get_write_ptr();
+ let block = JITState::new(blockid, Context::default(), dumm_addr, ptr::null())
+ .into_block(0, dumm_addr, dumm_addr, vec![]);
+ let _dropper = BlockDropper(block);
+
+ // Outside of brief moments during construction,
+ // we're always working with &Branch (a shared reference to a Branch).
+ let branch: &Branch = &Branch {
+ gen_fn: BranchGenFn::JZToTarget0,
+ block,
+ start_addr: dumm_addr,
+ end_addr: Cell::new(dumm_addr),
+ targets: [Cell::new(None), Cell::new(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
+ iseq: Cell::new(ptr::null()),
+ iseq_idx: 0,
+ address: None,
+ ctx: Context::default(),
+ })))))]
+ };
+ // For easier soundness reasoning, make sure the reference returned does not out live the
+ // `unsafe` block! It's tempting to do, but it leads to non-local issues.
+ // Here is an example where it goes wrong:
+ if false {
+ for target in branch.targets.iter().as_ref() {
+ if let Some(btarget) = unsafe { target.ref_unchecked() } {
+ // btarget is derived from the usnafe block!
+ target.set(None); // This drops the contents of the cell...
+ assert!(btarget.get_address().is_none()); // but `btarget` is still live! UB.
+ }
+ }
+ }
+
+ // Do something like this instead. It's not pretty, but it's easier to vet for UB this way.
+ for target in branch.targets.iter().as_ref() {
+ // SAFETY: no mutation within unsafe
+ if unsafe { target.ref_unchecked().is_none() } {
+ continue;
+ }
+ // SAFETY: no mutation within unsafe
+ assert!(unsafe { target.ref_unchecked().as_ref().unwrap().get_address().is_none() });
+ target.set(None);
+ }
+
+ // A more subtle situation where we do Cell/UnsafeCell mutation over the
+ // lifetime of the reference released by ref_unchecked().
+ branch.targets[0].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub {
+ iseq: Cell::new(ptr::null()),
+ iseq_idx: 0,
+ address: None,
+ ctx: Context::default(),
+ })))));
+ // Invalid ISeq; we never dereference it.
+ let secret_iseq = NonNull::<rb_iseq_t>::dangling().as_ptr();
+ unsafe {
+ if let Some(branch_target) = branch.targets[0].ref_unchecked().as_ref() {
+ if let BranchTarget::Stub(stub) = branch_target.as_ref() {
+ // SAFETY:
+ // This is a Cell mutation, but it mutates the contents
+ // of a a Cell<IseqPtr>, which is a different type
+ // from the type of Cell found in `Branch::targets`, so
+ // there is no chance of mutating the Cell that we called
+ // ref_unchecked() on above.
+ Cell::set(&stub.iseq, secret_iseq);
+ }
+ }
+ };
+ // Check that we indeed changed the iseq of the stub
+ // Cell::take moves out of the cell.
+ assert_eq!(
+ secret_iseq as usize,
+ branch.targets[0].take().unwrap().get_blockid().iseq as usize
+ );
+
+ struct BlockDropper(BlockRef);
+ impl Drop for BlockDropper {
+ fn drop(&mut self) {
+ // SAFETY: we have ownership because the test doesn't stash
+ // the block away in any global structure.
+ // Note that the test being self-contained is also why we
+ // use dealloc_block() over free_block(), as free_block() touches
+ // the global invariants tables unavailable in tests.
+ unsafe { dealloc_block(self.0) };
+ }
+ }
+ }
}
diff --git a/yjit/src/disasm.rs b/yjit/src/disasm.rs
index 0b464b9333..f9a5744979 100644
--- a/yjit/src/disasm.rs
+++ b/yjit/src/disasm.rs
@@ -37,18 +37,23 @@ pub extern "C" fn rb_yjit_disasm_iseq(_ec: EcPtr, _ruby_self: VALUE, iseqw: VALU
// This will truncate disassembly of methods with 10k+ bytecodes.
// That's a good thing - this prints to console.
- let out_string = disasm_iseq_insn_range(iseq, 0, 9999);
+ let out_string = with_vm_lock(src_loc!(), || disasm_iseq_insn_range(iseq, 0, 9999));
return rust_str_to_ruby(&out_string);
}
}
+/// Only call while holding the VM lock.
#[cfg(feature = "disasm")]
pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> String {
let mut out = String::from("");
// Get a list of block versions generated for this iseq
- let mut block_list = get_or_create_iseq_block_list(iseq);
+ let block_list = get_or_create_iseq_block_list(iseq);
+ let mut block_list: Vec<&Block> = block_list.into_iter().map(|blockref| {
+ // SAFETY: We have the VM lock here and all the blocks on iseqs are valid.
+ unsafe { blockref.as_ref() }
+ }).collect();
// Get a list of codeblocks relevant to this iseq
let global_cb = crate::codegen::CodegenGlobals::get_inline_cb();
@@ -58,8 +63,8 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St
use std::cmp::Ordering;
// Get the start addresses for each block
- let addr_a = a.borrow().get_start_addr().raw_ptr();
- let addr_b = b.borrow().get_start_addr().raw_ptr();
+ let addr_a = a.get_start_addr().raw_ptr();
+ let addr_b = b.get_start_addr().raw_ptr();
if addr_a < addr_b {
Ordering::Less
@@ -73,20 +78,19 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St
// Compute total code size in bytes for all blocks in the function
let mut total_code_size = 0;
for blockref in &block_list {
- total_code_size += blockref.borrow().code_size();
+ total_code_size += blockref.code_size();
}
writeln!(out, "NUM BLOCK VERSIONS: {}", block_list.len()).unwrap();
writeln!(out, "TOTAL INLINE CODE SIZE: {} bytes", total_code_size).unwrap();
// For each block, sorted by increasing start address
- for block_idx in 0..block_list.len() {
- let block = block_list[block_idx].borrow();
+ for (block_idx, block) in block_list.iter().enumerate() {
let blockid = block.get_blockid();
if blockid.idx >= start_idx && blockid.idx < end_idx {
let end_idx = block.get_end_idx();
let start_addr = block.get_start_addr();
- let end_addr = block.get_end_addr().unwrap();
+ let end_addr = block.get_end_addr();
let code_size = block.code_size();
// Write some info about the current block
@@ -110,7 +114,7 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St
// If this is not the last block
if block_idx < block_list.len() - 1 {
// Compute the size of the gap between this block and the next
- let next_block = block_list[block_idx + 1].borrow();
+ let next_block = block_list[block_idx + 1];
let next_start_addr = next_block.get_start_addr();
let gap_size = next_start_addr.into_usize() - end_addr.into_usize();
@@ -318,7 +322,9 @@ fn insns_compiled(iseq: IseqPtr) -> Vec<(String, u16)> {
// For each block associated with this iseq
for blockref in &block_list {
- let block = blockref.borrow();
+ // SAFETY: Called as part of a Ruby method, which ensures the graph is
+ // well connected for the given iseq.
+ let block = unsafe { blockref.as_ref() };
let start_idx = block.get_blockid().idx;
let end_idx = block.get_end_idx();
assert!(u32::from(end_idx) <= unsafe { get_iseq_encoded_size(iseq) });
diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs
index dbeafe1969..0a969905dc 100644
--- a/yjit/src/invariants.rs
+++ b/yjit/src/invariants.rs
@@ -16,8 +16,8 @@ use std::mem;
// Invariants to track:
// assume_bop_not_redefined(jit, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS)
// assume_method_lookup_stable(comptime_recv_klass, cme, jit);
-// assume_single_ractor_mode(jit)
-// assume_stable_global_constant_state(jit);
+// assume_single_ractor_mode()
+// track_stable_constant_names_assumption()
/// Used to track all of the various block references that contain assumptions
/// about the state of the virtual machine.
@@ -78,9 +78,9 @@ impl Invariants {
}
}
-/// A public function that can be called from within the code generation
-/// functions to ensure that the block being generated is invalidated when the
-/// basic operator is redefined.
+/// Mark the pending block as assuming that certain basic operators (e.g. Integer#==)
+/// have not been redefined.
+#[must_use]
pub fn assume_bop_not_redefined(
jit: &mut JITState,
ocb: &mut OutlinedCb,
@@ -89,18 +89,7 @@ pub fn assume_bop_not_redefined(
) -> bool {
if unsafe { BASIC_OP_UNREDEFINED_P(bop, klass) } {
jit_ensure_block_entry_exit(jit, ocb);
-
- let invariants = Invariants::get_instance();
- invariants
- .basic_operator_blocks
- .entry((klass, bop))
- .or_default()
- .insert(jit.get_block());
- invariants
- .block_basic_operators
- .entry(jit.get_block())
- .or_default()
- .insert((klass, bop));
+ jit.bop_assumptions.push((klass, bop));
return true;
} else {
@@ -108,28 +97,33 @@ pub fn assume_bop_not_redefined(
}
}
-// Remember that a block assumes that
-// `rb_callable_method_entry(receiver_klass, cme->called_id) == cme` and that
-// `cme` is valid.
-// When either of these assumptions becomes invalid, rb_yjit_method_lookup_change() or
-// rb_yjit_cme_invalidate() invalidates the block.
-//
-// @raise NoMemoryError
-pub fn assume_method_lookup_stable(
- jit: &mut JITState,
- ocb: &mut OutlinedCb,
+/// Track that a block is only valid when a certain basic operator has not been redefined
+/// since the block's inception.
+pub fn track_bop_assumption(uninit_block: BlockRef, bop: (RedefinitionFlag, ruby_basic_operators)) {
+ let invariants = Invariants::get_instance();
+ invariants
+ .basic_operator_blocks
+ .entry(bop)
+ .or_default()
+ .insert(uninit_block);
+ invariants
+ .block_basic_operators
+ .entry(uninit_block)
+ .or_default()
+ .insert(bop);
+}
+
+/// Track that a block will assume that `cme` is valid (false == METHOD_ENTRY_INVALIDATED(cme)).
+/// [rb_yjit_cme_invalidate] invalidates the block when `cme` is invalidated.
+pub fn track_method_lookup_stability_assumption(
+ uninit_block: BlockRef,
callee_cme: *const rb_callable_method_entry_t,
) {
- jit_ensure_block_entry_exit(jit, ocb);
-
- let block = jit.get_block();
- jit.push_cme_dependency(callee_cme);
-
Invariants::get_instance()
.cme_validity
.entry(callee_cme)
.or_default()
- .insert(block);
+ .insert(uninit_block);
}
// Checks rb_method_basic_definition_p and registers the current block for invalidation if method
@@ -141,10 +135,10 @@ pub fn assume_method_basic_definition(
ocb: &mut OutlinedCb,
klass: VALUE,
mid: ID
- ) -> bool {
+) -> bool {
if unsafe { rb_method_basic_definition_p(klass, mid) } != 0 {
let cme = unsafe { rb_callable_method_entry(klass, mid) };
- assume_method_lookup_stable(jit, ocb, cme);
+ jit.assume_method_lookup_stable(ocb, cme);
true
} else {
false
@@ -158,22 +152,24 @@ pub fn assume_single_ractor_mode(jit: &mut JITState, ocb: &mut OutlinedCb) -> bo
false
} else {
jit_ensure_block_entry_exit(jit, ocb);
- Invariants::get_instance()
- .single_ractor
- .insert(jit.get_block());
+ jit.block_assumes_single_ractor = true;
+
true
}
}
-/// Walk through the ISEQ to go from the current opt_getinlinecache to the
-/// subsequent opt_setinlinecache and find all of the name components that are
-/// associated with this constant (which correspond to the getconstant
-/// arguments).
-pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, idlist: *const ID) {
- /// Tracks that a block is assuming that the name component of a constant
- /// has not changed since the last call to this function.
+/// Track that the block will assume single ractor mode.
+pub fn track_single_ractor_assumption(uninit_block: BlockRef) {
+ Invariants::get_instance()
+ .single_ractor
+ .insert(uninit_block);
+}
+
+/// Track that a block will assume that the name components of a constant path expression
+/// has not changed since the block's full initialization.
+pub fn track_stable_constant_names_assumption(uninit_block: BlockRef, idlist: *const ID) {
fn assume_stable_constant_name(
- jit: &mut JITState,
+ uninit_block: BlockRef,
id: ID,
) {
if id == idNULL as u64 {
@@ -186,10 +182,10 @@ pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, id
.constant_state_blocks
.entry(id)
.or_default()
- .insert(jit.get_block());
+ .insert(uninit_block);
invariants
.block_constant_states
- .entry(jit.get_block())
+ .entry(uninit_block)
.or_default()
.insert(id);
}
@@ -198,12 +194,9 @@ pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, id
for i in 0.. {
match unsafe { *idlist.offset(i) } {
0 => break, // End of NULL terminated list
- id => assume_stable_constant_name(jit, id),
+ id => assume_stable_constant_name(uninit_block, id),
}
}
-
- jit_ensure_block_entry_exit(jit, ocb);
-
}
/// Called when a basic operator is redefined. Note that all the blocks assuming
@@ -344,19 +337,22 @@ pub extern "C" fn rb_yjit_root_mark() {
/// Remove all invariant assumptions made by the block by removing the block as
/// as a key in all of the relevant tables.
-pub fn block_assumptions_free(blockref: &BlockRef) {
+/// For safety, the block has to be initialized and the vm lock must be held.
+/// However, outgoing/incoming references to the block does _not_ need to be valid.
+pub fn block_assumptions_free(blockref: BlockRef) {
let invariants = Invariants::get_instance();
{
- let block = blockref.borrow();
+ // SAFETY: caller ensures that this reference is valid
+ let block = unsafe { blockref.as_ref() };
// For each method lookup dependency
for dep in block.iter_cme_deps() {
// Remove tracking for cme validity
- if let Some(blockset) = invariants.cme_validity.get_mut(dep) {
- blockset.remove(blockref);
+ if let Some(blockset) = invariants.cme_validity.get_mut(&dep) {
+ blockset.remove(&blockref);
if blockset.is_empty() {
- invariants.cme_validity.remove(dep);
+ invariants.cme_validity.remove(&dep);
}
}
}
@@ -506,17 +502,18 @@ pub extern "C" fn rb_yjit_tracing_invalidate_all() {
if on_stack_iseqs.contains(&iseq) {
// This ISEQ is running, so we can't free blocks immediately
for block in blocks {
- delayed_deallocation(&block);
+ delayed_deallocation(block);
}
payload.dead_blocks.shrink_to_fit();
} else {
// Safe to free dead blocks since the ISEQ isn't running
+ // Since we're freeing _all_ blocks, we don't need to keep the graph well formed
for block in blocks {
- free_block(&block);
+ unsafe { free_block(block, false) };
}
mem::take(&mut payload.dead_blocks)
- .iter()
- .for_each(free_block);
+ .into_iter()
+ .for_each(|block| unsafe { free_block(block, false) });
}
}
diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs
index b213af8503..785dc9b0f9 100644
--- a/yjit/src/stats.rs
+++ b/yjit/src/stats.rs
@@ -534,11 +534,11 @@ fn get_live_context_count() -> usize {
for_each_iseq_payload(|iseq_payload| {
for blocks in iseq_payload.version_map.iter() {
for block in blocks.iter() {
- count += block.borrow().get_ctx_count();
+ count += unsafe { block.as_ref() }.get_ctx_count();
}
}
for block in iseq_payload.dead_blocks.iter() {
- count += block.borrow().get_ctx_count();
+ count += unsafe { block.as_ref() }.get_ctx_count();
}
});
count