summaryrefslogtreecommitdiff
path: root/zjit
diff options
context:
space:
mode:
authorTakashi Kokubun <takashikkbn@gmail.com>2026-03-24 08:45:00 -0700
committerTakashi Kokubun <takashikkbn@gmail.com>2026-03-25 13:34:11 -0700
commitcfaef1e5ff212ea190d786a182aaad2e455682a0 (patch)
treeec57ca27e2b37e010dabac05c0b0d5cff60d5aaf /zjit
parent111215de2758666894ba0a2095d86b731ffd5af3 (diff)
ZJIT: Recompile ISEQs with no-profile sends via exit profiling
When a send instruction has no profile data (e.g., the code path was not reached during interpreter profiling), the first JIT compilation converts it to a SideExit that profiles operands on exit. After --zjit-num-profiles exits (default 5), the exit triggers ISEQ invalidation and recompilation with the newly gathered profile data, allowing the send to be optimized (e.g., as SendDirect) in the second version. This approach avoids adding Profile instructions to the HIR (which could interfere with optimization passes) and keeps profile_time low by only profiling on side exits rather than during normal execution.
Diffstat (limited to 'zjit')
-rw-r--r--zjit/src/backend/arm64/mod.rs2
-rw-r--r--zjit/src/backend/lir.rs27
-rw-r--r--zjit/src/backend/x86_64/mod.rs2
-rw-r--r--zjit/src/codegen.rs73
-rw-r--r--zjit/src/hir.rs137
-rw-r--r--zjit/src/hir/opt_tests.rs100
-rw-r--r--zjit/src/profile.rs25
-rw-r--r--zjit/src/stats.rs2
8 files changed, 299 insertions, 69 deletions
diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs
index 90bf28e1c0..9083824c4b 100644
--- a/zjit/src/backend/arm64/mod.rs
+++ b/zjit/src/backend/arm64/mod.rs
@@ -1754,7 +1754,7 @@ mod tests {
let val64 = asm.add(CFP, Opnd::UImm(64));
asm.store(Opnd::mem(64, SP, 0x10), val64);
- let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } };
+ let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } };
asm.push_insn(Insn::Joz(val64, side_exit));
asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32));
asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8));
diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs
index 00a80b9cf4..207538ec57 100644
--- a/zjit/src/backend/lir.rs
+++ b/zjit/src/backend/lir.rs
@@ -551,6 +551,17 @@ pub struct SideExit {
pub pc: Opnd,
pub stack: Vec<Opnd>,
pub locals: Vec<Opnd>,
+ /// If set, the side exit will call the recompile function with these arguments
+ /// to profile the send and invalidate the ISEQ for recompilation.
+ pub recompile: Option<SideExitRecompile>,
+}
+
+/// Arguments for the no-profile-send recompile callback.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct SideExitRecompile {
+ pub iseq: Opnd,
+ pub insn_idx: u32,
+ pub argc: i32,
}
/// Branch target (something that we can jump to)
@@ -2607,7 +2618,7 @@ impl Assembler
pub fn compile_exits(&mut self) -> Vec<Insn> {
/// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit.
fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) {
- let SideExit { pc, stack, locals } = exit;
+ let SideExit { pc, stack, locals, .. } = exit;
// Side exit blocks are not part of the CFG at the moment,
// so we need to manually ensure that patchpoints get padded
@@ -2646,6 +2657,20 @@ impl Assembler
/// that it can be safely deduplicated by using SideExit as a dedup key.
fn compile_exit(asm: &mut Assembler, exit: &SideExit) {
compile_exit_save_state(asm, exit);
+ // If this side exit should trigger recompilation, call the recompile
+ // function after saving VM state. The ccall must happen after
+ // compile_exit_save_state because it clobbers caller-saved registers
+ // that may hold stack/local operands we need to save.
+ if let Some(recompile) = &exit.recompile {
+ use crate::codegen::no_profile_send_recompile;
+ asm_comment!(asm, "profile and maybe recompile for no-profile send");
+ asm_ccall!(asm, no_profile_send_recompile,
+ EC,
+ recompile.iseq,
+ Opnd::UImm(recompile.insn_idx as u64),
+ Opnd::Imm(recompile.argc as i64)
+ );
+ }
compile_exit_return(asm);
}
diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs
index 570546cbd2..3cf744990d 100644
--- a/zjit/src/backend/x86_64/mod.rs
+++ b/zjit/src/backend/x86_64/mod.rs
@@ -1388,7 +1388,7 @@ mod tests {
let val64 = asm.add(CFP, Opnd::UImm(64));
asm.store(Opnd::mem(64, SP, 0x10), val64);
- let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } };
+ let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } };
asm.push_insn(Insn::Joz(val64, side_exit));
asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32));
asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8));
diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs
index af6e881bd0..bbdb1c986f 100644
--- a/zjit/src/codegen.rs
+++ b/zjit/src/codegen.rs
@@ -19,7 +19,7 @@ use crate::state::ZJITState;
use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type};
use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}};
use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr};
-use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_STACK_PTR, Opnd, SP, SideExit, Target, asm_ccall, asm_comment};
+use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_STACK_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment};
use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType};
use crate::hir::{Const, FrameState, Function, Insn, InsnId, SendFallbackReason};
use crate::hir_type::{types, Type};
@@ -474,7 +474,7 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func
Insn::InvokeBuiltin { .. } => SideExitReason::UnhandledHIRInvokeBuiltin,
_ => SideExitReason::UnhandledHIRUnknown(insn_id),
};
- gen_side_exit(&mut jit, &mut asm, &reason, &function.frame_state(last_snapshot));
+ gen_side_exit(&mut jit, &mut asm, &reason, &None, &function.frame_state(last_snapshot));
// Don't bother generating code after a side-exit. We won't run it.
// TODO(max): Generate ud2 or equivalent.
break;
@@ -666,7 +666,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))),
Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))),
Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index),
- Insn::SideExit { state, reason } => no_output!(gen_side_exit(jit, asm, reason, &function.frame_state(*state))),
+ Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, reason, recompile, &function.frame_state(*state))),
Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type),
Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)),
Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state)),
@@ -1186,8 +1186,14 @@ fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, sta
}
/// Side-exit into the interpreter
-fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) {
- asm.jmp(side_exit(jit, state, *reason));
+fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, recompile: &Option<i32>, state: &FrameState) {
+ let mut exit = build_side_exit(jit, state);
+ exit.recompile = recompile.map(|argc| SideExitRecompile {
+ iseq: Opnd::Value(VALUE::from(jit.iseq)),
+ insn_idx: state.insn_idx() as u32,
+ argc,
+ });
+ asm.jmp(Target::SideExit { exit, reason: *reason });
}
/// Emit a special object lookup
@@ -2824,6 +2830,7 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit {
pc: Opnd::const_ptr(state.pc),
stack,
locals,
+ recompile: None,
}
}
@@ -2844,23 +2851,71 @@ fn max_num_params(function: &Function) -> usize {
#[cfg(target_arch = "x86_64")]
macro_rules! c_callable {
($(#[$outer:meta])*
- fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => {
+ $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => {
$(#[$outer])*
- extern "sysv64" fn $f $args $(-> $ret)? $body
+ $vis extern "sysv64" fn $f $args $(-> $ret)? $body
};
}
#[cfg(target_arch = "aarch64")]
macro_rules! c_callable {
($(#[$outer:meta])*
- fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => {
+ $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => {
$(#[$outer])*
- extern "C" fn $f $args $(-> $ret)? $body
+ $vis extern "C" fn $f $args $(-> $ret)? $body
};
}
#[cfg(test)]
pub(crate) use c_callable;
c_callable! {
+ /// Called from JIT side-exit code when a send instruction had no profile data. This function
+ /// profiles the receiver and arguments on the stack, then (once enough profiles are gathered)
+ /// invalidates the current ISEQ version so that the ISEQ will be recompiled with the new
+ /// profile data on the next call.
+ pub(crate) fn no_profile_send_recompile(ec: EcPtr, iseq_raw: VALUE, insn_idx: u32, argc: i32) {
+ with_vm_lock(src_loc!(), || {
+ let iseq: IseqPtr = iseq_raw.as_iseq();
+ let payload = get_or_create_iseq_payload(iseq);
+
+ // Already gathered enough profiles; nothing to do
+ if payload.profile.done_profiling_at(insn_idx as usize) {
+ return;
+ }
+
+ with_time_stat(Counter::profile_time_ns, || {
+ let cfp = unsafe { get_ec_cfp(ec) };
+ let sp = unsafe { get_cfp_sp(cfp) };
+
+ // Profile the receiver and arguments for this send instruction
+ let should_recompile = payload.profile.profile_send_at(iseq, insn_idx as usize, sp, argc as usize);
+
+ // Once we have enough profiles, invalidate and recompile the ISEQ
+ if should_recompile {
+ let num_versions = payload.versions.len();
+ if let Some(version) = payload.versions.last_mut() {
+ if unsafe { version.as_ref() }.status != IseqStatus::Invalidated
+ && num_versions < MAX_ISEQ_VERSIONS
+ {
+ unsafe { version.as_mut() }.status = IseqStatus::Invalidated;
+ unsafe { rb_iseq_reset_jit_func(iseq) };
+
+ // Recompile JIT-to-JIT calls into the invalidated ISEQ
+ let cb = ZJITState::get_code_block();
+ for incoming in unsafe { version.as_ref() }.incoming.iter() {
+ if let Err(err) = gen_iseq_call(cb, incoming) {
+ debug!("{err:?}: gen_iseq_call failed on no-profile recompile: {}", iseq_get_location(incoming.iseq.get(), 0));
+ }
+ }
+ cb.mark_all_executable();
+ }
+ }
+ }
+ });
+ });
+ }
+}
+
+c_callable! {
/// Generated code calls this function with the SysV calling convention. See [gen_function_stub].
/// This function is expected to be called repeatedly when ZJIT fails to compile the stub.
/// We should be able to compile most (if not all) function stubs by side-exiting at unsupported
diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs
index 0891a59fa2..e7c6b06980 100644
--- a/zjit/src/hir.rs
+++ b/zjit/src/hir.rs
@@ -537,6 +537,7 @@ pub enum SideExitReason {
SplatKwNotProfiled,
DirectiveInduced,
SendWhileTracing,
+ NoProfileSend,
}
#[derive(Debug, Clone, Copy)]
@@ -1070,7 +1071,9 @@ pub enum Insn {
PatchPoint { invariant: Invariant, state: InsnId },
/// Side-exit into the interpreter.
- SideExit { state: InsnId, reason: SideExitReason },
+ /// If `recompile` is set, the side exit will profile the send and invalidate the ISEQ
+ /// so that it gets recompiled with the new profile data.
+ SideExit { state: InsnId, reason: SideExitReason, recompile: Option<i32> },
/// Increment a counter in ZJIT stats
IncrCounter(Counter),
@@ -2061,7 +2064,13 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> {
Insn::ObjToString { val, .. } => { write!(f, "ObjToString {val}") },
Insn::StringIntern { val, .. } => { write!(f, "StringIntern {val}") },
Insn::AnyToString { val, str, .. } => { write!(f, "AnyToString {val}, str: {str}") },
- Insn::SideExit { reason, .. } => write!(f, "SideExit {reason}"),
+ Insn::SideExit { reason, recompile, .. } => {
+ if recompile.is_some() {
+ write!(f, "SideExit {reason} recompile")
+ } else {
+ write!(f, "SideExit {reason}")
+ }
+ }
Insn::PutSpecialObject { value_type } => write!(f, "PutSpecialObject {value_type}"),
Insn::Throw { throw_state, val, .. } => {
write!(f, "Throw ")?;
@@ -2846,16 +2855,16 @@ impl Function {
/// Update DynamicSendReason for the instruction at insn_id
fn set_dynamic_send_reason(&mut self, insn_id: InsnId, dynamic_send_reason: SendFallbackReason) {
use Insn::*;
- if get_option!(stats) || get_option!(dump_hir_opt).is_some() || cfg!(test) {
- match self.insns.get_mut(insn_id.0).unwrap() {
- Send { reason, .. }
- | SendForward { reason, .. }
- | InvokeSuper { reason, .. }
- | InvokeSuperForward { reason, .. }
- | InvokeBlock { reason, .. }
- => *reason = dynamic_send_reason,
- _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id))
- }
+ // Always set the reason: convert_no_profile_sends depends on it to identify
+ // sends that should be converted to side exits for exit-based recompilation.
+ match self.insns.get_mut(insn_id.0).unwrap() {
+ Send { reason, .. }
+ | SendForward { reason, .. }
+ | InvokeSuper { reason, .. }
+ | InvokeSuperForward { reason, .. }
+ | InvokeBlock { reason, .. }
+ => *reason = dynamic_send_reason,
+ _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id))
}
}
@@ -3565,10 +3574,8 @@ impl Function {
continue;
}
ReceiverTypeResolution::NoProfile => {
- if get_option!(stats) {
- let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles };
- self.set_dynamic_send_reason(insn_id, reason);
- }
+ let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles };
+ self.set_dynamic_send_reason(insn_id, reason);
self.push_insn_id(block, insn_id);
continue;
}
@@ -4993,6 +5000,31 @@ impl Function {
self.infer_types();
}
+ /// Convert `Send` instructions with no profile data into `SideExit` with recompile info.
+ /// This runs after strength reduction passes (type_specialize, inline, optimize_c_calls) so
+ /// that sends that can be optimized without profiling (e.g. known CFUNCs) are already handled.
+ /// The remaining no-profile sends are turned into side exits that trigger recompilation with
+ /// fresh profile data.
+ fn convert_no_profile_sends(&mut self) {
+ for block in self.rpo() {
+ let old_insns = std::mem::take(&mut self.blocks[block.0].insns);
+ assert!(self.blocks[block.0].insns.is_empty());
+ for insn_id in old_insns {
+ match self.find(insn_id) {
+ Insn::Send { cd, state, reason: SendFallbackReason::SendWithoutBlockNoProfiles, .. } => {
+ let argc = unsafe { vm_ci_argc((*cd).ci) } as i32;
+ self.push_insn(block, Insn::SideExit { state, reason: SideExitReason::NoProfileSend, recompile: Some(argc) });
+ // SideExit is a terminator; don't add remaining instructions
+ break;
+ }
+ _ => {
+ self.push_insn_id(block, insn_id);
+ }
+ }
+ }
+ }
+ }
+
fn optimize_load_store(&mut self) {
for block in self.rpo() {
let mut compile_time_heap: HashMap<(InsnId, i32), InsnId> = HashMap::new();
@@ -5130,7 +5162,7 @@ impl Function {
self.make_equal_to(insn_id, left);
continue
},
- (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason }),
+ (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason, recompile: None }),
_ => insn_id,
}
},
@@ -5700,6 +5732,7 @@ impl Function {
(inline) => { Counter::compile_hir_strength_reduce_time_ns };
(optimize_getivar) => { Counter::compile_hir_strength_reduce_time_ns };
(optimize_c_calls) => { Counter::compile_hir_strength_reduce_time_ns };
+ (convert_no_profile_sends) => { Counter::compile_hir_strength_reduce_time_ns };
// End strength reduction bucket
(optimize_load_store) => { Counter::compile_hir_optimize_load_store_time_ns };
(fold_constants) => { Counter::compile_hir_fold_constants_time_ns };
@@ -5732,6 +5765,7 @@ impl Function {
run_pass!(inline);
run_pass!(optimize_getivar);
run_pass!(optimize_c_calls);
+ run_pass!(convert_no_profile_sends);
run_pass!(optimize_load_store);
run_pass!(fold_constants);
run_pass!(clean_cfg);
@@ -6385,6 +6419,11 @@ pub struct FrameState {
}
impl FrameState {
+ /// Get the YARV instruction index for the current instruction
+ pub fn insn_idx(&self) -> usize {
+ self.insn_idx
+ }
+
/// Return itself without locals. Useful for side-exiting without spilling locals.
fn without_locals(&self) -> Self {
let mut state = self.clone();
@@ -6997,13 +7036,13 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
}
_ => {
// Unknown opcode; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method), recompile: None });
break; // End the block
}
};
if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } {
// If the basic operation is already redefined, we cannot optimize it.
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None });
break; // End the block
}
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id });
@@ -7025,12 +7064,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let bop = match method_id {
x if x == ID!(include_p).0 => BOP_INCLUDE_P,
_ => {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id), recompile: None });
break;
},
};
if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None });
break;
}
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id });
@@ -7074,11 +7113,11 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
.and_then(|types| types.first())
.map(|dist| TypeDistributionSummary::new(dist));
let Some(summary) = summary else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotProfiled });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotProfiled, recompile: None });
break; // End the block
};
if !summary.is_monomorphic() {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwPolymorphic });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwPolymorphic, recompile: None });
break; // End the block
}
let ty = Type::from_profiled_type(summary.bucket(0));
@@ -7087,7 +7126,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
} else if ty.is_subtype(types::HashExact) {
fun.push_insn(block, Insn::GuardType { val: hash, guard_type: types::HashExact, state: exit_id })
} else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotNilOrHash });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotNilOrHash, recompile: None });
break; // End the block
};
state.stack_push(obj);
@@ -7145,7 +7184,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
// This can only happen in iseqs taking more than 32 keywords.
// In this case, we side exit to the interpreter.
if unsafe {(*rb_get_iseq_body_param_keyword(iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap()} {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters, recompile: None });
break;
}
let ep_offset = get_arg(pc, 0).as_u32();
@@ -7563,14 +7602,14 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let flags = unsafe { rb_vm_ci_flag(call_info) };
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle the call type; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let args = state.stack_pop_n(argc as usize)?;
@@ -7586,7 +7625,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) });
state.stack_push(recv);
} else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None });
break; // End the block
}
}
@@ -7598,7 +7637,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) });
state.stack_push(recv);
} else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None });
break; // End the block
}
}
@@ -7610,7 +7649,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) });
state.stack_push(recv);
} else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None });
break; // End the block
}
}
@@ -7622,7 +7661,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) });
state.stack_push(recv);
} else {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None });
break; // End the block
}
}
@@ -7668,7 +7707,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let flags = unsafe { rb_vm_ci_flag(call_info) };
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle tailcall; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7685,7 +7724,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
if mid == ID!(induce_side_exit_bang)
&& state::zjit_module_method_match_serial(ID!(induce_side_exit_bang), &state::INDUCE_SIDE_EXIT_SERIAL)
{
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::DirectiveInduced });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::DirectiveInduced, recompile: None });
break; // End the block
}
if mid == ID!(induce_compile_failure_bang)
@@ -7704,7 +7743,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
@@ -7791,12 +7830,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let flags = unsafe { rb_vm_ci_flag(call_info) };
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle tailcall; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7830,12 +7869,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let forwarding = (flags & VM_CALL_FORWARDING) != 0;
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle the call type; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7864,12 +7903,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let flags = unsafe { rb_vm_ci_flag(call_info) };
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle tailcall; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7903,12 +7942,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let forwarding = (flags & VM_CALL_FORWARDING) != 0;
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle tailcall; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7938,12 +7977,12 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
let flags = unsafe { rb_vm_ci_flag(call_info) };
if let Err(call_type) = unhandled_call_type(flags) {
// Can't handle tailcall; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None });
break; // End the block
}
// Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics
if unsafe { rb_zjit_iseq_tracing_currently_enabled() } {
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None });
break;
}
let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -7970,7 +8009,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
// TODO: We only really need this if self_val is a class/module
if !fun.assume_single_ractor_mode(block, exit_id) {
// gen_getivar assumes single Ractor; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None });
break; // End the block
}
if let Some(summary) = fun.polymorphic_summary(&profiles, self_param, exit_state.insn_idx) {
@@ -8025,7 +8064,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
// TODO: We only really need this if self_val is a class/module
if !fun.assume_single_ractor_mode(block, exit_id) {
// gen_setivar assumes single Ractor; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None });
break; // End the block
}
let val = state.stack_pop()?;
@@ -8142,7 +8181,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
if svar == 0 {
// TODO: Handle non-backref
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key), recompile: None });
// End the block
break;
} else if svar & 0x01 != 0 {
@@ -8165,7 +8204,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
// (reverse?)
//
// Unhandled opcode; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None });
break; // End the block
}
let val = state.stack_pop()?;
@@ -8183,7 +8222,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
}
_ => {
// Unhandled opcode; side-exit into the interpreter
- fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) });
+ fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None });
break; // End the block
}
}
diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs
index af438c361b..610748c818 100644
--- a/zjit/src/hir/opt_tests.rs
+++ b/zjit/src/hir/opt_tests.rs
@@ -4740,7 +4740,7 @@ mod hir_opt_tests {
v18:CInt64 = LoadField v15, :_env_data_index_specval@0x1002
v19:CInt64 = GuardAnyBitSet v18, CUInt64(1)
v20:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008))
- v22:BasicObject = Send v9, 0x1001, :tap, v20 # SendFallbackReason: Uncategorized(send)
+ v22:BasicObject = Send v9, 0x1001, :tap, v20 # SendFallbackReason: Send: no profile data available
CheckInterrupts
Return v22
");
@@ -6254,9 +6254,7 @@ mod hir_opt_tests {
bb3(v9:BasicObject, v10:BasicObject):
v17:Fixnum[1] = Const Value(1)
v19:Fixnum[10] = Const Value(10)
- v23:BasicObject = Send v10, :[]=, v17, v19 # SendFallbackReason: Uncategorized(opt_aset)
- CheckInterrupts
- Return v19
+ SideExit NoProfileSend recompile
");
}
@@ -10506,9 +10504,7 @@ mod hir_opt_tests {
v9:BasicObject = LoadArg :y@2
Jump bb3(v7, v8, v9)
bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject):
- v18:BasicObject = Send v12, :^ # SendFallbackReason: Uncategorized(opt_send_without_block)
- CheckInterrupts
- Return v18
+ SideExit NoProfileSend recompile
");
}
@@ -11365,7 +11361,7 @@ mod hir_opt_tests {
Jump bb3(v4)
bb3(v6:BasicObject):
v11:StaticSymbol[:the_block] = Const Value(VALUE(0x1000))
- v13:BasicObject = Send v6, 0x1008, :callee, v11 # SendFallbackReason: Uncategorized(send)
+ v13:BasicObject = Send v6, 0x1008, :callee, v11 # SendFallbackReason: Send: no profile data available
CheckInterrupts
Return v13
");
@@ -14860,4 +14856,92 @@ mod hir_opt_tests {
Return v13
");
}
+
+ #[test]
+ fn test_recompile_no_profile_send() {
+ // Define a callee method and a test method that calls it
+ eval("
+ def greet_recompile(x) = x.to_s
+ def test_no_profile_recompile(flag)
+ if flag
+ greet_recompile(42)
+ else
+ 'hello'
+ end
+ end
+ ");
+
+ // With call_threshold=2, num_profiles=1:
+ // 1st call profiles (flag=false, so greet is never reached)
+ // 2nd call compiles (greet has no profile data -> SideExit recompile)
+ eval("test_no_profile_recompile(false); test_no_profile_recompile(false)");
+
+ // The first compilation should have SideExit NoProfileSend recompile
+ // for the greet_recompile(42) callsite since it was never profiled.
+ assert_snapshot!(hir_string("test_no_profile_recompile"), @r"
+ fn test_no_profile_recompile@<compiled>:4:
+ bb1():
+ EntryPoint interpreter
+ v1:BasicObject = LoadSelf
+ v2:CPtr = LoadSP
+ v3:BasicObject = LoadField v2, :flag@0x1000
+ Jump bb3(v1, v3)
+ bb2():
+ EntryPoint JIT(0)
+ v6:BasicObject = LoadArg :self@0
+ v7:BasicObject = LoadArg :flag@1
+ Jump bb3(v6, v7)
+ bb3(v9:BasicObject, v10:BasicObject):
+ CheckInterrupts
+ v16:CBool = Test v10
+ v17:Falsy = RefineType v10, Falsy
+ IfFalse v16, bb4(v9, v17)
+ v19:Truthy = RefineType v10, Truthy
+ v23:Fixnum[42] = Const Value(42)
+ SideExit NoProfileSend recompile
+ bb4(v30:BasicObject, v31:Falsy):
+ v35:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008))
+ v36:StringExact = StringCopy v35
+ CheckInterrupts
+ Return v36
+ ");
+
+ // Now call with flag=true. This hits the SideExit, which profiles
+ // the send and invalidates the ISEQ for recompilation.
+ eval("test_no_profile_recompile(true)");
+
+ // After profiling via the side exit, rebuilding HIR should now
+ // have a SendDirect for greet_recompile instead of SideExit.
+ assert_snapshot!(hir_string("test_no_profile_recompile"), @r"
+ fn test_no_profile_recompile@<compiled>:4:
+ bb1():
+ EntryPoint interpreter
+ v1:BasicObject = LoadSelf
+ v2:CPtr = LoadSP
+ v3:BasicObject = LoadField v2, :flag@0x1000
+ Jump bb3(v1, v3)
+ bb2():
+ EntryPoint JIT(0)
+ v6:BasicObject = LoadArg :self@0
+ v7:BasicObject = LoadArg :flag@1
+ Jump bb3(v6, v7)
+ bb3(v9:BasicObject, v10:BasicObject):
+ CheckInterrupts
+ v16:CBool = Test v10
+ v17:Falsy = RefineType v10, Falsy
+ IfFalse v16, bb4(v9, v17)
+ v19:Truthy = RefineType v10, Truthy
+ v23:Fixnum[42] = Const Value(42)
+ PatchPoint MethodRedefined(Object@0x1008, greet_recompile@0x1010, cme:0x1018)
+ v43:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)]
+ v44:BasicObject = SendDirect v43, 0x1040, :greet_recompile (0x1050), v23
+ CheckInterrupts
+ Return v44
+ bb4(v30:BasicObject, v31:Falsy):
+ v35:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058))
+ v36:StringExact = StringCopy v35
+ CheckInterrupts
+ Return v36
+ ");
+ }
}
diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs
index a4981156a6..fd2ccdeb5f 100644
--- a/zjit/src/profile.rs
+++ b/zjit/src/profile.rs
@@ -427,6 +427,31 @@ impl IseqProfile {
.ok().map(|i| &self.entries[i])
}
+ /// Profile send operands from the stack at runtime.
+ /// `sp` is the current stack pointer (after the args and receiver).
+ /// `argc` is the number of arguments (not counting receiver).
+ /// Returns true if enough profiles have been gathered and the ISEQ should be recompiled.
+ /// Check if enough profiles have been gathered for this instruction.
+ pub fn done_profiling_at(&self, insn_idx: usize) -> bool {
+ self.entry(insn_idx).map_or(false, |e| e.num_profiles >= get_option!(num_profiles))
+ }
+
+ pub fn profile_send_at(&mut self, iseq: IseqPtr, insn_idx: usize, sp: *const VALUE, argc: usize) -> bool {
+ let n = argc + 1; // args + receiver
+ let entry = self.entry_mut(insn_idx);
+ if entry.opnd_types.is_empty() {
+ entry.opnd_types.resize(n, TypeDistribution::new());
+ }
+ for i in 0..n {
+ let obj = unsafe { *sp.offset(-1 - (n - i - 1) as isize) };
+ let ty = ProfiledType::new(obj);
+ VALUE::from(iseq).write_barrier(ty.class());
+ entry.opnd_types[i].observe(ty);
+ }
+ entry.num_profiles = entry.num_profiles.saturating_add(1);
+ entry.num_profiles == get_option!(num_profiles)
+ }
+
/// Get profiled operand types for a given instruction index
pub fn get_operand_types(&self, insn_idx: usize) -> Option<&[TypeDistribution]> {
self.entry(insn_idx).map(|e| e.opnd_types.as_slice()).filter(|s| !s.is_empty())
diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs
index 28bd623893..bd36464bb7 100644
--- a/zjit/src/stats.rs
+++ b/zjit/src/stats.rs
@@ -229,6 +229,7 @@ make_counters! {
exit_block_param_proxy_not_nil,
exit_block_param_wb_required,
exit_too_many_keyword_parameters,
+ exit_no_profile_send,
exit_splatkw_not_nil_or_hash,
exit_splatkw_polymorphic,
exit_splatkw_not_profiled,
@@ -626,6 +627,7 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter {
PatchPoint(Invariant::RootBoxOnly)
=> exit_patchpoint_root_box_only,
SendWhileTracing => exit_send_while_tracing,
+ NoProfileSend => exit_no_profile_send,
}
}