summaryrefslogtreecommitdiff
path: root/yjit/src
diff options
context:
space:
mode:
Diffstat (limited to 'yjit/src')
-rw-r--r--yjit/src/asm/mod.rs4
-rw-r--r--yjit/src/backend/arm64/mod.rs17
-rw-r--r--yjit/src/backend/ir.rs16
-rw-r--r--yjit/src/backend/tests.rs4
-rw-r--r--yjit/src/codegen.rs608
-rw-r--r--yjit/src/core.rs133
-rw-r--r--yjit/src/cruby.rs34
-rw-r--r--yjit/src/cruby_bindings.inc.rs588
-rw-r--r--yjit/src/invariants.rs10
-rw-r--r--yjit/src/stats.rs74
-rw-r--r--yjit/src/utils.rs15
-rw-r--r--yjit/src/virtualmem.rs13
-rw-r--r--yjit/src/yjit.rs18
13 files changed, 770 insertions, 764 deletions
diff --git a/yjit/src/asm/mod.rs b/yjit/src/asm/mod.rs
index ebdc205da9..9ef675b34d 100644
--- a/yjit/src/asm/mod.rs
+++ b/yjit/src/asm/mod.rs
@@ -441,12 +441,12 @@ impl CodeBlock {
// Ignore empty code ranges
if start_addr == end_addr {
- return (0..0).into_iter();
+ return 0..0;
}
let start_page = (start_addr.raw_addr(self) - mem_start) / self.page_size;
let end_page = (end_addr.raw_addr(self) - mem_start - 1) / self.page_size;
- (start_page..end_page + 1).into_iter()
+ start_page..end_page + 1
}
/// Get a (possibly dangling) direct pointer to the current write position
diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs
index 66e333f867..1712ad0302 100644
--- a/yjit/src/backend/arm64/mod.rs
+++ b/yjit/src/backend/arm64/mod.rs
@@ -98,7 +98,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) {
#[cfg(not(test))]
{
let end = cb.get_write_ptr();
- unsafe { rb_yjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) };
+ unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) };
}
}
@@ -878,14 +878,13 @@ impl Assembler
}
}
- /// Emit a push instruction for the given operand by adding to the stack
- /// pointer and then storing the given value.
+ /// Push a value to the stack by subtracting from the stack pointer then storing,
+ /// leaving an 8-byte gap for alignment.
fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) {
str_pre(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP));
}
- /// Emit a pop instruction into the given operand by loading the value
- /// and then subtracting from the stack pointer.
+ /// Pop a value from the stack by loading `[sp]` then adding to the stack pointer.
fn emit_pop(cb: &mut CodeBlock, opnd: A64Opnd) {
ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP));
}
@@ -1155,8 +1154,8 @@ impl Assembler
let regs = Assembler::get_caller_save_regs();
// Pop the state/flags register
- msr(cb, SystemRegister::NZCV, Self::SCRATCH0);
emit_pop(cb, Self::SCRATCH0);
+ msr(cb, SystemRegister::NZCV, Self::SCRATCH0);
for reg in regs.into_iter().rev() {
emit_pop(cb, A64Opnd::Reg(reg));
@@ -1361,7 +1360,7 @@ impl Assembler
#[cfg(not(test))]
cb.without_page_end_reserve(|cb| {
for (start, end) in cb.writable_addrs(start_ptr, cb.get_write_ptr()) {
- unsafe { rb_yjit_icache_invalidate(start as _, end as _) };
+ unsafe { rb_jit_icache_invalidate(start as _, end as _) };
}
});
@@ -1419,7 +1418,7 @@ mod tests {
fn test_emit_cpop_all() {
let (mut asm, mut cb) = setup_asm();
- asm.cpop_all();
+ asm.cpop_all(crate::core::RegMapping::default());
asm.compile_with_num_regs(&mut cb, 0);
}
@@ -1778,7 +1777,7 @@ mod tests {
assert_disasm!(cb, "e1ff9fd2e10370b2", {"
0x0: mov x1, #0xffff
- 0x4: orr x1, xzr, #0x10000
+ 0x4: mov x1, #0x10000
"});
}
diff --git a/yjit/src/backend/ir.rs b/yjit/src/backend/ir.rs
index b704a24985..e14f3f9cb2 100644
--- a/yjit/src/backend/ir.rs
+++ b/yjit/src/backend/ir.rs
@@ -6,7 +6,7 @@ use crate::codegen::{gen_counted_exit, gen_outlined_exit};
use crate::cruby::{vm_stack_canary, SIZEOF_VALUE_I32, VALUE, VM_ENV_DATA_SIZE};
use crate::virtualmem::CodePtr;
use crate::asm::{CodeBlock, OutlinedCb};
-use crate::core::{Context, RegMapping, RegOpnd, MAX_CTX_TEMPS};
+use crate::core::{Context, RegMapping, RegOpnd, MAX_CTX_LOCALS, MAX_CTX_TEMPS};
use crate::options::*;
use crate::stats::*;
@@ -242,7 +242,9 @@ impl Opnd
let last_idx = stack_size as i32 + VM_ENV_DATA_SIZE as i32 - 1;
assert!(last_idx <= idx, "Local index {} must be >= last local index {}", idx, last_idx);
assert!(idx <= last_idx + num_locals as i32, "Local index {} must be < last local index {} + local size {}", idx, last_idx, num_locals);
- RegOpnd::Local((last_idx + num_locals as i32 - idx) as u8)
+ // Indices that don't fit in u8 are capped to MAX_CTX_LOCALS, which is untrackable.
+ let local_idx = last_idx + num_locals as i32 - idx;
+ RegOpnd::Local(local_idx.try_into().unwrap_or(MAX_CTX_LOCALS as u8))
} else {
assert!(idx < stack_size as i32);
RegOpnd::Stack((stack_size as i32 - idx - 1) as u8)
@@ -1602,7 +1604,7 @@ impl Assembler
if c_args.len() > 0 {
// Resolve C argument dependencies
let c_args_len = c_args.len() as isize;
- let moves = Self::reorder_reg_moves(&c_args.drain(..).into_iter().collect());
+ let moves = Self::reorder_reg_moves(&std::mem::take(&mut c_args));
shift_live_ranges(&mut shifted_live_ranges, asm.insns.len(), moves.len() as isize - c_args_len);
// Push batched C arguments
@@ -1824,12 +1826,12 @@ impl Assembler {
out
}
- pub fn cpop_all(&mut self) {
+ pub fn cpop_all(&mut self, reg_mapping: RegMapping) {
self.push_insn(Insn::CPopAll);
// Re-enable ccall's RegMappings assertion disabled by cpush_all.
// cpush_all + cpop_all preserve all stack temp registers, so it's safe.
- self.set_reg_mapping(self.ctx.get_reg_mapping());
+ self.set_reg_mapping(reg_mapping);
}
pub fn cpop_into(&mut self, opnd: Opnd) {
@@ -1840,14 +1842,16 @@ impl Assembler {
self.push_insn(Insn::CPush(opnd));
}
- pub fn cpush_all(&mut self) {
+ pub fn cpush_all(&mut self) -> RegMapping {
self.push_insn(Insn::CPushAll);
// Mark all temps as not being in registers.
// Temps will be marked back as being in registers by cpop_all.
// We assume that cpush_all + cpop_all are used for C functions in utils.rs
// that don't require spill_regs for GC.
+ let mapping = self.ctx.get_reg_mapping();
self.set_reg_mapping(RegMapping::default());
+ mapping
}
pub fn cret(&mut self, opnd: Opnd) {
diff --git a/yjit/src/backend/tests.rs b/yjit/src/backend/tests.rs
index ac2f35b3d9..bfeea5163a 100644
--- a/yjit/src/backend/tests.rs
+++ b/yjit/src/backend/tests.rs
@@ -232,9 +232,9 @@ fn test_jcc_ptr()
let (mut asm, mut cb) = setup_asm();
let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4));
- let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK));
+ let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32));
asm.test(
- Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG),
+ Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32),
not_mask,
);
asm.jnz(side_exit);
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 9c06010527..743a10e15a 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -234,20 +234,36 @@ impl<'a> JITState<'a> {
result
}
- /// Return true if the current ISEQ could escape an environment.
+ /// Return true if the JIT code can use [`Self::assume_no_ep_escape`]
+ /// and will run with an on-stack (`!VM_ENV_ESCAPED_P`) environment.
///
- /// As of vm_push_frame(), EP is always equal to BP. However, after pushing
- /// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP.
- /// Also, some method calls escape the environment to the heap.
- fn escapes_ep(&self) -> bool {
+ /// ## Reasoning about ISEQs that are not currently running
+ ///
+ /// As of vm_push_frame() and its JIT code equivalent, EP is always equal to BP (the
+ /// environment is on-stack and has not escaped). We can usually assume this is the starting
+ /// condition upon entry into JIT code. However, after pushing a frame and before entry into
+ /// JIT code, some ISEQ setups call vm_bind_update_env(), which redirects EP.
+ ///
+ /// ## After making the assumption
+ ///
+ /// After JIT code entry, many ruby operations can have the environment escape to heap. These
+ /// are handled by [`crate::invariants`].
+ ///
+ /// Exceptional entry through jit_exec_exception() is an extreme case of the environment state
+ /// changing between vm_push_frame() and entry into JIT code. The frame could have been pushed
+ /// before YJIT is enabled. The exception entry point refuses entry with an escaped environment.
+ fn can_assume_on_stack_env(&self) -> bool {
match unsafe { get_iseq_body_type(self.iseq) } {
// <main> frame is always associated to TOPLEVEL_BINDING.
ISEQ_TYPE_MAIN |
// Kernel#eval uses a heap EP when a Binding argument is not nil.
- ISEQ_TYPE_EVAL => true,
- // If this ISEQ has previously escaped EP, give up the optimization.
- _ if iseq_escapes_ep(self.iseq) => true,
- _ => false,
+ ISEQ_TYPE_EVAL => false,
+ // Check the running environment if compiling for it
+ _ if unsafe { self.iseq == get_cfp_iseq(self.get_cfp()) && cfp_env_has_escaped(self.get_cfp()) } => false,
+ // If we've seen this ISEQ run with an escaped environment, give up the optimization
+ // to avoid excessive invalidations (even though it may be fine for soundness).
+ _ if seen_escaped_env(self.iseq) => false,
+ _ => true,
}
}
@@ -376,8 +392,8 @@ impl<'a> JITState<'a> {
if jit_ensure_block_entry_exit(self, asm).is_none() {
return false; // out of space, give up
}
- if self.escapes_ep() {
- return false; // EP has been escaped in this ISEQ. disable the optimization to avoid an invalidation loop.
+ if !self.can_assume_on_stack_env() {
+ return false; // Unsound or unprofitable to make the assumption
}
self.no_ep_escape = true;
true
@@ -438,7 +454,7 @@ impl<'a> JITState<'a> {
fn flush_perf_symbols(&self, cb: &CodeBlock) {
assert_eq!(0, self.perf_stack.len());
let path = format!("/tmp/perf-{}.map", std::process::id());
- let mut f = std::fs::File::options().create(true).append(true).open(path).unwrap();
+ let mut f = std::io::BufWriter::new(std::fs::File::options().create(true).append(true).open(path).unwrap());
for sym in self.perf_map.borrow().iter() {
if let (start, Some(end), name) = sym {
// In case the code straddles two pages, part of it belongs to the symbol.
@@ -821,11 +837,11 @@ fn gen_stub_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> {
/// Generate an exit to return to the interpreter
fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) {
- #[cfg(all(feature = "disasm", not(test)))]
- {
+ #[cfg(not(test))]
+ asm_comment!(asm, "exit to interpreter on {}", {
let opcode = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) };
- asm_comment!(asm, "exit to interpreter on {}", insn_name(opcode as usize));
- }
+ insn_name(opcode as usize)
+ });
if asm.ctx.is_return_landing() {
asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP));
@@ -1094,11 +1110,7 @@ pub fn gen_entry_prologue(
let code_ptr = cb.get_write_ptr();
let mut asm = Assembler::new(unsafe { get_iseq_body_local_table_size(iseq) });
- if get_option_ref!(dump_disasm).is_some() {
- asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0));
- } else {
- asm_comment!(asm, "YJIT entry");
- }
+ asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0));
asm.frame_setup();
@@ -1212,7 +1224,7 @@ fn gen_check_ints(
// Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages,
// signal_exec, or rb_postponed_job_flush.
- let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG));
+ let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32));
asm.test(interrupt_flag, interrupt_flag);
asm.jnz(Target::side_exit(counter));
@@ -1296,7 +1308,6 @@ pub fn gen_single_block(
let mut asm = Assembler::new(jit.num_locals());
asm.ctx = ctx;
- #[cfg(feature = "disasm")]
if get_option_ref!(dump_disasm).is_some() {
let blockid_idx = blockid.idx;
let chain_depth = if asm.ctx.get_chain_depth() > 0 { format!("(chain_depth: {})", asm.ctx.get_chain_depth()) } else { "".to_string() };
@@ -2263,7 +2274,8 @@ fn gen_expandarray(
let comptime_recv = jit.peek_at_stack(&asm.ctx, 0);
- // If the comptime receiver is not an array
+ // If the comptime receiver is not an array, speculate for when the `rb_check_array_type()`
+ // conversion returns nil and without side-effects (e.g. arbitrary method calls).
if !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_ARRAY) } {
// at compile time, ensure to_ary is not defined
let target_cme = unsafe { rb_callable_method_entry_or_negative(comptime_recv.class_of(), ID!(to_ary)) };
@@ -2275,13 +2287,19 @@ fn gen_expandarray(
return None;
}
+ // Bail when method_missing is defined to avoid generating code to call it.
+ // Also, for simplicity, bail when BasicObject#method_missing has been removed.
+ if !assume_method_basic_definition(jit, asm, comptime_recv.class_of(), ID!(method_missing)) {
+ gen_counter_incr(jit, asm, Counter::expandarray_method_missing);
+ return None;
+ }
+
// invalidate compile block if to_ary is later defined
jit.assume_method_lookup_stable(asm, target_cme);
jit_guard_known_klass(
jit,
asm,
- comptime_recv.class_of(),
array_opnd,
array_opnd.into(),
comptime_recv,
@@ -2311,7 +2329,7 @@ fn gen_expandarray(
}
// Get the compile-time array length
- let comptime_len = unsafe { rb_yjit_array_len(comptime_recv) as u32 };
+ let comptime_len = unsafe { rb_jit_array_len(comptime_recv) as u32 };
// Move the array from the stack and check that it's an array.
guard_object_is_array(
@@ -2423,22 +2441,83 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd {
ep_opnd
}
-// Gets the EP of the ISeq of the containing method, or "local level".
-// Equivalent of GET_LEP() macro.
+/// Get the EP of the ISeq of the containing method, or "local level EP".
+/// Equivalent to `GET_LEP()` with a constraint:
+/// `ISEQ_TYPE_METHOD == iseq_under_compilation->body->local_iseq->body->type`.
+/// No bad memory access happens when this condition is not met, just that the
+/// EP returned may not be `VM_ENV_FLAG_LOCAL`. Practically, ISeqs that don't
+/// meet this condition, such as `ISEQ_TYPE_TOP`, also don't need to use this operation.
fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd {
- // Equivalent of get_lvar_level() in compile.c
- fn get_lvar_level(iseq: IseqPtr) -> u32 {
- if iseq == unsafe { rb_get_iseq_body_local_iseq(iseq) } {
- 0
- } else {
- 1 + get_lvar_level(unsafe { rb_get_iseq_body_parent_iseq(iseq) })
- }
- }
-
- let level = get_lvar_level(jit.get_iseq());
+ // GET_LEP() chases the parent environment pointer to reach the local environment. Inside
+ // a descendant of ISEQ_TYPE_METHOD, it chases the same number of times as chasing
+ // the parent iseq pointer to reach the local iseq. See get_lvar_level() in compile.c
+ let mut iseq = jit.get_iseq();
+ let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) };
+ let mut level = 0;
+ while iseq != local_iseq {
+ iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) };
+ level += 1;
+ }
+ asm_comment!(asm, "get_lep(level: {level})");
gen_get_ep(asm, level)
}
+/// Load the value in the ME_CREF slot of the EP that contains the running CME.
+/// Returns the slot value directly, which is only useful for guarding that the
+/// CME has not changed. Unlike rb_vm_frame_method_entry(), we never dereference
+/// `ep[VM_ENV_DATA_INDEX_ME_CREF]` but rather rely on `ep[VM_ENV_DATA_INDEX_FLAGS]`
+/// to terminate the search, since we load the flags anyways for EP hopping.
+/// When `ep[VM_ENV_DATA_INDEX_ME_CREF]` is not a CME, it can't match the expected
+/// CME, and the guard fails.
+fn gen_get_running_cme_or_sentinal(jit: &JITState, asm: &mut Assembler) -> Opnd {
+ // When not in a block, the running CME is at `cfp->ep`.
+ if jit.iseq == unsafe { rb_get_iseq_body_local_iseq(jit.iseq) } {
+ asm_comment!(asm, "cfp->ep[VM_ENV_DATA_INDEX_ME_CREF]");
+ let lep_opnd = gen_get_ep(asm, 0);
+ Opnd::mem(
+ VALUE_BITS,
+ lep_opnd,
+ SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF,
+ )
+ } else {
+ // Loop through EPs to find the one containing the CME.
+ // Stop when we find an EP with VM_FRAME_FLAG_BMETHOD (bmethod frame with CME)
+ // or VM_ENV_FLAG_LOCAL (local frame which is METHOD/CFUNC/IFUNC with CME).
+ //
+ // We cannot unroll to a static hop count like gen_get_lep() because bmethods
+ // defined inside methods may have a CME that lives at a EP cloers to the
+ // starting EP than at the local and final EP level. Each level of nesting can
+ // dynamically run with and without VM_FRAME_FLAG_BMETHOD set.
+ asm_comment!(asm, "search for running cme");
+ let loop_label = asm.new_label("cme_loop");
+ let done_label = asm.new_label("cme_done");
+
+ let ep_opnd = Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_EP);
+ let ep_opnd = asm.load(ep_opnd);
+
+ asm.write_label(loop_label);
+ // Load flags from ep[VM_ENV_DATA_INDEX_FLAGS]
+ let flags = asm.load(Opnd::mem(VALUE_BITS, ep_opnd, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)));
+ // Check if VM_FRAME_FLAG_BMETHOD or VM_ENV_FLAG_LOCAL is set.
+ // If either is set, this EP contains the CME.
+ let check_flags = (VM_FRAME_FLAG_BMETHOD | VM_ENV_FLAG_LOCAL).as_usize();
+ asm.test(flags, check_flags.into());
+ asm.jnz(done_label);
+ // Get the previous EP from the current EP
+ // See GET_PREV_EP(ep) macro
+ // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03))
+ let offs = SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL;
+ let next_ep_opnd = asm.load(Opnd::mem(VALUE_BITS, ep_opnd, offs));
+ let next_ep_opnd = asm.and(next_ep_opnd, (!0x03_i64).into());
+ asm.load_into(ep_opnd, next_ep_opnd);
+ asm.jmp(loop_label);
+ asm.write_label(done_label);
+
+ // Load and return the CME from ep[VM_ENV_DATA_INDEX_ME_CREF]
+ asm.load(Opnd::mem(VALUE_BITS, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF))
+ }
+}
+
fn gen_getlocal_generic(
jit: &mut JITState,
asm: &mut Assembler,
@@ -2446,7 +2525,7 @@ fn gen_getlocal_generic(
level: u32,
) -> Option<CodegenStatus> {
// Split the block if we need to invalidate this instruction when EP escapes
- if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() {
+ if level == 0 && jit.can_assume_on_stack_env() && !jit.at_compile_target() {
return jit.defer_compilation(asm);
}
@@ -2516,6 +2595,7 @@ fn gen_setlocal_generic(
ep_offset: u32,
level: u32,
) -> Option<CodegenStatus> {
+ // Post condition: The type of of the set local is updated in the Context.
let value_type = asm.ctx.get_opnd_type(StackOpnd(0));
// Fallback because of write barrier
@@ -2537,11 +2617,16 @@ fn gen_setlocal_generic(
);
asm.stack_pop(1);
+ // Set local type in the context
+ if level == 0 {
+ let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize();
+ asm.ctx.set_local_type(local_idx, value_type);
+ }
return Some(KeepCompiling);
}
// Split the block if we need to invalidate this instruction when EP escapes
- if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() {
+ if level == 0 && jit.can_assume_on_stack_env() && !jit.at_compile_target() {
return jit.defer_compilation(asm);
}
@@ -2589,6 +2674,7 @@ fn gen_setlocal_generic(
);
}
+ // Set local type in the context
if level == 0 {
let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize();
asm.ctx.set_local_type(local_idx, value_type);
@@ -2678,7 +2764,7 @@ fn gen_newhash(
Some(KeepCompiling)
}
-fn gen_putstring(
+fn gen_dupstring(
jit: &mut JITState,
asm: &mut Assembler,
) -> Option<CodegenStatus> {
@@ -2698,7 +2784,7 @@ fn gen_putstring(
Some(KeepCompiling)
}
-fn gen_putchilledstring(
+fn gen_dupchilledstring(
jit: &mut JITState,
asm: &mut Assembler,
) -> Option<CodegenStatus> {
@@ -2753,7 +2839,7 @@ fn gen_checkkeyword(
) -> Option<CodegenStatus> {
// When a keyword is unspecified past index 32, a hash will be used
// instead. This can only happen in iseqs taking more than 32 keywords.
- if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= 32 } {
+ if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap() } {
return None;
}
@@ -2848,24 +2934,12 @@ fn gen_get_ivar(
recv: Opnd,
recv_opnd: YARVOpnd,
) -> Option<CodegenStatus> {
- let comptime_val_klass = comptime_receiver.class_of();
-
// If recv isn't already a register, load it.
let recv = match recv {
Opnd::InsnOut { .. } => recv,
_ => asm.load(recv),
};
- // Check if the comptime class uses a custom allocator
- let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) };
- let uses_custom_allocator = match custom_allocator {
- Some(alloc_fun) => {
- let allocate_instance = rb_class_allocate_instance as *const u8;
- alloc_fun as *const u8 != allocate_instance
- }
- None => false,
- };
-
// Check if the comptime receiver is a T_OBJECT
let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) };
// Use a general C call at the last chain to avoid exits on megamorphic shapes
@@ -2874,12 +2948,9 @@ fn gen_get_ivar(
gen_counter_incr(jit, asm, Counter::num_getivar_megamorphic);
}
- // If the class uses the default allocator, instances should all be T_OBJECT
- // NOTE: This assumes nobody changes the allocator of the class after allocation.
- // Eventually, we can encode whether an object is T_OBJECT or not
- // inside object shapes.
+ // NOTE: This assumes T_OBJECT can't ever have the same shape_id as any other type.
// too-complex shapes can't use index access, so we use rb_ivar_get for them too.
- if !receiver_t_object || uses_custom_allocator || comptime_receiver.shape_too_complex() || megamorphic {
+ if !comptime_receiver.heap_object_p() || comptime_receiver.shape_complex() || megamorphic {
// General case. Call rb_ivar_get().
// VALUE rb_ivar_get(VALUE obj, ID id)
asm_comment!(asm, "call rb_ivar_get()");
@@ -2904,7 +2975,7 @@ fn gen_get_ivar(
let ivar_index = unsafe {
let shape_id = comptime_receiver.shape_id_of();
- let mut ivar_index: u16 = 0;
+ let mut ivar_index: attr_index_t = 0;
if rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) {
Some(ivar_index as usize)
} else {
@@ -2915,9 +2986,6 @@ fn gen_get_ivar(
// Guard heap object (recv_opnd must be used before stack_pop)
guard_object_is_heap(asm, recv, recv_opnd, Counter::getivar_not_heap);
- // Compile time self is embedded and the ivar index lands within the object
- let embed_test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) };
-
let expected_shape = unsafe { rb_obj_shape_id(comptime_receiver) };
let shape_id_offset = unsafe { rb_shape_id_offset() };
let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, recv, shape_id_offset);
@@ -2946,28 +3014,37 @@ fn gen_get_ivar(
asm.mov(out_opnd, Qnil.into());
}
Some(ivar_index) => {
- if embed_test_result {
- // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h
-
- // Load the variable
- let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32;
- let ivar_opnd = Opnd::mem(64, recv, offs);
-
- // Push the ivar on the stack
- let out_opnd = asm.stack_push(Type::Unknown);
- asm.mov(out_opnd, ivar_opnd);
+ let ivar_opnd = if receiver_t_object {
+ if comptime_receiver.embedded_p() {
+ // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h
+
+ // Load the variable
+ let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32;
+ Opnd::mem(64, recv, offs)
+ } else {
+ // Compile time value is *not* embedded.
+
+ // Get a pointer to the extended table
+ let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32));
+
+ // Read the ivar from the extended table
+ Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32)
+ }
} else {
- // Compile time value is *not* embedded.
-
- // Get a pointer to the extended table
- let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32));
+ asm_comment!(asm, "call rb_ivar_get_at()");
- // Read the ivar from the extended table
- let ivar_opnd = Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32);
+ if assume_single_ractor_mode(jit, asm) {
+ asm.ccall(rb_ivar_get_at_no_ractor_check as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into())])
+ } else {
+ // The function could raise RactorIsolationError.
+ jit_prepare_non_leaf_call(jit, asm);
+ asm.ccall(rb_ivar_get_at as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into()), Opnd::UImm(ivar_name)])
+ }
+ };
- let out_opnd = asm.stack_push(Type::Unknown);
- asm.mov(out_opnd, ivar_opnd);
- }
+ // Push the ivar on the stack
+ let out_opnd = asm.stack_push(Type::Unknown);
+ asm.mov(out_opnd, ivar_opnd);
}
}
@@ -3073,8 +3150,6 @@ fn gen_set_ivar(
recv_opnd: YARVOpnd,
ic: Option<*const iseq_inline_iv_cache_entry>,
) -> Option<CodegenStatus> {
- let comptime_val_klass = comptime_receiver.class_of();
-
// If the comptime receiver is frozen, writing an IV will raise an exception
// and we don't want to JIT code to deal with that situation.
if comptime_receiver.is_frozen() {
@@ -3084,16 +3159,6 @@ fn gen_set_ivar(
let stack_type = asm.ctx.get_opnd_type(StackOpnd(0));
- // Check if the comptime class uses a custom allocator
- let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) };
- let uses_custom_allocator = match custom_allocator {
- Some(alloc_fun) => {
- let allocate_instance = rb_class_allocate_instance as *const u8;
- alloc_fun as *const u8 != allocate_instance
- }
- None => false,
- };
-
// Check if the comptime receiver is a T_OBJECT
let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) };
// Use a general C call at the last chain to avoid exits on megamorphic shapes
@@ -3103,10 +3168,10 @@ fn gen_set_ivar(
}
// Get the iv index
- let shape_too_complex = comptime_receiver.shape_too_complex();
- let ivar_index = if !comptime_receiver.special_const_p() && !shape_too_complex {
+ let shape_complex = comptime_receiver.shape_complex();
+ let ivar_index = if !comptime_receiver.special_const_p() && !shape_complex {
let shape_id = comptime_receiver.shape_id_of();
- let mut ivar_index: u16 = 0;
+ let mut ivar_index: attr_index_t = 0;
if unsafe { rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) } {
Some(ivar_index as usize)
} else {
@@ -3117,21 +3182,23 @@ fn gen_set_ivar(
};
// The current shape doesn't contain this iv, we need to transition to another shape.
- let mut new_shape_too_complex = false;
- let new_shape = if !shape_too_complex && receiver_t_object && ivar_index.is_none() {
+ let mut new_shape_complex = false;
+ let new_shape = if !shape_complex && receiver_t_object && ivar_index.is_none() {
let current_shape_id = comptime_receiver.shape_id_of();
- let next_shape_id = unsafe { rb_shape_transition_add_ivar_no_warnings(comptime_receiver, ivar_name) };
+ // We don't need to check about imemo_fields here because we're definitely looking at a T_OBJECT.
+ let klass = unsafe { rb_obj_class(comptime_receiver) };
+ let next_shape_id = unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id, ivar_name, klass) };
// If the VM ran out of shapes, or this class generated too many leaf,
- // it may be de-optimized into OBJ_TOO_COMPLEX_SHAPE (hash-table).
- new_shape_too_complex = unsafe { rb_yjit_shape_too_complex_p(next_shape_id) };
- if new_shape_too_complex {
+ // it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table).
+ new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id) };
+ if new_shape_complex {
Some((next_shape_id, None, 0_usize))
} else {
let current_capacity = unsafe { rb_yjit_shape_capacity(current_shape_id) };
let next_capacity = unsafe { rb_yjit_shape_capacity(next_shape_id) };
- // If the new shape has a different capacity, or is TOO_COMPLEX, we'll have to
+ // If the new shape has a different capacity, or is COMPLEX, we'll have to
// reallocate it.
let needs_extension = next_capacity != current_capacity;
@@ -3149,10 +3216,9 @@ fn gen_set_ivar(
None
};
- // If the receiver isn't a T_OBJECT, or uses a custom allocator,
- // then just write out the IV write as a function call.
+ // If the receiver isn't a T_OBJECT, then just write out the IV write as a function call.
// too-complex shapes can't use index access, so we use rb_ivar_get for them too.
- if !receiver_t_object || uses_custom_allocator || shape_too_complex || new_shape_too_complex || megamorphic {
+ if !receiver_t_object || shape_complex || new_shape_complex || megamorphic {
// The function could raise FrozenError.
// Note that this modifies REG_SP, which is why we do it first
jit_prepare_non_leaf_call(jit, asm);
@@ -3176,7 +3242,7 @@ fn gen_set_ivar(
asm.ccall(
rb_vm_setinstancevariable as *const u8,
vec![
- Opnd::const_ptr(jit.iseq as *const u8),
+ VALUE(jit.iseq as usize).into(),
Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF),
ivar_name.into(),
val_opnd,
@@ -3369,7 +3435,7 @@ fn gen_definedivar(
// Specialize base on compile time values
let comptime_receiver = jit.peek_at_self();
- if comptime_receiver.special_const_p() || comptime_receiver.shape_too_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH {
+ if comptime_receiver.special_const_p() || comptime_receiver.shape_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH {
// Fall back to calling rb_ivar_defined
// Save the PC and SP because the callee may allocate
@@ -3395,7 +3461,7 @@ fn gen_definedivar(
let shape_id = comptime_receiver.shape_id_of();
let ivar_exists = unsafe {
- let mut ivar_index: u16 = 0;
+ let mut ivar_index: attr_index_t = 0;
rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index)
};
@@ -3694,7 +3760,6 @@ fn gen_equality_specialized(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cString },
a_opnd,
a_opnd.into(),
comptime_a,
@@ -3720,7 +3785,6 @@ fn gen_equality_specialized(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cString },
b_opnd,
b_opnd.into(),
comptime_b,
@@ -3817,7 +3881,6 @@ fn gen_opt_aref(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cArray },
recv_opnd,
recv_opnd.into(),
comptime_recv,
@@ -3857,7 +3920,6 @@ fn gen_opt_aref(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cHash },
recv_opnd,
recv_opnd.into(),
comptime_recv,
@@ -3888,40 +3950,6 @@ fn gen_opt_aref(
}
}
-fn gen_opt_aset_with(
- jit: &mut JITState,
- asm: &mut Assembler,
-) -> Option<CodegenStatus> {
- // We might allocate or raise
- jit_prepare_non_leaf_call(jit, asm);
-
- let key_opnd = Opnd::Value(jit.get_arg(0));
- let recv_opnd = asm.stack_opnd(1);
- let value_opnd = asm.stack_opnd(0);
-
- extern "C" {
- fn rb_vm_opt_aset_with(recv: VALUE, key: VALUE, value: VALUE) -> VALUE;
- }
-
- let val_opnd = asm.ccall(
- rb_vm_opt_aset_with as *const u8,
- vec![
- recv_opnd,
- key_opnd,
- value_opnd,
- ],
- );
- asm.stack_pop(2); // Keep it on stack during GC
-
- asm.cmp(val_opnd, Qundef.into());
- asm.je(Target::side_exit(Counter::opt_aset_with_qundef));
-
- let top = asm.stack_push(Type::Unknown);
- asm.mov(top, val_opnd);
-
- return Some(KeepCompiling);
-}
-
fn gen_opt_aset(
jit: &mut JITState,
asm: &mut Assembler,
@@ -3944,7 +3972,6 @@ fn gen_opt_aset(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cArray },
recv,
recv.into(),
comptime_recv,
@@ -3956,7 +3983,6 @@ fn gen_opt_aset(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cInteger },
key,
key.into(),
comptime_key,
@@ -3989,7 +4015,6 @@ fn gen_opt_aset(
jit_guard_known_klass(
jit,
asm,
- unsafe { rb_cHash },
recv,
recv.into(),
comptime_recv,
@@ -4017,38 +4042,6 @@ fn gen_opt_aset(
}
}
-fn gen_opt_aref_with(
- jit: &mut JITState,
- asm: &mut Assembler,
-) -> Option<CodegenStatus>{
- // We might allocate or raise
- jit_prepare_non_leaf_call(jit, asm);
-
- let key_opnd = Opnd::Value(jit.get_arg(0));
- let recv_opnd = asm.stack_opnd(0);
-
- extern "C" {
- fn rb_vm_opt_aref_with(recv: VALUE, key: VALUE) -> VALUE;
- }
-
- let val_opnd = asm.ccall(
- rb_vm_opt_aref_with as *const u8,
- vec![
- recv_opnd,
- key_opnd
- ],
- );
- asm.stack_pop(1); // Keep it on stack during GC
-
- asm.cmp(val_opnd, Qundef.into());
- asm.je(Target::side_exit(Counter::opt_aref_with_qundef));
-
- let top = asm.stack_push(Type::Unknown);
- asm.mov(top, val_opnd);
-
- return Some(KeepCompiling);
-}
-
fn gen_opt_and(
jit: &mut JITState,
asm: &mut Assembler,
@@ -4658,21 +4651,8 @@ fn gen_opt_case_dispatch(
// Check that all cases are fixnums to avoid having to register BOP assumptions on
// all the types that case hashes support. This spends compile time to save memory.
- fn case_hash_all_fixnum_p(hash: VALUE) -> bool {
- let mut all_fixnum = true;
- unsafe {
- unsafe extern "C" fn per_case(key: st_data_t, _value: st_data_t, data: st_data_t) -> c_int {
- (if VALUE(key as usize).fixnum_p() {
- ST_CONTINUE
- } else {
- (data as *mut bool).write(false);
- ST_STOP
- }) as c_int
- }
- rb_hash_stlike_foreach(hash, Some(per_case), (&mut all_fixnum) as *mut _ as st_data_t);
- }
-
- all_fixnum
+ fn case_hash_all_fixnum_p(cdhash: VALUE) -> bool {
+ unsafe { rb_yjit_cdhash_all_fixnum_p(cdhash) }
}
// If megamorphic, fallback to compiling branch instructions after opt_case_dispatch
@@ -4699,12 +4679,11 @@ fn gen_opt_case_dispatch(
// Get the offset for the compile-time key
let mut offset = 0;
- unsafe { rb_hash_stlike_lookup(case_hash, comptime_key.0 as _, &mut offset) };
- let jump_offset = if offset == 0 {
+ let jump_offset = if unsafe { rb_yjit_cdhash_lookup(case_hash, comptime_key.0 as _, &mut offset) } == 0 {
// NOTE: If we hit the else branch with various values, it could negatively impact the performance.
else_offset
} else {
- (offset as u32) >> 1 // FIX2LONG
+ offset as u32
};
// Jump to the offset of case or else
@@ -4725,7 +4704,7 @@ fn gen_branchif(
let jump_offset = jit.get_arg(0).as_i32();
// Check for interrupts, but only on backward branches that may create loops
- if jump_offset < 0 {
+ if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchif_without_ints as usize {
gen_check_ints(asm, Counter::branchif_interrupted);
}
@@ -4777,7 +4756,7 @@ fn gen_branchunless(
let jump_offset = jit.get_arg(0).as_i32();
// Check for interrupts, but only on backward branches that may create loops
- if jump_offset < 0 {
+ if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchunless_without_ints as usize {
gen_check_ints(asm, Counter::branchunless_interrupted);
}
@@ -4830,7 +4809,7 @@ fn gen_branchnil(
let jump_offset = jit.get_arg(0).as_i32();
// Check for interrupts, but only on backward branches that may create loops
- if jump_offset < 0 {
+ if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchnil_without_ints as usize {
gen_check_ints(asm, Counter::branchnil_interrupted);
}
@@ -4941,7 +4920,6 @@ fn gen_opt_new(
perf_call!("opt_new: ", jit_guard_known_klass(
jit,
asm,
- comptime_recv_klass,
recv,
recv.into(),
comptime_recv,
@@ -4986,7 +4964,7 @@ fn gen_jump(
let jump_offset = jit.get_arg(0).as_i32();
// Check for interrupts, but only on backward branches that may create loops
- if jump_offset < 0 {
+ if jump_offset < 0 && jit.get_opcode() != YARVINSN_jump_without_ints as usize {
gen_check_ints(asm, Counter::jump_interrupted);
}
@@ -5012,13 +4990,13 @@ fn gen_jump(
fn jit_guard_known_klass(
jit: &mut JITState,
asm: &mut Assembler,
- known_klass: VALUE,
obj_opnd: Opnd,
insn_opnd: YARVOpnd,
sample_instance: VALUE,
max_chain_depth: u8,
counter: Counter,
) {
+ let known_klass = sample_instance.class_of();
let val_type = asm.ctx.get_opnd_type(insn_opnd);
if val_type.known_class() == Some(known_klass) {
@@ -5124,7 +5102,7 @@ fn jit_guard_known_klass(
assert_eq!(sample_instance.class_of(), rb_cString, "context says class is exactly ::String")
};
} else {
- assert!(!val_type.is_imm());
+ assert!(!val_type.is_imm(), "{insn_opnd:?} should be a heap object, but was {val_type:?} for {sample_instance:?}");
// Check that the receiver is a heap object
// Note: if we get here, the class doesn't have immediate instances.
@@ -5768,7 +5746,6 @@ fn jit_rb_float_plus(
jit_guard_known_klass(
jit,
asm,
- comptime_obj.class_of(),
obj,
obj.into(),
comptime_obj,
@@ -5810,7 +5787,6 @@ fn jit_rb_float_minus(
jit_guard_known_klass(
jit,
asm,
- comptime_obj.class_of(),
obj,
obj.into(),
comptime_obj,
@@ -5852,7 +5828,6 @@ fn jit_rb_float_mul(
jit_guard_known_klass(
jit,
asm,
- comptime_obj.class_of(),
obj,
obj.into(),
comptime_obj,
@@ -5894,7 +5869,6 @@ fn jit_rb_float_div(
jit_guard_known_klass(
jit,
asm,
- comptime_obj.class_of(),
obj,
obj.into(),
comptime_obj,
@@ -6158,7 +6132,6 @@ fn jit_rb_str_getbyte(
jit_guard_known_klass(
jit,
asm,
- comptime_idx.class_of(),
idx,
idx.into(),
comptime_idx,
@@ -6275,7 +6248,7 @@ fn jit_rb_str_dup(
jit_prepare_call_with_gc(jit, asm);
- let recv_opnd = asm.stack_pop(1);
+ let recv_opnd = asm.stack_opnd(0);
let recv_opnd = asm.load(recv_opnd);
let shape_id_offset = unsafe { rb_shape_id_offset() };
@@ -6284,8 +6257,10 @@ fn jit_rb_str_dup(
asm.jnz(Target::side_exit(Counter::send_str_dup_exivar));
// Call rb_str_dup
- let stack_ret = asm.stack_push(Type::CString);
let ret_opnd = asm.ccall(rb_str_dup as *const u8, vec![recv_opnd]);
+
+ asm.stack_pop(1);
+ let stack_ret = asm.stack_push(Type::CString);
asm.mov(stack_ret, ret_opnd);
true
@@ -6341,7 +6316,7 @@ fn jit_rb_str_concat_codepoint(
guard_object_is_fixnum(jit, asm, codepoint, StackOpnd(0));
- asm.ccall(rb_yjit_str_concat_codepoint as *const u8, vec![recv, codepoint]);
+ asm.ccall(rb_jit_str_concat_codepoint as *const u8, vec![recv, codepoint]);
// The receiver is the return value, so we only need to pop the codepoint argument off the stack.
// We can reuse the receiver slot in the stack as the return value.
@@ -6647,6 +6622,7 @@ fn jit_rb_f_block_given_p(
true
}
+/// Codegen for `block_given?` and `defined?(yield)`
fn gen_block_given(
jit: &mut JITState,
asm: &mut Assembler,
@@ -6746,7 +6722,7 @@ fn jit_thread_s_current(
asm.stack_pop(1);
// ec->thread_ptr
- let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR));
+ let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32));
// thread->self
let thread_self = Opnd::mem(64, ec_thread_opnd, RUBY_OFFSET_THREAD_SELF);
@@ -7211,7 +7187,7 @@ fn gen_send_cfunc(
asm_comment!(asm, "set ec->cfp");
let new_cfp = asm.lea(Opnd::mem(64, CFP, -(RUBY_SIZEOF_CONTROL_FRAME as i32)));
- asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), new_cfp);
+ asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), new_cfp);
if !kw_arg.is_null() {
// Build a hash from all kwargs passed
@@ -7307,7 +7283,7 @@ fn gen_send_cfunc(
// Pop the stack frame (ec->cfp++)
// Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved
// register
- let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP);
+ let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32);
asm.store(ec_cfp_opnd, CFP);
// cfunc calls may corrupt types
@@ -7473,7 +7449,7 @@ fn gen_send_bmethod(
) -> Option<CodegenStatus> {
let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) };
- let proc = unsafe { rb_yjit_get_proc_ptr(procv) };
+ let proc = unsafe { rb_jit_get_proc_ptr(procv) };
let proc_block = unsafe { &(*proc).block };
if proc_block.type_ != block_type_iseq {
@@ -7483,11 +7459,12 @@ fn gen_send_bmethod(
let capture = unsafe { proc_block.as_.captured.as_ref() };
let iseq = unsafe { *capture.code.iseq.as_ref() };
- // Optimize for single ractor mode and avoid runtime check for
- // "defined with an un-shareable Proc in a different Ractor"
- if !assume_single_ractor_mode(jit, asm) {
- gen_counter_incr(jit, asm, Counter::send_bmethod_ractor);
- return None;
+ if !procv.shareable_p() {
+ let ractor_serial = unsafe { rb_yjit_cme_ractor_serial(cme) };
+ asm_comment!(asm, "guard current ractor == {}", ractor_serial);
+ let current_ractor_serial = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_RACTOR_ID as i32));
+ asm.cmp(current_ractor_serial, ractor_serial.into());
+ asm.jne(Target::side_exit(Counter::send_bmethod_ractor));
}
// Passing a block to a block needs logic different from passing
@@ -7553,6 +7530,12 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, block: Opti
let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32();
let local_idx = ep_offset_to_local_idx(iseq, ep_offset);
+ // Only inline getlocal on a parameter. DCE in the IESQ builder can
+ // make a two-instruction ISEQ that does not return a parameter.
+ if local_idx >= unsafe { get_iseq_body_param_size(iseq) } {
+ return None;
+ }
+
if unsafe { rb_simple_iseq_p(iseq) } {
return Some(IseqReturn::LocalVariable(local_idx));
} else if unsafe { rb_iseq_only_kwparam_p(iseq) } {
@@ -7709,7 +7692,7 @@ fn gen_send_iseq(
gen_counter_incr(jit, asm, Counter::send_iseq_splat_not_array);
return None;
} else {
- unsafe { rb_yjit_array_len(array) as u32}
+ unsafe { rb_jit_array_len(array) as u32}
};
// Arity check accounting for size of the splat. When callee has rest parameters, we insert
@@ -7800,7 +7783,7 @@ fn gen_send_iseq(
gen_counter_incr(jit, asm, Counter::num_send_iseq);
// Shortcut for special `Primitive.attr! :leaf` builtins
- let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) };
+ let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) };
let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) };
let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) };
let opt_send_call = flags & VM_CALL_OPT_SEND != 0; // .send call is not currently supported for builtins
@@ -7977,6 +7960,11 @@ fn gen_send_iseq(
gen_counter_incr(jit, asm, Counter::send_iseq_clobbering_block_arg);
return None;
}
+ if iseq_has_rest || has_kwrest {
+ // The proc would be stored above the current stack top, where GC can't see it
+ gen_counter_incr(jit, asm, Counter::send_iseq_block_arg_gc_unsafe);
+ return None;
+ }
let proc = asm.stack_pop(1); // Pop first, as argc doesn't account for the block arg
let callee_specval = asm.ctx.sp_opnd(callee_specval);
asm.store(callee_specval, proc);
@@ -8389,7 +8377,7 @@ fn gen_send_iseq(
// We also do this after spill_regs() to avoid doubly spilling the same thing on asm.ccall().
if get_option!(gen_stats) {
// Protect caller-saved registers in case they're used for arguments
- asm.cpush_all();
+ let mapping = asm.cpush_all();
// Assemble the ISEQ name string
let name_str = get_iseq_name(iseq);
@@ -8399,7 +8387,7 @@ fn gen_send_iseq(
// Increment the counter for this cfunc
asm.ccall(incr_iseq_counter as *const u8, vec![iseq_idx.into()]);
- asm.cpop_all();
+ asm.cpop_all(mapping);
}
// The callee might change locals through Kernel#binding and other means.
@@ -8434,7 +8422,7 @@ fn gen_send_iseq(
asm_comment!(asm, "switch to new CFP");
let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into());
asm.mov(CFP, new_cfp);
- asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP);
+ asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP);
// Directly jump to the entry point of the callee
gen_direct_jump(
@@ -8985,9 +8973,9 @@ fn gen_struct_aref(
handle_opt_send_shift_stack(asm, argc);
}
- // All structs from the same Struct class should have the same
+ // All structs from the same Struct class and shape_id should have the same
// length. So if our comptime_recv is embedded all runtime
- // structs of the same class should be as well, and the same is
+ // structs of the same class and shape_id should be as well, and the same is
// true of the converse.
let embedded = unsafe { FL_TEST_RAW(comptime_recv, VALUE(RSTRUCT_EMBED_LEN_MASK)) };
@@ -9022,6 +9010,12 @@ fn gen_struct_aset(
return None;
}
+ // If the comptime receiver is frozen, writing a struct member will raise an exception
+ // and we don't want to JIT code to deal with that situation.
+ if comptime_recv.is_frozen() {
+ return None;
+ }
+
if c_method_tracing_currently_enabled(jit) {
// Struct accesses need fire c_call and c_return events, which we can't support
// See :attr-tracing:
@@ -9042,6 +9036,17 @@ fn gen_struct_aset(
assert!(unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRUCT) });
assert!((off as i64) < unsafe { RSTRUCT_LEN(comptime_recv) });
+ // Even if the comptime recv was not frozen, future recv may be. So we need to emit a guard
+ // that the recv is not frozen.
+ // We know all structs are heap objects, so we can check the flag directly.
+ let recv = asm.stack_opnd(1);
+ let recv = asm.load(recv);
+ let flags = asm.load(Opnd::mem(VALUE_BITS, recv, RUBY_OFFSET_RBASIC_FLAGS));
+ asm.test(flags, (RUBY_FL_FREEZE as u64).into());
+ asm.jnz(Target::side_exit(Counter::opt_aset_frozen));
+
+ // Not frozen, so we can proceed.
+
asm_comment!(asm, "struct aset");
let val = asm.stack_pop(1);
@@ -9153,7 +9158,6 @@ fn gen_send_general(
let recv_opnd: YARVOpnd = recv.into();
// Log the name of the method we're calling to
- #[cfg(feature = "disasm")]
asm_comment!(asm, "call to {}", get_method_name(Some(comptime_recv_klass), mid));
// Gather some statistics about sends
@@ -9173,7 +9177,6 @@ fn gen_send_general(
perf_call!("gen_send_general: ", jit_guard_known_klass(
jit,
asm,
- comptime_recv_klass,
recv,
recv_opnd,
comptime_recv,
@@ -9441,13 +9444,6 @@ fn gen_send_general(
return None;
}
- // Optimize for single ractor mode and avoid runtime check for
- // "defined with an un-shareable Proc in a different Ractor"
- if !assume_single_ractor_mode(jit, asm) {
- gen_counter_incr(jit, asm, Counter::send_call_multi_ractor);
- return None;
- }
-
// If this is a .send call we need to adjust the stack
if flags & VM_CALL_OPT_SEND != 0 {
handle_opt_send_shift_stack(asm, argc);
@@ -9652,7 +9648,24 @@ fn gen_sendforward(
jit: &mut JITState,
asm: &mut Assembler,
) -> Option<CodegenStatus> {
- return gen_send(jit, asm);
+ // Generate specialized code if possible
+ let cd = jit.get_arg(0).as_ptr();
+ let block = jit.get_arg(1).as_optional_ptr().map(|iseq| BlockHandler::BlockISeq(iseq));
+ if let Some(status) = perf_call! { gen_send_general(jit, asm, cd, block) } {
+ return Some(status);
+ }
+
+ // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of sendforward
+ let blockiseq = jit.get_arg(1).as_iseq();
+ gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| {
+ extern "C" {
+ fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE;
+ }
+ asm.ccall(
+ rb_vm_sendforward as *const u8,
+ vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()],
+ )
+ })
}
fn gen_invokeblock(
@@ -9726,7 +9739,7 @@ fn gen_invokeblock_specialized(
// If the current ISEQ is annotated to be inlined but it's not being inlined here,
// generate a dynamic dispatch to avoid making this yield megamorphic.
- if unsafe { rb_yjit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() {
+ if unsafe { rb_jit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() {
gen_counter_incr(jit, asm, Counter::invokeblock_iseq_not_inlined);
return None;
}
@@ -9817,7 +9830,7 @@ fn gen_invokesuper(
return Some(status);
}
- // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send
+ // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuper
let blockiseq = jit.get_arg(1).as_iseq();
gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| {
extern "C" {
@@ -9834,7 +9847,23 @@ fn gen_invokesuperforward(
jit: &mut JITState,
asm: &mut Assembler,
) -> Option<CodegenStatus> {
- return gen_invokesuper(jit, asm);
+ // Generate specialized code if possible
+ let cd = jit.get_arg(0).as_ptr();
+ if let Some(status) = gen_invokesuper_specialized(jit, asm, cd) {
+ return Some(status);
+ }
+
+ // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuperforward
+ let blockiseq = jit.get_arg(1).as_iseq();
+ gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| {
+ extern "C" {
+ fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE;
+ }
+ asm.ccall(
+ rb_vm_invokesuperforward as *const u8,
+ vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()],
+ )
+ })
}
fn gen_invokesuper_specialized(
@@ -9866,6 +9895,15 @@ fn gen_invokesuper_specialized(
return None;
}
+ let ci = unsafe { get_call_data_ci(cd) };
+ let ci_flags = unsafe { vm_ci_flag(ci) };
+
+ // Bail on ZSUPER inside a block method. They always raise.
+ if ci_flags & VM_CALL_ZSUPER != 0 && VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(me) } {
+ gen_counter_incr(jit, asm, Counter::invokesuper_bmethod_zsuper);
+ return None;
+ }
+
// FIXME: We should track and invalidate this block when this cme is invalidated
let current_defined_class = unsafe { (*me).defined_class };
let mid = unsafe { get_def_original_id((*me).def) };
@@ -9881,14 +9919,8 @@ fn gen_invokesuper_specialized(
let comptime_superclass =
unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) };
- let ci = unsafe { get_call_data_ci(cd) };
- let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap();
-
- let ci_flags = unsafe { vm_ci_flag(ci) };
-
// Don't JIT calls that aren't simple
// Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block.
-
if ci_flags & VM_CALL_KWARG != 0 {
gen_counter_incr(jit, asm, Counter::invokesuper_kwarg);
return None;
@@ -9907,6 +9939,7 @@ fn gen_invokesuper_specialized(
// cheaper calculations first, but since we specialize on the method entry
// and so only have to do this once at compile time this is fine to always
// check and side exit.
+ let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap();
let comptime_recv = jit.peek_at_stack(&asm.ctx, argc as isize);
if unsafe { rb_obj_is_kind_of(comptime_recv, current_defined_class) } == VALUE(0) {
gen_counter_incr(jit, asm, Counter::invokesuper_defined_class_mismatch);
@@ -9934,16 +9967,8 @@ fn gen_invokesuper_specialized(
return None;
}
- asm_comment!(asm, "guard known me");
- let lep_opnd = gen_get_lep(jit, asm);
- let ep_me_opnd = Opnd::mem(
- 64,
- lep_opnd,
- SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF,
- );
-
- let me_as_value = VALUE(me as usize);
- asm.cmp(ep_me_opnd, me_as_value.into());
+ let cme_opnd = gen_get_running_cme_or_sentinal(jit, asm);
+ asm.cmp(cme_opnd, VALUE::from(me).into());
jit_chain_guard(
JCC_JNE,
jit,
@@ -9988,7 +10013,7 @@ fn gen_leave(
asm_comment!(asm, "pop stack frame");
let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into());
asm.mov(CFP, incr_cfp);
- asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP);
+ asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP);
// Load the return value
let retval_opnd = asm.stack_pop(1);
@@ -10087,7 +10112,6 @@ fn gen_objtostring(
jit_guard_known_klass(
jit,
asm,
- comptime_recv.class_of(),
recv,
recv.into(),
comptime_recv,
@@ -10101,7 +10125,6 @@ fn gen_objtostring(
jit_guard_known_klass(
jit,
asm,
- comptime_recv.class_of(),
recv,
recv.into(),
comptime_recv,
@@ -10157,45 +10180,18 @@ fn gen_toregexp(
let opt = jit.get_arg(0).as_i64();
let cnt = jit.get_arg(1).as_usize();
- // Save the PC and SP because this allocates an object and could
- // raise an exception.
+ // Allocates objects and could raise an exception.
jit_prepare_non_leaf_call(jit, asm);
let values_ptr = asm.lea(asm.ctx.sp_opnd(-(cnt as i32)));
- let ary = asm.ccall(
- rb_ary_tmp_new_from_values as *const u8,
- vec![
- Opnd::Imm(0),
- cnt.into(),
- values_ptr,
- ]
+ let regexp = asm.ccall(
+ rb_reg_new_from_values as _,
+ vec![cnt.into(), values_ptr, opt.into()],
);
asm.stack_pop(cnt); // Let ccall spill them
-
- // Save the array so we can clear it later
- asm.cpush(ary);
- asm.cpush(ary); // Alignment
-
- let val = asm.ccall(
- rb_reg_new_ary as *const u8,
- vec![
- ary,
- Opnd::Imm(opt),
- ]
- );
-
- // The actual regex is in RAX now. Pop the temp array from
- // rb_ary_tmp_new_from_values into C arg regs so we can clear it
- let ary = asm.cpop(); // Alignment
- asm.cpop_into(ary);
-
- // The value we want to push on the stack is in RAX right now
let stack_ret = asm.stack_push(Type::UnknownHeap);
- asm.mov(stack_ret, val);
-
- // Clear the temp array.
- asm.ccall(rb_ary_clear as *const u8, vec![ary]);
+ asm.mov(stack_ret, regexp);
Some(KeepCompiling)
}
@@ -10285,7 +10281,7 @@ fn gen_getclassvariable(
let val_opnd = asm.ccall(
rb_vm_getclassvariable as *const u8,
vec![
- Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ),
+ VALUE(jit.iseq as usize).into(),
CFP,
Opnd::UImm(jit.get_arg(0).as_u64()),
Opnd::UImm(jit.get_arg(1).as_u64()),
@@ -10309,7 +10305,7 @@ fn gen_setclassvariable(
asm.ccall(
rb_vm_setclassvariable as *const u8,
vec![
- Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ),
+ VALUE(jit.iseq as usize).into(),
CFP,
Opnd::UImm(jit.get_arg(0).as_u64()),
val,
@@ -10777,8 +10773,8 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> {
YARVINSN_concattoarray => Some(gen_concattoarray),
YARVINSN_pushtoarray => Some(gen_pushtoarray),
YARVINSN_newrange => Some(gen_newrange),
- YARVINSN_putstring => Some(gen_putstring),
- YARVINSN_putchilledstring => Some(gen_putchilledstring),
+ YARVINSN_dupstring => Some(gen_dupstring),
+ YARVINSN_dupchilledstring => Some(gen_dupchilledstring),
YARVINSN_expandarray => Some(gen_expandarray),
YARVINSN_defined => Some(gen_defined),
YARVINSN_definedivar => Some(gen_definedivar),
@@ -10792,8 +10788,6 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> {
YARVINSN_opt_neq => Some(gen_opt_neq),
YARVINSN_opt_aref => Some(gen_opt_aref),
YARVINSN_opt_aset => Some(gen_opt_aset),
- YARVINSN_opt_aref_with => Some(gen_opt_aref_with),
- YARVINSN_opt_aset_with => Some(gen_opt_aset_with),
YARVINSN_opt_mult => Some(gen_opt_mult),
YARVINSN_opt_div => Some(gen_opt_div),
YARVINSN_opt_ltlt => Some(gen_opt_ltlt),
@@ -10815,6 +10809,10 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> {
YARVINSN_branchnil => Some(gen_branchnil),
YARVINSN_throw => Some(gen_throw),
YARVINSN_jump => Some(gen_jump),
+ YARVINSN_branchif_without_ints => Some(gen_branchif),
+ YARVINSN_branchunless_without_ints => Some(gen_branchunless),
+ YARVINSN_branchnil_without_ints => Some(gen_branchnil),
+ YARVINSN_jump_without_ints => Some(gen_jump),
YARVINSN_opt_new => Some(gen_opt_new),
YARVINSN_getblockparamproxy => Some(gen_getblockparamproxy),
@@ -11025,7 +11023,7 @@ impl CodegenGlobals {
#[cfg(not(test))]
let (mut cb, mut ocb) = {
- let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(exec_mem_size as u32) };
+ let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_size as u32) };
// Memory protection syscalls need page-aligned addresses, so check it here. Assuming
// `virt_block` is page-aligned, `second_half` should be page-aligned as long as the
@@ -11034,7 +11032,7 @@ impl CodegenGlobals {
//
// Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB
// (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though.
- let page_size = unsafe { rb_yjit_get_page_size() };
+ let page_size = unsafe { rb_jit_get_page_size() };
assert_eq!(
virt_block as usize % page_size.as_usize(), 0,
"Start of virtual address block should be page-aligned",
diff --git a/yjit/src/core.rs b/yjit/src/core.rs
index d42726bcc7..d08cd1fb26 100644
--- a/yjit/src/core.rs
+++ b/yjit/src/core.rs
@@ -34,7 +34,7 @@ use crate::invariants::*;
pub const MAX_CTX_TEMPS: usize = 8;
// Maximum number of local variable types or registers we keep track of
-const MAX_CTX_LOCALS: usize = 8;
+pub const MAX_CTX_LOCALS: usize = 8;
/// An index into `ISEQ_BODY(iseq)->iseq_encoded`. Points
/// to a YARV instruction or an instruction operand.
@@ -1099,7 +1099,7 @@ impl Context {
MapToLocal(local_idx) => {
bits.push_op(CtxOp::MapTempLocal);
bits.push_u3(stack_idx as u8);
- bits.push_u3(local_idx as u8);
+ bits.push_u3(local_idx);
}
MapToSelf => {
@@ -1769,6 +1769,35 @@ impl IseqPayload {
// Turn it into an iterator that owns the blocks and return
version_map.into_iter().flatten()
}
+
+ /// Get or create all blocks for a particular place in an iseq.
+ fn get_or_create_version_list(&mut self, insn_idx: usize) -> &mut VersionList {
+ if insn_idx >= self.version_map.len() {
+ self.version_map.resize(insn_idx + 1, VersionList::default());
+ }
+
+ return self.version_map.get_mut(insn_idx).unwrap();
+ }
+
+ // We cannot deallocate blocks immediately after invalidation since patching
+ // the code for setting up return addresses does not affect outstanding return
+ // addresses that are on stack and will use invalidated branch pointers when
+ // hit. Example:
+ // def foo(n)
+ // if n == 2
+ // # 1.times.each to create a cfunc frame to preserve the JIT frame
+ // # which will return to a stub housed in an invalidated block
+ // return 1.times.each { Object.define_method(:foo) {} }
+ // end
+ //
+ // foo(n + 1) # The block for this call houses the return branch stub
+ // end
+ // p foo(1)
+ pub fn delayed_deallocation(&mut self, blockref: BlockRef) {
+ block_assumptions_free(blockref);
+ unsafe { blockref.as_ref() }.set_iseq_null();
+ self.dead_blocks.push(blockref);
+ }
}
/// Get the payload for an iseq. For safety it's up to the caller to ensure the returned `&mut`
@@ -1814,22 +1843,24 @@ pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload {
pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) {
unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) {
// SAFETY: points to the local below
- let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) };
- callback(iseq);
+ let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast();
+ unsafe { (**callback)(iseq) };
}
- let mut data: &mut dyn FnMut(IseqPtr) = &mut callback;
- unsafe { rb_yjit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) };
+ let mut data: *mut dyn FnMut(IseqPtr) = &mut callback;
+ let data: *mut *mut dyn FnMut(IseqPtr) = &mut data;
+ unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), data.cast()) };
}
/// Iterate over all on-stack ISEQs
pub fn for_each_on_stack_iseq<F: FnMut(IseqPtr)>(mut callback: F) {
unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) {
// SAFETY: points to the local below
- let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) };
- callback(iseq);
+ let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast();
+ unsafe { (**callback)(iseq) };
}
- let mut data: &mut dyn FnMut(IseqPtr) = &mut callback;
- unsafe { rb_jit_cont_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) };
+ let mut data: *mut dyn FnMut(IseqPtr) = &mut callback;
+ let data: *mut *mut dyn FnMut(IseqPtr) = &mut data;
+ unsafe { rb_jit_cont_each_iseq(Some(callback_wrapper), data.cast()) };
}
/// Iterate over all on-stack ISEQ payloads
@@ -2140,21 +2171,6 @@ fn get_version_list(blockid: BlockId) -> Option<&'static mut VersionList> {
}
}
-/// Get or create all blocks for a particular place in an iseq.
-fn get_or_create_version_list(blockid: BlockId) -> &'static mut VersionList {
- let payload = get_or_create_iseq_payload(blockid.iseq);
- let insn_idx = blockid.idx.as_usize();
-
- // Expand the version map as necessary
- if insn_idx >= payload.version_map.len() {
- payload
- .version_map
- .resize(insn_idx + 1, VersionList::default());
- }
-
- return payload.version_map.get_mut(insn_idx).unwrap();
-}
-
/// Take all of the blocks for a particular place in an iseq
pub fn take_version_list(blockid: BlockId) -> VersionList {
let insn_idx = blockid.idx.as_usize();
@@ -2343,15 +2359,21 @@ unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) {
// Function entry blocks must have stack size 0
debug_assert!(!(block.iseq_range.start == 0 && Context::decode(block.ctx).stack_size > 0));
- let version_list = get_or_create_version_list(block.get_blockid());
+ // Use a single payload reference for both version_map and pages access
+ // to avoid mutable aliasing UB from multiple get_iseq_payload calls.
+ let iseq_payload = get_or_create_iseq_payload(block.iseq.get());
+ let insn_idx = block.get_blockid().idx.as_usize();
- // If this the first block being compiled with this block id
- if version_list.len() == 0 {
- incr_counter!(compiled_blockid_count);
- }
+ {
+ let version_list = iseq_payload.get_or_create_version_list(insn_idx);
- version_list.push(blockref);
- version_list.shrink_to_fit();
+ if version_list.is_empty() {
+ incr_counter!(compiled_blockid_count);
+ }
+
+ version_list.push(blockref);
+ version_list.shrink_to_fit();
+ }
// By writing the new block to the iseq, the iseq now
// contains new references to Ruby objects. Run write barriers.
@@ -2376,7 +2398,6 @@ unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) {
}
// Mark code pages for code GC
- let iseq_payload = get_iseq_payload(block.iseq.get()).unwrap();
for page in cb.addrs_to_pages(block.start_addr, block.end_addr.get()) {
iseq_payload.pages.insert(page);
}
@@ -2419,7 +2440,9 @@ impl<'a> JITState<'a> {
// Pending branches => actual branches
outgoing: MutableBranchList(Cell::new(self.pending_outgoing.into_iter().map(|pending_out| {
let pending_out = Rc::try_unwrap(pending_out)
- .ok().expect("all PendingBranchRefs should be unique when ready to construct a Block");
+ .unwrap_or_else(|rc| panic!(
+ "PendingBranchRef should be unique when ready to construct a Block. \
+ strong={} weak={}", Rc::strong_count(&rc), Rc::weak_count(&rc)));
pending_out.into_branch(NonNull::new(blockref as *mut Block).expect("no null from Box"))
}).collect()))
});
@@ -2427,7 +2450,7 @@ impl<'a> JITState<'a> {
// SAFETY: allocated with Box above
unsafe { ptr::write(blockref, block) };
- // Block is initialized now. Note that MaybeUnint<T> has the same layout as T.
+ // Block is initialized now. Note that MaybeUninit<T> has the same layout as T.
let blockref = NonNull::new(blockref as *mut Block).expect("no null from Box");
// Track all the assumptions the block makes as invariants
@@ -2493,6 +2516,11 @@ impl Block {
self.incoming.push(branch);
}
+ /// Mark this block as dead by setting its iseq to null.
+ pub fn set_iseq_null(&self) {
+ self.iseq.replace(ptr::null());
+ }
+
// Compute the size of the block code
pub fn code_size(&self) -> usize {
(self.end_addr.get().as_offset() - self.start_addr.as_offset()).try_into().unwrap()
@@ -3591,6 +3619,13 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) -
return CodegenGlobals::get_stub_exit_code().raw_ptr(cb);
}
+ // Bail if this branch is housed in an invalidated (dead) block.
+ // This only happens in rare invalidation scenarios and we need
+ // to avoid linking a dead block to a live block with a branch.
+ if branch.block.get().as_ref().iseq.get().is_null() {
+ return CodegenGlobals::get_stub_exit_code().raw_ptr(cb);
+ }
+
(cfp, original_interp_sp)
};
@@ -3790,7 +3825,7 @@ pub fn gen_branch_stub_hit_trampoline(ocb: &mut OutlinedCb) -> Option<CodePtr> {
let mut asm = Assembler::new_without_iseq();
// For `branch_stub_hit(branch_ptr, target_idx, ec)`,
- // `branch_ptr` and `target_idx` is different for each stub,
+ // `branch_ptr` and `target_idx` are different for each stub,
// but the call and what's after is the same. This trampoline
// is the unchanging part.
// Since this trampoline is static, it allows code GC inside
@@ -4289,7 +4324,9 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
// in this function before we removed it, so it's well connected.
unsafe { remove_from_graph(*blockref) };
- delayed_deallocation(*blockref);
+ if let Some(payload) = get_iseq_payload(id_being_invalidated.iseq) {
+ payload.delayed_deallocation(*blockref);
+ }
ocb.unwrap().mark_all_executable();
cb.mark_all_executable();
@@ -4297,28 +4334,6 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
incr_counter!(invalidation_count);
}
-// We cannot deallocate blocks immediately after invalidation since there
-// could be stubs waiting to access branch pointers. Return stubs can do
-// this since patching the code for setting up return addresses does not
-// affect old return addresses that are already set up to use potentially
-// invalidated branch pointers. Example:
-// def foo(n)
-// if n == 2
-// # 1.times.each to create a cfunc frame to preserve the JIT frame
-// # which will return to a stub housed in an invalidated block
-// return 1.times.each { Object.define_method(:foo) {} }
-// end
-//
-// foo(n + 1)
-// end
-// p foo(1)
-pub fn delayed_deallocation(blockref: BlockRef) {
- block_assumptions_free(blockref);
-
- let payload = get_iseq_payload(unsafe { blockref.as_ref() }.iseq.get()).unwrap();
- payload.dead_blocks.push(blockref);
-}
-
trait RefUnchecked {
type Contained;
unsafe fn ref_unchecked(&self) -> &Self::Contained;
diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs
index f7a08b3b18..8b99a8154a 100644
--- a/yjit/src/cruby.rs
+++ b/yjit/src/cruby.rs
@@ -123,7 +123,6 @@ extern "C" {
pub fn rb_float_new(d: f64) -> VALUE;
pub fn rb_hash_empty_p(hash: VALUE) -> VALUE;
- pub fn rb_yjit_str_concat_codepoint(str: VALUE, codepoint: VALUE);
pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE;
pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE;
pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE;
@@ -198,8 +197,8 @@ pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx;
pub use rb_get_call_data_ci as get_call_data_ci;
pub use rb_yarv_str_eql_internal as rb_str_eql_internal;
pub use rb_yarv_ary_entry_internal as rb_ary_entry_internal;
-pub use rb_yjit_fix_div_fix as rb_fix_div_fix;
-pub use rb_yjit_fix_mod_fix as rb_fix_mod_fix;
+pub use rb_jit_fix_div_fix as rb_fix_div_fix;
+pub use rb_jit_fix_mod_fix as rb_fix_mod_fix;
pub use rb_FL_TEST as FL_TEST;
pub use rb_FL_TEST_RAW as FL_TEST_RAW;
pub use rb_RB_TYPE_P as RB_TYPE_P;
@@ -362,6 +361,11 @@ impl VALUE {
!self.special_const_p()
}
+ /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`.
+ pub fn shareable_p(self) -> bool {
+ (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0
+ }
+
/// Return true if the value is a Ruby Fixnum (immediate-size integer)
pub fn fixnum_p(self) -> bool {
let VALUE(cval) = self;
@@ -440,8 +444,8 @@ impl VALUE {
unsafe { rb_obj_frozen_p(self) != VALUE(0) }
}
- pub fn shape_too_complex(self) -> bool {
- unsafe { rb_yjit_shape_obj_too_complex_p(self) }
+ pub fn shape_complex(self) -> bool {
+ unsafe { rb_yjit_shape_obj_complex_p(self) }
}
pub fn shape_id_of(self) -> u32 {
@@ -450,7 +454,7 @@ impl VALUE {
pub fn embedded_p(self) -> bool {
unsafe {
- FL_TEST_RAW(self, VALUE(ROBJECT_EMBED as usize)) != VALUE(0)
+ FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0)
}
}
@@ -594,6 +598,13 @@ impl From<VALUE> for u16 {
}
}
+/// Check whether a control frame has an escaped environment
+pub unsafe fn cfp_env_has_escaped(cfp: *mut rb_control_frame_struct) -> bool {
+ use crate::utils::IntoUsize;
+ let ep = get_cfp_ep(cfp);
+ 0 != ep.offset(VM_ENV_DATA_INDEX_FLAGS as isize).read().0 & VM_ENV_FLAG_ESCAPED.as_usize()
+}
+
/// Produce a Ruby string from a Rust string slice
pub fn rust_str_to_ruby(str: &str) -> VALUE {
unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) }
@@ -677,7 +688,7 @@ where
let line = loc.line;
let mut recursive_lock_level: c_uint = 0;
- unsafe { rb_yjit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) };
+ unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) };
let ret = match catch_unwind(func) {
Ok(result) => result,
@@ -697,7 +708,7 @@ where
}
};
- unsafe { rb_yjit_vm_unlock(&mut recursive_lock_level, file, line) };
+ unsafe { rb_jit_vm_unlock(&mut recursive_lock_level, file, line) };
ret
}
@@ -768,12 +779,6 @@ mod manual_defs {
pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48;
pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56;
- // Constants from rb_execution_context_t vm_core.h
- pub const RUBY_OFFSET_EC_CFP: i32 = 16;
- pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: i32 = 32; // rb_atomic_t (u32)
- pub const RUBY_OFFSET_EC_INTERRUPT_MASK: i32 = 36; // rb_atomic_t (u32)
- pub const RUBY_OFFSET_EC_THREAD_PTR: i32 = 48;
-
// Constants from rb_thread_t in vm_core.h
pub const RUBY_OFFSET_THREAD_SELF: i32 = 16;
@@ -816,6 +821,7 @@ pub(crate) mod ids {
def_ids! {
name: NULL content: b""
name: respond_to_missing content: b"respond_to_missing?"
+ name: method_missing content: b"method_missing"
name: to_ary content: b"to_ary"
name: to_s content: b"to_s"
name: eq content: b"=="
diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs
index c8a58f424e..272c10fde9 100644
--- a/yjit/src/cruby_bindings.inc.rs
+++ b/yjit/src/cruby_bindings.inc.rs
@@ -165,6 +165,7 @@ pub const NIL_REDEFINED_OP_FLAG: u32 = 512;
pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024;
pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048;
pub const PROC_REDEFINED_OP_FLAG: u32 = 4096;
+pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31;
pub const VM_ENV_DATA_SIZE: u32 = 3;
pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2;
pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1;
@@ -224,11 +225,9 @@ pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32;
pub const RUBY_FL_PROMOTED: ruby_fl_type = 32;
pub const RUBY_FL_UNUSED6: ruby_fl_type = 64;
pub const RUBY_FL_FINALIZE: ruby_fl_type = 128;
-pub const RUBY_FL_TAINT: ruby_fl_type = 0;
pub const RUBY_FL_EXIVAR: ruby_fl_type = 0;
pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256;
-pub const RUBY_FL_UNTRUSTED: ruby_fl_type = 0;
-pub const RUBY_FL_UNUSED9: ruby_fl_type = 512;
+pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512;
pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024;
pub const RUBY_FL_FREEZE: ruby_fl_type = 2048;
pub const RUBY_FL_USER0: ruby_fl_type = 4096;
@@ -259,46 +258,12 @@ pub const RSTRING_FSTR: ruby_rstring_flags = 536870912;
pub type ruby_rstring_flags = u32;
pub type st_data_t = ::std::os::raw::c_ulong;
pub type st_index_t = st_data_t;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct st_hash_type {
- pub compare: ::std::option::Option<
- unsafe extern "C" fn(arg1: st_data_t, arg2: st_data_t) -> ::std::os::raw::c_int,
- >,
- pub hash: ::std::option::Option<unsafe extern "C" fn(arg1: st_data_t) -> st_index_t>,
-}
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct st_table_entry {
- _unused: [u8; 0],
-}
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct st_table {
- pub entry_power: ::std::os::raw::c_uchar,
- pub bin_power: ::std::os::raw::c_uchar,
- pub size_ind: ::std::os::raw::c_uchar,
- pub rebuilds_num: ::std::os::raw::c_uint,
- pub type_: *const st_hash_type,
- pub num_entries: st_index_t,
- pub bins: *mut st_index_t,
- pub entries_start: st_index_t,
- pub entries_bound: st_index_t,
- pub entries: *mut st_table_entry,
-}
pub const ST_CONTINUE: st_retval = 0;
pub const ST_STOP: st_retval = 1;
pub const ST_DELETE: st_retval = 2;
pub const ST_CHECK: st_retval = 3;
pub const ST_REPLACE: st_retval = 4;
pub type st_retval = u32;
-pub type st_foreach_callback_func = ::std::option::Option<
- unsafe extern "C" fn(
- arg1: st_data_t,
- arg2: st_data_t,
- arg3: st_data_t,
- ) -> ::std::os::raw::c_int,
->;
pub const RARRAY_EMBED_FLAG: ruby_rarray_flags = 8192;
pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536;
pub type ruby_rarray_flags = u32;
@@ -306,7 +271,7 @@ pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15;
pub type ruby_rarray_consts = u32;
pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192;
pub type ruby_rmodule_flags = u32;
-pub const ROBJECT_EMBED: ruby_robject_flags = 8192;
+pub const ROBJECT_HEAP: ruby_robject_flags = 65536;
pub type ruby_robject_flags = u32;
pub type rb_block_call_func = ::std::option::Option<
unsafe extern "C" fn(
@@ -356,45 +321,24 @@ pub const BOP_NIL_P: ruby_basic_operators = 15;
pub const BOP_SUCC: ruby_basic_operators = 16;
pub const BOP_GT: ruby_basic_operators = 17;
pub const BOP_GE: ruby_basic_operators = 18;
-pub const BOP_NOT: ruby_basic_operators = 19;
-pub const BOP_NEQ: ruby_basic_operators = 20;
-pub const BOP_MATCH: ruby_basic_operators = 21;
-pub const BOP_FREEZE: ruby_basic_operators = 22;
-pub const BOP_UMINUS: ruby_basic_operators = 23;
-pub const BOP_MAX: ruby_basic_operators = 24;
-pub const BOP_MIN: ruby_basic_operators = 25;
-pub const BOP_HASH: ruby_basic_operators = 26;
-pub const BOP_CALL: ruby_basic_operators = 27;
-pub const BOP_AND: ruby_basic_operators = 28;
-pub const BOP_OR: ruby_basic_operators = 29;
-pub const BOP_CMP: ruby_basic_operators = 30;
-pub const BOP_DEFAULT: ruby_basic_operators = 31;
-pub const BOP_PACK: ruby_basic_operators = 32;
-pub const BOP_INCLUDE_P: ruby_basic_operators = 33;
-pub const BOP_LAST_: ruby_basic_operators = 34;
+pub const BOP_GTGT: ruby_basic_operators = 19;
+pub const BOP_NOT: ruby_basic_operators = 20;
+pub const BOP_NEQ: ruby_basic_operators = 21;
+pub const BOP_MATCH: ruby_basic_operators = 22;
+pub const BOP_FREEZE: ruby_basic_operators = 23;
+pub const BOP_UMINUS: ruby_basic_operators = 24;
+pub const BOP_MAX: ruby_basic_operators = 25;
+pub const BOP_MIN: ruby_basic_operators = 26;
+pub const BOP_HASH: ruby_basic_operators = 27;
+pub const BOP_CALL: ruby_basic_operators = 28;
+pub const BOP_AND: ruby_basic_operators = 29;
+pub const BOP_OR: ruby_basic_operators = 30;
+pub const BOP_CMP: ruby_basic_operators = 31;
+pub const BOP_DEFAULT: ruby_basic_operators = 32;
+pub const BOP_PACK: ruby_basic_operators = 33;
+pub const BOP_INCLUDE_P: ruby_basic_operators = 34;
+pub const BOP_LAST_: ruby_basic_operators = 35;
pub type ruby_basic_operators = u32;
-#[repr(C)]
-pub struct rb_namespace_struct {
- pub ns_object: VALUE,
- pub ns_id: ::std::os::raw::c_long,
- pub top_self: VALUE,
- pub load_path: VALUE,
- pub load_path_snapshot: VALUE,
- pub load_path_check_cache: VALUE,
- pub expanded_load_path: VALUE,
- pub loaded_features: VALUE,
- pub loaded_features_snapshot: VALUE,
- pub loaded_features_realpaths: VALUE,
- pub loaded_features_realpath_map: VALUE,
- pub loaded_features_index: *mut st_table,
- pub loading_table: *mut st_table,
- pub ruby_dln_libmap: VALUE,
- pub gvar_tbl: VALUE,
- pub is_builtin: bool,
- pub is_user: bool,
- pub is_optional: bool,
-}
-pub type rb_namespace_t = rb_namespace_struct;
pub type rb_serial_t = ::std::os::raw::c_ulonglong;
pub const imemo_env: imemo_type = 0;
pub const imemo_cref: imemo_type = 1;
@@ -405,10 +349,13 @@ pub const imemo_memo: imemo_type = 5;
pub const imemo_ment: imemo_type = 6;
pub const imemo_iseq: imemo_type = 7;
pub const imemo_tmpbuf: imemo_type = 8;
+pub const imemo_cvar_entry: imemo_type = 9;
pub const imemo_callinfo: imemo_type = 10;
pub const imemo_callcache: imemo_type = 11;
pub const imemo_constcache: imemo_type = 12;
pub const imemo_fields: imemo_type = 13;
+pub const imemo_subclasses: imemo_type = 14;
+pub const imemo_cdhash: imemo_type = 15;
pub type imemo_type = u32;
#[repr(C)]
#[derive(Debug, Copy, Clone)]
@@ -424,6 +371,7 @@ pub struct vm_ifunc {
pub data: *const ::std::os::raw::c_void,
pub argc: vm_ifunc_argc,
}
+pub type rb_atomic_t = ::std::os::raw::c_uint;
pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0;
pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1;
pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2;
@@ -502,8 +450,6 @@ pub type ruby_vm_throw_flags = u32;
pub struct iseq_inline_constant_cache_entry {
pub flags: VALUE,
pub value: VALUE,
- pub _unused1: VALUE,
- pub _unused2: VALUE,
pub ic_cref: *const rb_cref_t,
}
#[repr(C)]
@@ -537,10 +483,11 @@ pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1;
pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2;
pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4;
pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8;
+pub const BUILTIN_ATTR_WITHOUT_INTERRUPTS: rb_builtin_attr = 16;
pub type rb_builtin_attr = u32;
#[repr(C)]
#[derive(Debug, Copy, Clone)]
-pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword {
+pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword {
pub num: ::std::os::raw::c_int,
pub required_num: ::std::os::raw::c_int,
pub bits_start: ::std::os::raw::c_int,
@@ -582,7 +529,6 @@ pub type rb_control_frame_t = rb_control_frame_struct;
#[repr(C)]
pub struct rb_proc_t {
pub block: rb_block,
- pub ns: *const rb_namespace_t,
pub _bitfield_align_1: [u8; 0],
pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>,
pub __bindgen_padding_0: [u8; 7usize],
@@ -678,19 +624,19 @@ pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256;
pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512;
pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024;
pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048;
-pub const VM_FRAME_FLAG_NS_SWITCH: vm_frame_env_flags = 4096;
-pub const VM_FRAME_FLAG_LOAD_ISEQ: vm_frame_env_flags = 8192;
+pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096;
pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2;
pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4;
pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8;
pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16;
pub type vm_frame_env_flags = u32;
-pub type attr_index_t = u16;
+pub type attr_index_t = u8;
pub type shape_id_t = u32;
-pub const SHAPE_ID_HAS_IVAR_MASK: _bindgen_ty_37 = 134742014;
-pub type _bindgen_ty_37 = u32;
+pub const SHAPE_ID_HAS_IVAR_MASK: shape_id_mask = 8912894;
+pub type shape_id_mask = u32;
#[repr(C)]
pub struct rb_cvar_class_tbl_entry {
+ pub imemo_flags: VALUE,
pub index: u32,
pub global_cvar_state: rb_serial_t,
pub cref: *const rb_cref_t,
@@ -715,7 +661,7 @@ pub type vm_call_flag_bits = u32;
#[repr(C)]
pub struct rb_callinfo_kwarg {
pub keyword_len: ::std::os::raw::c_int,
- pub references: ::std::os::raw::c_int,
+ pub references: rb_atomic_t,
pub keywords: __IncompleteArrayField<VALUE>,
}
#[repr(C)]
@@ -723,8 +669,8 @@ pub struct rb_callinfo {
pub flags: VALUE,
pub kwarg: *const rb_callinfo_kwarg,
pub mid: VALUE,
- pub flag: VALUE,
- pub argc: VALUE,
+ pub flag: ::std::os::raw::c_uint,
+ pub argc: ::std::os::raw::c_uint,
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
@@ -773,8 +719,8 @@ pub const YARVINSN_putnil: ruby_vminsn_type = 17;
pub const YARVINSN_putself: ruby_vminsn_type = 18;
pub const YARVINSN_putobject: ruby_vminsn_type = 19;
pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20;
-pub const YARVINSN_putstring: ruby_vminsn_type = 21;
-pub const YARVINSN_putchilledstring: ruby_vminsn_type = 22;
+pub const YARVINSN_dupstring: ruby_vminsn_type = 21;
+pub const YARVINSN_dupchilledstring: ruby_vminsn_type = 22;
pub const YARVINSN_concatstrings: ruby_vminsn_type = 23;
pub const YARVINSN_anytostring: ruby_vminsn_type = 24;
pub const YARVINSN_toregexp: ruby_vminsn_type = 25;
@@ -828,153 +774,190 @@ pub const YARVINSN_jump: ruby_vminsn_type = 72;
pub const YARVINSN_branchif: ruby_vminsn_type = 73;
pub const YARVINSN_branchunless: ruby_vminsn_type = 74;
pub const YARVINSN_branchnil: ruby_vminsn_type = 75;
-pub const YARVINSN_once: ruby_vminsn_type = 76;
-pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 77;
-pub const YARVINSN_opt_plus: ruby_vminsn_type = 78;
-pub const YARVINSN_opt_minus: ruby_vminsn_type = 79;
-pub const YARVINSN_opt_mult: ruby_vminsn_type = 80;
-pub const YARVINSN_opt_div: ruby_vminsn_type = 81;
-pub const YARVINSN_opt_mod: ruby_vminsn_type = 82;
-pub const YARVINSN_opt_eq: ruby_vminsn_type = 83;
-pub const YARVINSN_opt_neq: ruby_vminsn_type = 84;
-pub const YARVINSN_opt_lt: ruby_vminsn_type = 85;
-pub const YARVINSN_opt_le: ruby_vminsn_type = 86;
-pub const YARVINSN_opt_gt: ruby_vminsn_type = 87;
-pub const YARVINSN_opt_ge: ruby_vminsn_type = 88;
-pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 89;
-pub const YARVINSN_opt_and: ruby_vminsn_type = 90;
-pub const YARVINSN_opt_or: ruby_vminsn_type = 91;
-pub const YARVINSN_opt_aref: ruby_vminsn_type = 92;
-pub const YARVINSN_opt_aset: ruby_vminsn_type = 93;
-pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 94;
-pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 95;
-pub const YARVINSN_opt_length: ruby_vminsn_type = 96;
-pub const YARVINSN_opt_size: ruby_vminsn_type = 97;
-pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 98;
-pub const YARVINSN_opt_succ: ruby_vminsn_type = 99;
-pub const YARVINSN_opt_not: ruby_vminsn_type = 100;
-pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 101;
-pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 102;
-pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 103;
-pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 104;
-pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 105;
-pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 106;
-pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 107;
-pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 108;
-pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 109;
-pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 110;
-pub const YARVINSN_trace_nop: ruby_vminsn_type = 111;
-pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 112;
-pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 113;
-pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 114;
-pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 115;
-pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 116;
-pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 117;
-pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 118;
-pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 119;
-pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 120;
-pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 121;
-pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 122;
-pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 123;
-pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 124;
-pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 125;
-pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 126;
-pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 127;
-pub const YARVINSN_trace_putnil: ruby_vminsn_type = 128;
-pub const YARVINSN_trace_putself: ruby_vminsn_type = 129;
-pub const YARVINSN_trace_putobject: ruby_vminsn_type = 130;
-pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 131;
-pub const YARVINSN_trace_putstring: ruby_vminsn_type = 132;
-pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 133;
-pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 134;
-pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 135;
-pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 136;
-pub const YARVINSN_trace_intern: ruby_vminsn_type = 137;
-pub const YARVINSN_trace_newarray: ruby_vminsn_type = 138;
-pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 139;
-pub const YARVINSN_trace_duparray: ruby_vminsn_type = 140;
-pub const YARVINSN_trace_duphash: ruby_vminsn_type = 141;
-pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 142;
-pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 143;
-pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 144;
-pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 145;
-pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 146;
-pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 147;
-pub const YARVINSN_trace_newhash: ruby_vminsn_type = 148;
-pub const YARVINSN_trace_newrange: ruby_vminsn_type = 149;
-pub const YARVINSN_trace_pop: ruby_vminsn_type = 150;
-pub const YARVINSN_trace_dup: ruby_vminsn_type = 151;
-pub const YARVINSN_trace_dupn: ruby_vminsn_type = 152;
-pub const YARVINSN_trace_swap: ruby_vminsn_type = 153;
-pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 154;
-pub const YARVINSN_trace_topn: ruby_vminsn_type = 155;
-pub const YARVINSN_trace_setn: ruby_vminsn_type = 156;
-pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 157;
-pub const YARVINSN_trace_defined: ruby_vminsn_type = 158;
-pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 159;
-pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 160;
-pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 161;
-pub const YARVINSN_trace_checktype: ruby_vminsn_type = 162;
-pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 163;
-pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 164;
-pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 165;
-pub const YARVINSN_trace_send: ruby_vminsn_type = 166;
-pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 167;
-pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 168;
-pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 169;
-pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 170;
-pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 171;
-pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 172;
-pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 173;
-pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 174;
-pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 175;
-pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 176;
-pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 177;
-pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 178;
-pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 179;
-pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 180;
-pub const YARVINSN_trace_leave: ruby_vminsn_type = 181;
-pub const YARVINSN_trace_throw: ruby_vminsn_type = 182;
-pub const YARVINSN_trace_jump: ruby_vminsn_type = 183;
-pub const YARVINSN_trace_branchif: ruby_vminsn_type = 184;
-pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 185;
-pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 186;
-pub const YARVINSN_trace_once: ruby_vminsn_type = 187;
-pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 188;
-pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 189;
-pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 190;
-pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 191;
-pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 192;
-pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 193;
-pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 194;
-pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 195;
-pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 196;
-pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 197;
-pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 198;
-pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 199;
-pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 200;
-pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 201;
-pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 202;
-pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 203;
-pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 204;
-pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 205;
-pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 206;
-pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 207;
-pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 208;
-pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 209;
-pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 210;
-pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 211;
-pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 212;
-pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 213;
-pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 214;
-pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 215;
-pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 216;
-pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 217;
-pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 218;
-pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 219;
-pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 220;
-pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 221;
-pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 222;
+pub const YARVINSN_jump_without_ints: ruby_vminsn_type = 76;
+pub const YARVINSN_branchif_without_ints: ruby_vminsn_type = 77;
+pub const YARVINSN_branchunless_without_ints: ruby_vminsn_type = 78;
+pub const YARVINSN_branchnil_without_ints: ruby_vminsn_type = 79;
+pub const YARVINSN_once: ruby_vminsn_type = 80;
+pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 81;
+pub const YARVINSN_opt_plus: ruby_vminsn_type = 82;
+pub const YARVINSN_opt_minus: ruby_vminsn_type = 83;
+pub const YARVINSN_opt_mult: ruby_vminsn_type = 84;
+pub const YARVINSN_opt_div: ruby_vminsn_type = 85;
+pub const YARVINSN_opt_mod: ruby_vminsn_type = 86;
+pub const YARVINSN_opt_eq: ruby_vminsn_type = 87;
+pub const YARVINSN_opt_neq: ruby_vminsn_type = 88;
+pub const YARVINSN_opt_lt: ruby_vminsn_type = 89;
+pub const YARVINSN_opt_le: ruby_vminsn_type = 90;
+pub const YARVINSN_opt_gt: ruby_vminsn_type = 91;
+pub const YARVINSN_opt_ge: ruby_vminsn_type = 92;
+pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 93;
+pub const YARVINSN_opt_and: ruby_vminsn_type = 94;
+pub const YARVINSN_opt_or: ruby_vminsn_type = 95;
+pub const YARVINSN_opt_aref: ruby_vminsn_type = 96;
+pub const YARVINSN_opt_aset: ruby_vminsn_type = 97;
+pub const YARVINSN_opt_length: ruby_vminsn_type = 98;
+pub const YARVINSN_opt_size: ruby_vminsn_type = 99;
+pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 100;
+pub const YARVINSN_opt_succ: ruby_vminsn_type = 101;
+pub const YARVINSN_opt_not: ruby_vminsn_type = 102;
+pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 103;
+pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 104;
+pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 105;
+pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 106;
+pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 107;
+pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 108;
+pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 109;
+pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 110;
+pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 111;
+pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 112;
+pub const YARVINSN_trace_nop: ruby_vminsn_type = 113;
+pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 114;
+pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 115;
+pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 116;
+pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 117;
+pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 118;
+pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 119;
+pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 120;
+pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 121;
+pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 122;
+pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 123;
+pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 124;
+pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 125;
+pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 126;
+pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 127;
+pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 128;
+pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 129;
+pub const YARVINSN_trace_putnil: ruby_vminsn_type = 130;
+pub const YARVINSN_trace_putself: ruby_vminsn_type = 131;
+pub const YARVINSN_trace_putobject: ruby_vminsn_type = 132;
+pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 133;
+pub const YARVINSN_trace_dupstring: ruby_vminsn_type = 134;
+pub const YARVINSN_trace_dupchilledstring: ruby_vminsn_type = 135;
+pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 136;
+pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 137;
+pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 138;
+pub const YARVINSN_trace_intern: ruby_vminsn_type = 139;
+pub const YARVINSN_trace_newarray: ruby_vminsn_type = 140;
+pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 141;
+pub const YARVINSN_trace_duparray: ruby_vminsn_type = 142;
+pub const YARVINSN_trace_duphash: ruby_vminsn_type = 143;
+pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 144;
+pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 145;
+pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 146;
+pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 147;
+pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 148;
+pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 149;
+pub const YARVINSN_trace_newhash: ruby_vminsn_type = 150;
+pub const YARVINSN_trace_newrange: ruby_vminsn_type = 151;
+pub const YARVINSN_trace_pop: ruby_vminsn_type = 152;
+pub const YARVINSN_trace_dup: ruby_vminsn_type = 153;
+pub const YARVINSN_trace_dupn: ruby_vminsn_type = 154;
+pub const YARVINSN_trace_swap: ruby_vminsn_type = 155;
+pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 156;
+pub const YARVINSN_trace_topn: ruby_vminsn_type = 157;
+pub const YARVINSN_trace_setn: ruby_vminsn_type = 158;
+pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 159;
+pub const YARVINSN_trace_defined: ruby_vminsn_type = 160;
+pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 161;
+pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 162;
+pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 163;
+pub const YARVINSN_trace_checktype: ruby_vminsn_type = 164;
+pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 165;
+pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 166;
+pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 167;
+pub const YARVINSN_trace_send: ruby_vminsn_type = 168;
+pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 169;
+pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 170;
+pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 171;
+pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 172;
+pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 173;
+pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 174;
+pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 175;
+pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 176;
+pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 177;
+pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 178;
+pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 179;
+pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 180;
+pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 181;
+pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 182;
+pub const YARVINSN_trace_leave: ruby_vminsn_type = 183;
+pub const YARVINSN_trace_throw: ruby_vminsn_type = 184;
+pub const YARVINSN_trace_jump: ruby_vminsn_type = 185;
+pub const YARVINSN_trace_branchif: ruby_vminsn_type = 186;
+pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 187;
+pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 188;
+pub const YARVINSN_trace_jump_without_ints: ruby_vminsn_type = 189;
+pub const YARVINSN_trace_branchif_without_ints: ruby_vminsn_type = 190;
+pub const YARVINSN_trace_branchunless_without_ints: ruby_vminsn_type = 191;
+pub const YARVINSN_trace_branchnil_without_ints: ruby_vminsn_type = 192;
+pub const YARVINSN_trace_once: ruby_vminsn_type = 193;
+pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 194;
+pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 195;
+pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 196;
+pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 197;
+pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 198;
+pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 199;
+pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 200;
+pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 201;
+pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 202;
+pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 203;
+pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 204;
+pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 205;
+pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 206;
+pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 207;
+pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 208;
+pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 209;
+pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 210;
+pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 211;
+pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 212;
+pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 213;
+pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 214;
+pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 215;
+pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 216;
+pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 217;
+pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 218;
+pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 219;
+pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 220;
+pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 221;
+pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222;
+pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223;
+pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224;
+pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225;
+pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226;
+pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227;
+pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228;
+pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229;
+pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230;
+pub const YARVINSN_zjit_send: ruby_vminsn_type = 231;
+pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232;
+pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233;
+pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234;
+pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235;
+pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236;
+pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237;
+pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238;
+pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239;
+pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240;
+pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241;
+pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242;
+pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243;
+pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244;
+pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245;
+pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246;
+pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247;
+pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248;
+pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249;
+pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250;
+pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251;
+pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252;
+pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253;
+pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254;
+pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255;
+pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256;
+pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257;
+pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258;
+pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259;
pub type ruby_vminsn_type = u32;
pub type rb_iseq_callback = ::std::option::Option<
unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void),
@@ -998,13 +981,22 @@ pub const DEFINED_REF: defined_type = 15;
pub const DEFINED_FUNC: defined_type = 16;
pub const DEFINED_CONST_FROM: defined_type = 17;
pub type defined_type = u32;
-pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: robject_offsets = 16;
-pub const ROBJECT_OFFSET_AS_ARY: robject_offsets = 16;
-pub type robject_offsets = u32;
-pub const RUBY_OFFSET_RSTRING_LEN: rstring_offsets = 16;
-pub type rstring_offsets = u32;
-pub type rb_seq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword;
-pub type rb_iseq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword;
+pub type rb_seq_param_keyword_struct =
+ rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword;
+pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16;
+pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16;
+pub const RCLASS_OFFSET_PRIME_FIELDS_OBJ: jit_bindgen_constants = 40;
+pub const TDATA_OFFSET_FIELDS_OBJ: jit_bindgen_constants = 16;
+pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16;
+pub const RB_SHAPE_FLAG_SHIFT: jit_bindgen_constants = 32;
+pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16;
+pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32;
+pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36;
+pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48;
+pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64;
+pub type jit_bindgen_constants = u32;
+pub type rb_iseq_param_keyword_struct =
+ rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword;
extern "C" {
pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void);
pub fn rb_class_attached_object(klass: VALUE) -> VALUE;
@@ -1084,7 +1076,11 @@ extern "C" {
pub fn rb_obj_info_dump(obj: VALUE);
pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE;
pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE;
- pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE;
+ pub fn rb_reg_new_from_values(
+ cnt: ::std::os::raw::c_long,
+ elements: *const VALUE,
+ opt: ::std::os::raw::c_int,
+ ) -> VALUE;
pub fn rb_ary_tmp_new_from_values(
arg1: VALUE,
arg2: ::std::os::raw::c_long,
@@ -1096,7 +1092,7 @@ extern "C" {
elts: *const VALUE,
) -> VALUE;
pub fn rb_vm_top_self() -> VALUE;
- pub static mut rb_vm_insns_count: u64;
+ pub static mut rb_vm_insn_count: u64;
pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t;
pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t;
pub fn rb_callable_method_entry_or_negative(
@@ -1118,7 +1114,13 @@ extern "C" {
pub fn rb_shape_id_offset() -> i32;
pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t;
pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool;
- pub fn rb_shape_transition_add_ivar_no_warnings(obj: VALUE, id: ID) -> shape_id_t;
+ pub fn rb_shape_transition_add_ivar_no_warnings(
+ shape_id: shape_id_t,
+ id: ID,
+ klass: VALUE,
+ ) -> shape_id_t;
+ pub fn rb_ivar_get_at(obj: VALUE, index: attr_index_t, id: ID) -> VALUE;
+ pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE;
pub fn rb_gvar_get(arg1: ID) -> VALUE;
pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE;
pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32);
@@ -1138,11 +1140,6 @@ extern "C" {
chilled: bool,
) -> VALUE;
pub fn rb_to_hash_type(obj: VALUE) -> VALUE;
- pub fn rb_hash_stlike_foreach(
- hash: VALUE,
- func: st_foreach_callback_func,
- arg: st_data_t,
- ) -> ::std::os::raw::c_int;
pub fn rb_hash_new_with_size(size: st_index_t) -> VALUE;
pub fn rb_hash_resurrect(hash: VALUE) -> VALUE;
pub fn rb_hash_stlike_lookup(
@@ -1168,47 +1165,22 @@ extern "C" {
lines: *mut ::std::os::raw::c_int,
) -> ::std::os::raw::c_int;
pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void);
- pub fn rb_yjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool;
- pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32);
- pub fn rb_yjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool;
- pub fn rb_yjit_array_len(a: VALUE) -> ::std::os::raw::c_long;
- pub fn rb_yjit_icache_invalidate(
- start: *mut ::std::os::raw::c_void,
- end: *mut ::std::os::raw::c_void,
- );
pub fn rb_yjit_exit_locations_dict(
yjit_raw_samples: *mut VALUE,
yjit_line_samples: *mut ::std::os::raw::c_int,
samples_len: ::std::os::raw::c_int,
) -> VALUE;
- pub fn rb_yjit_get_page_size() -> u32;
- pub fn rb_yjit_reserve_addr_space(mem_size: u32) -> *mut u8;
pub fn rb_c_method_tracing_currently_enabled(ec: *const rb_execution_context_t) -> bool;
pub fn rb_full_cfunc_return(ec: *mut rb_execution_context_t, return_value: VALUE);
pub fn rb_iseq_get_yjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void;
pub fn rb_iseq_set_yjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void);
- pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t);
- pub fn rb_yjit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t;
pub fn rb_get_symbol_id(namep: VALUE) -> ID;
- pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE;
- pub fn rb_optimized_call(
- recv: *mut VALUE,
- ec: *mut rb_execution_context_t,
- argc: ::std::os::raw::c_int,
- argv: *mut VALUE,
- kw_splat: ::std::os::raw::c_int,
- block_handler: VALUE,
- ) -> VALUE;
- pub fn rb_yjit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint;
pub fn rb_yjit_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function;
pub fn rb_yjit_str_simple_append(str1: VALUE, str2: VALUE) -> VALUE;
pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE;
- pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE;
pub fn rb_str_neq_internal(str1: VALUE, str2: VALUE) -> VALUE;
pub fn rb_ary_unshift_m(argc: ::std::os::raw::c_int, argv: *mut VALUE, ary: VALUE) -> VALUE;
pub fn rb_yjit_rb_ary_subseq_length(ary: VALUE, beg: ::std::os::raw::c_long) -> VALUE;
- pub fn rb_yjit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE;
- pub fn rb_yjit_fix_mod_fix(recv: VALUE, obj: VALUE) -> VALUE;
pub fn rb_yjit_ruby2_keywords_splat_p(obj: VALUE) -> usize;
pub fn rb_yjit_splat_varg_checks(
sp: *mut VALUE,
@@ -1220,37 +1192,32 @@ extern "C" {
pub fn rb_yjit_iseq_inspect(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_char;
pub fn rb_RSTRUCT_SET(st: VALUE, k: ::std::os::raw::c_int, v: VALUE);
pub fn rb_ENCODING_GET(obj: VALUE) -> ::std::os::raw::c_int;
- pub fn rb_yjit_multi_ractor_p() -> bool;
pub fn rb_yjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool;
- pub fn rb_yjit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void);
pub fn rb_yjit_obj_written(
old: VALUE,
young: VALUE,
file: *const ::std::os::raw::c_char,
line: ::std::os::raw::c_int,
);
- pub fn rb_yjit_vm_lock_then_barrier(
- recursive_lock_level: *mut ::std::os::raw::c_uint,
- file: *const ::std::os::raw::c_char,
- line: ::std::os::raw::c_int,
- );
- pub fn rb_yjit_vm_unlock(
- recursive_lock_level: *mut ::std::os::raw::c_uint,
- file: *const ::std::os::raw::c_char,
- line: ::std::os::raw::c_int,
- );
pub fn rb_object_shape_count() -> VALUE;
- pub fn rb_yjit_shape_too_complex_p(shape_id: shape_id_t) -> bool;
- pub fn rb_yjit_shape_obj_too_complex_p(obj: VALUE) -> bool;
+ pub fn rb_yjit_shape_obj_complex_p(obj: VALUE) -> bool;
pub fn rb_yjit_shape_capacity(shape_id: shape_id_t) -> attr_index_t;
pub fn rb_yjit_shape_index(shape_id: shape_id_t) -> attr_index_t;
pub fn rb_yjit_sendish_sp_pops(ci: *const rb_callinfo) -> usize;
pub fn rb_yjit_invokeblock_sp_pops(ci: *const rb_callinfo) -> usize;
+ pub fn rb_yjit_cme_ractor_serial(cme: *const rb_callable_method_entry_t) -> rb_serial_t;
pub fn rb_yjit_set_exception_return(
cfp: *mut rb_control_frame_t,
leave_exit: *mut ::std::os::raw::c_void,
leave_exception: *mut ::std::os::raw::c_void,
);
+ pub fn rb_vm_instruction_size() -> u32;
+ pub fn rb_yjit_cdhash_all_fixnum_p(cdhash: VALUE) -> bool;
+ pub fn rb_yjit_cdhash_lookup(
+ cdhash: VALUE,
+ key: st_data_t,
+ val: *mut st_data_t,
+ ) -> ::std::os::raw::c_int;
pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint;
pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE;
pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int;
@@ -1280,6 +1247,17 @@ extern "C" {
) -> *mut rb_method_cfunc_t;
pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize;
pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID;
+ pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE;
+ pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t;
+ pub fn rb_optimized_call(
+ recv: VALUE,
+ ec: *mut rb_execution_context_t,
+ argc: ::std::os::raw::c_int,
+ argv: *mut VALUE,
+ kw_splat: ::std::os::raw::c_int,
+ block_handler: VALUE,
+ ) -> VALUE;
+ pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint;
pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int;
pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void;
pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t;
@@ -1328,6 +1306,34 @@ extern "C" {
pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int;
pub fn rb_assert_cme_handle(handle: VALUE);
pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE;
+ pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long;
pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE);
pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE);
+ pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool;
+ pub fn rb_jit_multi_ractor_p() -> bool;
+ pub fn rb_jit_vm_lock_then_barrier(
+ recursive_lock_level: *mut ::std::os::raw::c_uint,
+ file: *const ::std::os::raw::c_char,
+ line: ::std::os::raw::c_int,
+ );
+ pub fn rb_jit_vm_unlock(
+ recursive_lock_level: *mut ::std::os::raw::c_uint,
+ file: *const ::std::os::raw::c_char,
+ line: ::std::os::raw::c_int,
+ );
+ pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t);
+ pub fn rb_jit_get_page_size() -> u32;
+ pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8;
+ pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void);
+ pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool;
+ pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32);
+ pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool;
+ pub fn rb_jit_icache_invalidate(
+ start: *mut ::std::os::raw::c_void,
+ end: *mut ::std::os::raw::c_void,
+ );
+ pub fn rb_jit_fix_mod_fix(recv: VALUE, obj: VALUE) -> VALUE;
+ pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE;
+ pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE;
+ pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE);
}
diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs
index 6ae1342ce3..f3bb91ddc1 100644
--- a/yjit/src/invariants.rs
+++ b/yjit/src/invariants.rs
@@ -168,8 +168,8 @@ pub fn track_no_ep_escape_assumption(uninit_block: BlockRef, iseq: IseqPtr) {
.insert(uninit_block);
}
-/// Returns true if a given ISEQ has previously escaped an environment.
-pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool {
+/// Returns true if a given ISEQ has escaped an environment since YJIT boot.
+pub fn seen_escaped_env(iseq: IseqPtr) -> bool {
Invariants::get_instance()
.no_ep_escape_iseqs
.get(&iseq)
@@ -206,7 +206,7 @@ pub fn assume_method_basic_definition(
/// Tracks that a block is assuming it is operating in single-ractor mode.
#[must_use]
pub fn assume_single_ractor_mode(jit: &mut JITState, asm: &mut Assembler) -> bool {
- if unsafe { rb_yjit_multi_ractor_p() } {
+ if unsafe { rb_jit_multi_ractor_p() } {
false
} else {
if jit_ensure_block_entry_exit(jit, asm).is_none() {
@@ -495,7 +495,7 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, ins
return;
};
- if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_yjit_multi_ractor_p() } {
+ if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_jit_multi_ractor_p() } {
// We can't generate code in these situations, so no need to invalidate.
// See gen_opt_getinlinecache.
return;
@@ -642,7 +642,7 @@ pub extern "C" fn rb_yjit_tracing_invalidate_all() {
if on_stack_iseqs.contains(&iseq) {
// This ISEQ is running, so we can't free blocks immediately
for block in blocks {
- delayed_deallocation(block);
+ payload.delayed_deallocation(block);
}
payload.dead_blocks.shrink_to_fit();
} else {
diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs
index ba84b7a549..b23cd91618 100644
--- a/yjit/src/stats.rs
+++ b/yjit/src/stats.rs
@@ -1,9 +1,8 @@
//! Everything related to the collection of runtime stats in YJIT
//! See the --yjit-stats command-line option
-use std::alloc::{GlobalAlloc, Layout, System};
use std::ptr::addr_of_mut;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::Ordering;
use std::time::Instant;
use std::collections::HashMap;
@@ -12,6 +11,10 @@ use crate::cruby::*;
use crate::options::*;
use crate::yjit::{yjit_enabled_p, YJIT_INIT_TIME};
+#[cfg(feature = "stats_allocator")]
+#[path = "../../jit/src/lib.rs"]
+mod jit;
+
/// Running total of how many ISeqs are in the system.
#[no_mangle]
pub static mut rb_yjit_live_iseq_count: u64 = 0;
@@ -20,43 +23,9 @@ pub static mut rb_yjit_live_iseq_count: u64 = 0;
#[no_mangle]
pub static mut rb_yjit_iseq_alloc_count: u64 = 0;
-/// A middleware to count Rust-allocated bytes as yjit_alloc_size.
-#[global_allocator]
-static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) };
-
-pub struct StatsAlloc {
- alloc_size: AtomicUsize,
-}
-
-unsafe impl GlobalAlloc for StatsAlloc {
- unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
- self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst);
- System.alloc(layout)
- }
-
- unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
- self.alloc_size.fetch_sub(layout.size(), Ordering::SeqCst);
- System.dealloc(ptr, layout)
- }
-
- unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
- self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst);
- System.alloc_zeroed(layout)
- }
-
- unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- if new_size > layout.size() {
- self.alloc_size.fetch_add(new_size - layout.size(), Ordering::SeqCst);
- } else if new_size < layout.size() {
- self.alloc_size.fetch_sub(layout.size() - new_size, Ordering::SeqCst);
- }
- System.realloc(ptr, layout, new_size)
- }
-}
-
/// The number of bytes YJIT has allocated on the Rust heap.
pub fn yjit_alloc_size() -> usize {
- GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst)
+ jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst)
}
/// Mapping of C function / ISEQ name to integer indices
@@ -121,7 +90,9 @@ pub extern "C" fn incr_iseq_counter(idx: usize) {
iseq_call_count[idx] += 1;
}
-// YJIT exit counts for each instruction type
+/// YJIT exit counts for each instruction type.
+/// Note that `VM_INSTRUCTION_SIZE` is an upper bound and the actual number
+/// of VM opcodes may be different in the build. See [`rb_vm_instruction_size()`]
const VM_INSTRUCTION_SIZE_USIZE: usize = VM_INSTRUCTION_SIZE as usize;
static mut EXIT_OP_COUNT: [u64; VM_INSTRUCTION_SIZE_USIZE] = [0; VM_INSTRUCTION_SIZE_USIZE];
@@ -277,6 +248,7 @@ pub const DEFAULT_COUNTERS: &'static [Counter] = &[
Counter::compiled_blockid_count,
Counter::compiled_block_count,
Counter::deleted_defer_block_count,
+ Counter::exceptional_entry_escaped_env,
Counter::compiled_branch_count,
Counter::compile_time_ns,
Counter::compilation_failure,
@@ -349,7 +321,6 @@ macro_rules! ptr_to_counter {
}
};
}
-pub(crate) use ptr_to_counter;
// Declare all the counters we track
make_counters! {
@@ -384,6 +355,7 @@ make_counters! {
send_iseq_arity_error,
send_iseq_block_arg_type,
send_iseq_clobbering_block_arg,
+ send_iseq_block_arg_gc_unsafe,
send_iseq_complex_discard_extras,
send_iseq_leaf_builtin_block_arg_block_param,
send_iseq_kw_splat_non_nil,
@@ -431,6 +403,7 @@ make_counters! {
invokesuper_megamorphic,
invokesuper_no_cme,
invokesuper_no_me,
+ invokesuper_bmethod_zsuper,
invokesuper_not_iseq_or_cfunc,
invokesuper_refinement,
invokesuper_singleton_class,
@@ -518,9 +491,7 @@ make_counters! {
opt_aset_not_array,
opt_aset_not_fixnum,
opt_aset_not_hash,
-
- opt_aref_with_qundef,
- opt_aset_with_qundef,
+ opt_aset_frozen,
opt_case_dispatch_megamorphic,
@@ -531,6 +502,7 @@ make_counters! {
expandarray_postarg,
expandarray_not_array,
expandarray_to_ary,
+ expandarray_method_missing,
expandarray_chain_max_depth,
// getblockparam
@@ -630,6 +602,8 @@ make_counters! {
iseq_stack_too_large,
iseq_too_long,
+ exceptional_entry_escaped_env,
+
temp_reg_opnd,
temp_mem_opnd,
temp_spill,
@@ -792,8 +766,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE {
set_stat_usize!(hash, "context_cache_bytes", crate::core::CTX_ENCODE_CACHE_BYTES + crate::core::CTX_DECODE_CACHE_BYTES);
// VM instructions count
- if rb_vm_insns_count > 0 {
- set_stat_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize);
+ if rb_vm_insn_count > 0 {
+ set_stat_usize!(hash, "vm_insns_count", rb_vm_insn_count as usize);
}
set_stat_usize!(hash, "live_iseq_count", rb_yjit_live_iseq_count as usize);
@@ -838,7 +812,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE {
// For each entry in exit_op_count, add a stats entry with key "exit_INSTRUCTION_NAME"
// and the value is the count of side exits for that instruction.
- for op_idx in 0..VM_INSTRUCTION_SIZE_USIZE {
+ use crate::utils::IntoUsize;
+ for op_idx in 0..rb_vm_instruction_size().as_usize() {
let op_name = insn_name(op_idx);
let key_string = "exit_".to_owned() + &op_name;
let count = EXIT_OP_COUNT[op_idx];
@@ -864,8 +839,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE {
set_stat_double!(hash, "avg_len_in_yjit", avg_len_in_yjit);
// Proportion of instructions that retire in YJIT
- if rb_vm_insns_count > 0 {
- let total_insns_count = retired_in_yjit + rb_vm_insns_count;
+ if rb_vm_insn_count > 0 {
+ let total_insns_count = retired_in_yjit + rb_vm_insn_count;
set_stat_usize!(hash, "total_insns_count", total_insns_count as usize);
let ratio_in_yjit: f64 = 100.0 * retired_in_yjit as f64 / total_insns_count as f64;
@@ -927,7 +902,7 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE {
/// and line samples. Their length should be the same, however the data stored in
/// them is different.
#[no_mangle]
-pub extern "C" fn rb_yjit_record_exit_stack(_exit_pc: *const VALUE)
+pub extern "C" fn rb_yjit_record_exit_stack(exit_pc: *const VALUE)
{
// Return if YJIT is not enabled
if !yjit_enabled_p() {
@@ -951,10 +926,11 @@ pub extern "C" fn rb_yjit_record_exit_stack(_exit_pc: *const VALUE)
// rb_vm_insn_addr2opcode won't work in cargo test --all-features
// because it's a C function. Without insn call, this function is useless
// so wrap the whole thing in a not test check.
+ let _ = exit_pc;
#[cfg(not(test))]
{
// Get the opcode from the encoded insn handler at this PC
- let insn = unsafe { rb_vm_insn_addr2opcode((*_exit_pc).as_ptr()) };
+ let insn = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) };
// Use the same buffer size as Stackprof.
const BUFF_LEN: usize = 2048;
diff --git a/yjit/src/utils.rs b/yjit/src/utils.rs
index 8c4133546d..251628fabf 100644
--- a/yjit/src/utils.rs
+++ b/yjit/src/utils.rs
@@ -92,10 +92,7 @@ pub fn ruby_str_to_rust(v: VALUE) -> String {
let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8;
let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap();
let str_slice: &[u8] = unsafe { slice::from_raw_parts(str_ptr, str_len) };
- match String::from_utf8(str_slice.to_vec()) {
- Ok(utf8) => utf8,
- Err(_) => String::new(),
- }
+ String::from_utf8(str_slice.to_vec()).unwrap_or_default()
}
// Location is the file defining the method, colon, method name.
@@ -163,8 +160,6 @@ pub fn print_int(asm: &mut Assembler, opnd: Opnd) {
}
}
- asm.cpush_all();
-
let argument = match opnd {
Opnd::Mem(_) | Opnd::Reg(_) | Opnd::InsnOut { .. } => {
// Sign-extend the value if necessary
@@ -179,7 +174,6 @@ pub fn print_int(asm: &mut Assembler, opnd: Opnd) {
};
asm.ccall(print_int_fn as *const u8, vec![argument]);
- asm.cpop_all();
}
/// Generate code to print a pointer
@@ -192,9 +186,7 @@ pub fn print_ptr(asm: &mut Assembler, opnd: Opnd) {
assert!(opnd.rm_num_bits() == 64);
- asm.cpush_all();
asm.ccall(print_ptr_fn as *const u8, vec![opnd]);
- asm.cpop_all();
}
/// Generate code to print a value
@@ -207,9 +199,7 @@ pub fn print_value(asm: &mut Assembler, opnd: Opnd) {
assert!(matches!(opnd, Opnd::Value(_)));
- asm.cpush_all();
asm.ccall(print_value_fn as *const u8, vec![opnd]);
- asm.cpop_all();
}
/// Generate code to print constant string to stdout
@@ -224,7 +214,6 @@ pub fn print_str(asm: &mut Assembler, str: &str) {
}
}
- asm.cpush_all();
let string_data = asm.new_label("string_data");
let after_string = asm.new_label("after_string");
@@ -236,8 +225,6 @@ pub fn print_str(asm: &mut Assembler, str: &str) {
let opnd = asm.lea_jump_target(string_data);
asm.ccall(print_str_cfun as *const u8, vec![opnd, Opnd::UImm(str.len() as u64)]);
-
- asm.cpop_all();
}
pub fn stdout_supports_colors() -> bool {
diff --git a/yjit/src/virtualmem.rs b/yjit/src/virtualmem.rs
index aa6d21f210..9126cf300e 100644
--- a/yjit/src/virtualmem.rs
+++ b/yjit/src/virtualmem.rs
@@ -7,6 +7,9 @@ use std::{cell::RefCell, ptr::NonNull};
use crate::{backend::ir::Target, stats::yjit_alloc_size, utils::IntoUsize};
+#[cfg(test)]
+use crate::options::get_option;
+
#[cfg(not(test))]
pub type VirtualMem = VirtualMemory<sys::SystemAllocator>;
@@ -43,7 +46,7 @@ pub struct VirtualMemory<A: Allocator> {
/// Mutable parts of [`VirtualMemory`].
pub struct VirtualMemoryMut<A: Allocator> {
/// Number of bytes that have we have allocated physical memory for starting at
- /// [Self::region_start].
+ /// [VirtualMemory::region_start].
mapped_region_bytes: usize,
/// Keep track of the address of the last written to page.
@@ -316,15 +319,15 @@ mod sys {
impl super::Allocator for SystemAllocator {
fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool {
- unsafe { rb_yjit_mark_writable(ptr as VoidPtr, size) }
+ unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) }
}
fn mark_executable(&mut self, ptr: *const u8, size: u32) {
- unsafe { rb_yjit_mark_executable(ptr as VoidPtr, size) }
+ unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) }
}
fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool {
- unsafe { rb_yjit_mark_unused(ptr as VoidPtr, size) }
+ unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) }
}
}
}
@@ -411,7 +414,7 @@ pub mod tests {
PAGE_SIZE.try_into().unwrap(),
NonNull::new(mem_start as *mut u8).unwrap(),
mem_size,
- 128 * 1024 * 1024,
+ get_option!(mem_size),
)
}
diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs
index 517a0daae5..380544ba33 100644
--- a/yjit/src/yjit.rs
+++ b/yjit/src/yjit.rs
@@ -37,12 +37,17 @@ pub fn yjit_enabled_p() -> bool {
unsafe { rb_yjit_enabled_p }
}
-/// This function is called from C code
+/// Register specialized codegen for builtin C method entries.
+/// Must be called at boot before ruby_init_prelude() since the prelude
+/// could redefine core methods (e.g. Kernel.prepend via bundler).
#[no_mangle]
-pub extern "C" fn rb_yjit_init(yjit_enabled: bool) {
- // Register the method codegen functions. This must be done at boot.
+pub extern "C" fn rb_yjit_init_builtin_cmes() {
yjit_reg_method_codegen_fns();
+}
+/// This function is called from C code
+#[no_mangle]
+pub extern "C" fn rb_yjit_init(yjit_enabled: bool) {
// If --yjit-disable, yjit_init() will not be called until RubyVM::YJIT.enable.
if yjit_enabled {
yjit_init();
@@ -157,6 +162,13 @@ pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exc
return std::ptr::null();
}
+ // In case of exceptional entry, reject escaped environment.
+ // This allows us to use the fact that new frames generally start with an on-stack environment.
+ if jit_exception && unsafe { cfp_env_has_escaped(get_ec_cfp(ec)) } {
+ incr_counter!(exceptional_entry_escaped_env);
+ return std::ptr::null();
+ }
+
// If a custom call threshold was not specified on the command-line and
// this is a large application (has very many ISEQs), switch to
// using the call threshold for large applications after this entry point