diff options
Diffstat (limited to 'yjit/src/invariants.rs')
| -rw-r--r-- | yjit/src/invariants.rs | 101 |
1 files changed, 92 insertions, 9 deletions
diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index e460293440..6639fd677b 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -59,6 +59,11 @@ pub struct Invariants { /// there has been a singleton class for the class after boot, so you cannot /// assume no singleton class going forward. no_singleton_classes: HashMap<VALUE, HashSet<BlockRef>>, + + /// A map from an ISEQ to a set of blocks that assume base pointer is equal + /// to environment pointer. When the set is empty, it means that EP has been + /// escaped in the ISEQ. + no_ep_escape_iseqs: HashMap<IseqPtr, HashSet<BlockRef>>, } /// Private singleton instance of the invariants global struct. @@ -76,6 +81,7 @@ impl Invariants { constant_state_blocks: HashMap::new(), block_constant_states: HashMap::new(), no_singleton_classes: HashMap::new(), + no_ep_escape_iseqs: HashMap::new(), }); } } @@ -154,6 +160,31 @@ pub fn has_singleton_class_of(klass: VALUE) -> bool { .map_or(false, |blocks| blocks.is_empty()) } +/// Track that a block will assume that base pointer is equal to environment pointer. +pub fn track_no_ep_escape_assumption(uninit_block: BlockRef, iseq: IseqPtr) { + Invariants::get_instance() + .no_ep_escape_iseqs + .entry(iseq) + .or_default() + .insert(uninit_block); +} + +/// Returns true if a given ISEQ has previously escaped an environment. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + Invariants::get_instance() + .no_ep_escape_iseqs + .get(&iseq) + .map_or(false, |blocks| blocks.is_empty()) +} + +/// Forget an ISEQ remembered in invariants +pub fn iseq_free_invariants(iseq: IseqPtr) { + if unsafe { INVARIANTS.is_none() } { + return; + } + Invariants::get_instance().no_ep_escape_iseqs.remove(&iseq); +} + // Checks rb_method_basic_definition_p and registers the current block for invalidation if method // lookup changes. // A "basic method" is one defined during VM boot, so we can use this to check assumptions based on @@ -317,7 +348,7 @@ pub extern "C" fn rb_yjit_constant_state_changed(id: ID) { /// Callback for marking GC objects inside [Invariants]. /// See `struct yjijt_root_struct` in C. #[no_mangle] -pub extern "C" fn rb_yjit_root_mark() { +pub extern "C" fn rb_yjit_root_mark(_: *mut c_void) { // Call rb_gc_mark on exit location's raw_samples to // wrap frames in a GC allocated object. This needs to be called // at the same time as root mark. @@ -345,6 +376,23 @@ pub extern "C" fn rb_yjit_root_mark() { } } +#[no_mangle] +pub extern "C" fn rb_yjit_root_update_references(_: *mut c_void) { + if unsafe { INVARIANTS.is_none() } { + return; + } + let no_ep_escape_iseqs = &mut Invariants::get_instance().no_ep_escape_iseqs; + + // Make a copy of the table with updated ISEQ keys + let mut updated_copy = HashMap::with_capacity(no_ep_escape_iseqs.len()); + for (iseq, blocks) in mem::take(no_ep_escape_iseqs) { + let new_iseq = unsafe { rb_gc_location(iseq.into()) }.as_iseq(); + updated_copy.insert(new_iseq, blocks); + } + + *no_ep_escape_iseqs = updated_copy; +} + /// Remove all invariant assumptions made by the block by removing the block as /// as a key in all of the relevant tables. /// For safety, the block has to be initialized and the vm lock must be held. @@ -420,6 +468,10 @@ pub fn block_assumptions_free(blockref: BlockRef) { for (_, blocks) in invariants.no_singleton_classes.iter_mut() { blocks.remove(&blockref); } + // Remove tracking for blocks assuming EP doesn't escape + for (_, blocks) in invariants.no_ep_escape_iseqs.iter_mut() { + blocks.remove(&blockref); + } } /// Callback from the opt_setinlinecache instruction in the interpreter. @@ -497,22 +549,53 @@ pub extern "C" fn rb_yjit_invalidate_no_singleton_class(klass: VALUE) { // We apply this optimization only to Array, Hash, and String for now. if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&klass) } { - let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; - match no_singleton_classes.get_mut(&klass) { + with_vm_lock(src_loc!(), || { + let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; + match no_singleton_classes.get_mut(&klass) { + Some(blocks) => { + // Invalidate existing blocks and let has_singleton_class_of() + // return true when they are compiled again + for block in mem::take(blocks) { + invalidate_block_version(&block); + incr_counter!(invalidate_no_singleton_class); + } + } + None => { + // Let has_singleton_class_of() return true for this class + no_singleton_classes.insert(klass, HashSet::new()); + } + } + }); + } +} + +/// Invalidate blocks for a given ISEQ that assumes environment pointer is +/// equal to base pointer. +#[no_mangle] +pub extern "C" fn rb_yjit_invalidate_ep_is_bp(iseq: IseqPtr) { + // Skip tracking EP escapes on boot. We don't need to invalidate anything during boot. + if unsafe { INVARIANTS.is_none() } { + return; + } + + with_vm_lock(src_loc!(), || { + // If an EP escape for this ISEQ is detected for the first time, invalidate all blocks + // associated to the ISEQ. + let no_ep_escape_iseqs = &mut Invariants::get_instance().no_ep_escape_iseqs; + match no_ep_escape_iseqs.get_mut(&iseq) { Some(blocks) => { - // Invalidate existing blocks and let has_singleton_class_of() - // return true when they are compiled again + // Invalidate existing blocks and make jit.ep_is_bp() return false for block in mem::take(blocks) { invalidate_block_version(&block); - incr_counter!(invalidate_no_singleton_class); + incr_counter!(invalidate_ep_escape); } } None => { - // Let has_singleton_class_of() return true for this class - no_singleton_classes.insert(klass, HashSet::new()); + // Let jit.ep_is_bp() return false for this ISEQ + no_ep_escape_iseqs.insert(iseq, HashSet::new()); } } - } + }); } // Invalidate all generated code and patch C method return code to contain |