YJIT: Fix large ISeq rejection (#7576)

We crashed in some edge cases due to the recent change to not compile
encoded iseqs that are larger than `u16::MAX`.

- Match the C signature of rb_yjit_constant_ic_update() and clamp down
  to `IseqIdx` size
- Return failure instead of panicking with `unwrap()` in codegen when
  the iseq is too large

Co-authored-by: Maxime Chevalier-Boisvert <maxime.chevalierboisvert@shopify.com>
Co-authored-by: Noah Gibbs <noah.gibbs@shopify.com>

2 files changed, 17 insertions, 3 deletions

diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 824415346e..fb760cfd6c 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -781,8 +781,13 @@ pub fn gen_single_block(
     // Instruction sequence to compile
     let iseq = blockid.iseq;
     let iseq_size = unsafe { get_iseq_encoded_size(iseq) };
-    let iseq_size: u16 = iseq_size.try_into().unwrap();
-    let mut insn_idx: u16 = blockid.idx;
+    let iseq_size: IseqIdx = if let Ok(size) = iseq_size.try_into() {
+        size
+    } else {
+        // ISeq too large to compile
+        return Err(());
+    };
+    let mut insn_idx: IseqIdx = blockid.idx;
 
     // Initialize a JIT state object
     let mut jit = JITState::new(blockid, ctx.clone(), cb.get_write_ptr(), ec);
diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs
index 5156ca6a26..c93213b484 100644
--- a/yjit/src/invariants.rs
+++ b/yjit/src/invariants.rs
@@ -389,12 +389,21 @@ pub fn block_assumptions_free(blockref: BlockRef) {
 /// Invalidate the block for the matching opt_getinlinecache so it could regenerate code
 /// using the new value in the constant cache.
 #[no_mangle]
-pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: u16) {
+pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: std::os::raw::c_uint) {
     // If YJIT isn't enabled, do nothing
     if !yjit_enabled_p() {
         return;
     }
 
+    // Try to downcast the iseq index
+    let insn_idx: IseqIdx = if let Ok(idx) = insn_idx.try_into() {
+        idx
+    } else {
+        // The index is too large, YJIT can't possibily have code for it,
+        // so there is nothing to invalidate.
+        return;
+    };
+
     if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_yjit_multi_ractor_p() } {
         // We can't generate code in these situations, so no need to invalidate.
         // See gen_opt_getinlinecache.