Introduce Primitive.attr! to annotate 'inline' (#3242)

[Feature #15589]

7 files changed, 74 insertions, 2 deletions

diff --git a/benchmark/mjit_int_zero_p.yml b/benchmark/mjit_int_zero_p.yml
new file mode 100644
index 0000000000..91e8ea0102
--- /dev/null
+++ b/benchmark/mjit_int_zero_p.yml
@@ -0,0 +1,36 @@
+prelude: |
+  def mjit_zero?(int)
+    int.zero?
+  end
+
+  def mjit_eq_0(int)
+    int == 0
+  end
+
+  def warmup(sym, int)
+    if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled?
+      jit_min_calls = 10000
+      i = 0
+      while i < jit_min_calls
+        send(sym, int)
+        i += 1
+      end
+      RubyVM::MJIT.pause
+    end
+  end
+
+benchmark:
+  - name: 0.zero?
+    prelude: warmup(:mjit_zero?, 0)
+    script: mjit_zero?(0)
+  - name: 1.zero?
+    prelude: warmup(:mjit_zero?, 1)
+    script: mjit_zero?(1)
+  - name: 0 == 0
+    prelude: warmup(:mjit_eq_0, 0)
+    script: mjit_eq_0(0)
+  - name: 1 == 0
+    prelude: warmup(:mjit_eq_0, 1)
+    script: mjit_eq_0(1)
+
+loop_count: 40000000
diff --git a/compile.c b/compile.c
index 972a2997ff..77af3162fe 100644
--- a/compile.c
+++ b/compile.c
@@ -7274,6 +7274,11 @@ compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, co
                     GET_VM()->builtin_inline_index++;
                     return COMPILE_OK;
                 }
+                else if (strcmp("attr!", builtin_func) == 0) {
+                    // There's only "inline" attribute for now
+                    iseq->body->builtin_inline_p = true;
+                    return COMPILE_OK;
+                }
 
                 if (1) {
                     rb_bug("can't find builtin function:%s", builtin_func);
@@ -10815,6 +10820,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq)
     ibf_dump_write_small_value(dump, body->ci_size);
     ibf_dump_write_small_value(dump, body->stack_max);
     ibf_dump_write_small_value(dump, body->catch_except_p);
+    ibf_dump_write_small_value(dump, body->builtin_inline_p);
 
 #undef IBF_BODY_OFFSET
 
@@ -10920,6 +10926,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
     const unsigned int ci_size = (unsigned int)ibf_load_small_value(load, &reading_pos);
     const unsigned int stack_max = (unsigned int)ibf_load_small_value(load, &reading_pos);
     const char catch_except_p = (char)ibf_load_small_value(load, &reading_pos);
+    const bool builtin_inline_p = (bool)ibf_load_small_value(load, &reading_pos);
 
 #undef IBF_BODY_OFFSET
 
@@ -10958,6 +10965,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
     load_body->location.code_location.end_pos.lineno = location_code_location_end_pos_lineno;
     load_body->location.code_location.end_pos.column = location_code_location_end_pos_column;
     load_body->catch_except_p = catch_except_p;
+    load_body->builtin_inline_p = builtin_inline_p;
 
     load_body->is_entries           = ZALLOC_N(union iseq_inline_storage_entry, is_size);
                                       ibf_load_ci_entries(load, ci_entries_offset, ci_size, &load_body->call_data);
diff --git a/integer.rb b/integer.rb
index b1e288d086..cc2e58948f 100644
--- a/integer.rb
+++ b/integer.rb
@@ -4,6 +4,7 @@ class Integer
   #
   # Returns +true+ if +num+ has a zero value.
   def zero?
+    Primitive.attr! 'inline'
     Primitive.cexpr! 'int_zero_p(self)'
   end
 end
diff --git a/mjit_compile.c b/mjit_compile.c
index fe5c2413e4..c4c31aa251 100644
--- a/mjit_compile.c
+++ b/mjit_compile.c
@@ -374,7 +374,12 @@ inlinable_iseq_p(const struct rb_iseq_constant_body *body)
         //   * Do not require `cfp->sp` motion
         //   * Do not move `cfp->pc`
         //   * Do not read any `cfp->pc`
-        if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->iseq_encoded + (pos + 1)))
+        if (insn == BIN(invokebuiltin) || insn == BIN(opt_invokebuiltin_delegate) || insn == BIN(opt_invokebuiltin_delegate_leave)) {
+            // builtin insn's inlinability is handled by `Primitive.attr! 'inline'` per iseq
+            if (!body->builtin_inline_p)
+                return false;
+        }
+        else if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->iseq_encoded + (pos + 1)))
             return false;
         // At this moment, `cfp->ep` in an inlined method is not working.
         switch (insn) {
diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb
index 01551592bf..9d4b6405a1 100644
--- a/tool/mk_builtin_loader.rb
+++ b/tool/mk_builtin_loader.rb
@@ -135,6 +135,12 @@ def collect_builtin base, tree, name, bs, inlines, params = nil
 
         if /(.+)\!\z/ =~ func_name
           case $1
+          when 'attr'
+            text = inline_text(argc, args.first)
+            if text != 'inline'
+              raise "Only 'inline' is allowed to be annotated (but got: '#{text}')"
+            end
+            break
           when 'cstmt'
             text = inline_text argc, args.first
 
diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb
index 0d3678dfb5..fa2e52e480 100644
--- a/tool/ruby_vm/views/mjit_compile.inc.erb
+++ b/tool/ruby_vm/views/mjit_compile.inc.erb
@@ -63,12 +63,26 @@ switch (insn) {
       }
 %   when 'getinstancevariable', 'setinstancevariable'
 <%=   render 'mjit_compile_ivar', locals: { insn: insn } -%>
-%   when 'leave'
+%   when 'leave', 'opt_invokebuiltin_delegate_leave'
+    {
+%     # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining.
+%     if insn.name == 'opt_invokebuiltin_delegate_leave'
+      RB_BUILTIN bf = (RB_BUILTIN)operands[0];
+      rb_num_t index = (rb_num_t)operands[0];
+      fprintf(f, "{\n");
+      fprintf(f, "    VALUE val;\n");
+      fprintf(f, "    RB_BUILTIN bf = (RB_BUILTIN)0x%"PRIxVALUE";\n", operands[0]);
+      fprintf(f, "    rb_num_t index = (rb_num_t)0x%"PRIxVALUE";\n", operands[1]);
+      fprintf(f, <%= rstring2cstr(insn.expr.expr.lines.find { |l| l =~ / vm_invoke_builtin_delegate\(/ }).gsub("\n", '\n') %>);
+      fprintf(f, "    stack[0] = val;\n");
+      fprintf(f, "}\n");
+%     else
       if (b->stack_size != 1) {
           if (mjit_opts.warnings || mjit_opts.verbose)
               fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size);
           status->success = false;
       }
+%     end
 %     # Skip vm_pop_frame for inlined call
       if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined
 %         # Cancel on interrupts to make leave insn leaf
@@ -84,6 +98,7 @@ switch (insn) {
       b->stack_size += <%= insn.call_attribute('sp_inc') %>;
       b->finish_p = TRUE;
       break;
+    }
 %   end
 %
 %   # Main insn implementation generated by insns.def
diff --git a/vm_core.h b/vm_core.h
index 4f122cacb8..fd49c243e3 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -418,6 +418,7 @@ struct rb_iseq_constant_body {
     unsigned int stack_max; /* for stack overflow check */
 
     char catch_except_p; /* If a frame of this ISeq may catch exception, set TRUE */
+    bool builtin_inline_p; // This ISeq's builtin func is safe to be inlined by MJIT
 
 #if USE_MJIT
     /* The following fields are MJIT related info.  */