diff options
88 files changed, 3465 insertions, 1017 deletions
diff --git a/.github/actions/setup/directories/action.yml b/.github/actions/setup/directories/action.yml index 0e8ffd59ef..99d1fc0151 100644 --- a/.github/actions/setup/directories/action.yml +++ b/.github/actions/setup/directories/action.yml @@ -95,7 +95,7 @@ runs: git config --global init.defaultBranch garbage - if: inputs.checkout - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: ${{ inputs.srcdir }} fetch-depth: ${{ inputs.fetch-depth }} diff --git a/.github/labeler.yml b/.github/labeler.yml index e81aed8e98..f39fcec386 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -4,3 +4,4 @@ Documentation: Backport: - base-branch: 'ruby_3_\d' +- base-branch: 'ruby_4_\d' diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml index 899d601aef..dd4c274ddb 100644 --- a/.github/workflows/annocheck.yml +++ b/.github/workflows/annocheck.yml @@ -61,7 +61,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -72,7 +72,7 @@ jobs: builddir: build makeup: true - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/auto_review_pr.yml b/.github/workflows/auto_review_pr.yml index ad0e63ba12..bd9bc9766e 100644 --- a/.github/workflows/auto_review_pr.yml +++ b/.github/workflows/auto_review_pr.yml @@ -19,9 +19,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6.0.1 + uses: actions/checkout@v6.0.2 - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.4' bundler: none diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml index d3e734f885..ab0d9f1f57 100644 --- a/.github/workflows/baseruby.yml +++ b/.github/workflows/baseruby.yml @@ -48,12 +48,12 @@ jobs: - ruby-3.3 steps: - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: ${{ matrix.ruby }} bundler: none - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup/ubuntu diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml index 59f64e8312..548db433fb 100644 --- a/.github/workflows/bundled_gems.yml +++ b/.github/workflows/bundled_gems.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml index c5dec65e48..1ba1b0388d 100644 --- a/.github/workflows/check_dependencies.yml +++ b/.github/workflows/check_dependencies.yml @@ -30,7 +30,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup/ubuntu if: ${{ contains(matrix.os, 'ubuntu') }} @@ -40,7 +40,7 @@ jobs: - uses: ./.github/actions/setup/directories - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/check_misc.yml b/.github/workflows/check_misc.yml index 2a2bd1df53..35bad2724e 100644 --- a/.github/workflows/check_misc.yml +++ b/.github/workflows/check_misc.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} @@ -77,7 +77,7 @@ jobs: echo RDOC='ruby -W0 --disable-gems tool/rdoc-srcdir -q' >> $GITHUB_ENV - name: Checkout rdoc - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ruby/rdoc ref: ${{ steps.rdoc.outputs.ref }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a92c93b476..69f38f3f48 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -58,7 +58,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install libraries if: ${{ contains(matrix.os, 'macos') }} diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 8c0ca54e0b..338a7d5248 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -51,7 +51,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } # Set fetch-depth: 10 so that Launchable can receive commits information. - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } @@ -74,7 +74,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - name: 'GCC 15 LTO' @@ -104,7 +104,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'clang 22', with: { tag: 'clang-22' }, timeout-minutes: 5 } @@ -125,7 +125,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'clang 13', with: { tag: 'clang-13' }, timeout-minutes: 5 } @@ -146,7 +146,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } # -Wno-strict-prototypes is necessary with current clang-15 since @@ -172,7 +172,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'C++20', with: { CXXFLAGS: '-std=c++20 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' }, timeout-minutes: 5 } @@ -192,7 +192,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'disable-jit', with: { append_configure: '--disable-yjit --disable-zjit' }, timeout-minutes: 5 } @@ -214,7 +214,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'NDEBUG', with: { cppflags: '-DNDEBUG' }, timeout-minutes: 5 } @@ -234,7 +234,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'HASH_DEBUG', with: { cppflags: '-DHASH_DEBUG' }, timeout-minutes: 5 } @@ -254,7 +254,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'USE_LAZY_LOAD', with: { cppflags: '-DUSE_LAZY_LOAD' }, timeout-minutes: 5 } @@ -274,7 +274,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'GC_DEBUG_STRESS_TO_CLASS', with: { cppflags: '-DGC_DEBUG_STRESS_TO_CLASS' }, timeout-minutes: 5 } @@ -293,7 +293,7 @@ jobs: timeout-minutes: 60 services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'VM_DEBUG_BP_CHECK', with: { cppflags: '-DVM_DEBUG_BP_CHECK' }, timeout-minutes: 5 } @@ -319,7 +319,7 @@ jobs: - 'compileB' - 'compileC' steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - uses: ./.github/actions/slack with: diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml index ac73991fe8..d5e6aae8bc 100644 --- a/.github/workflows/cygwin.yml +++ b/.github/workflows/cygwin.yml @@ -40,7 +40,7 @@ jobs: steps: - run: git config --global core.autocrlf input - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Cygwin uses: cygwin/cygwin-install-action@master diff --git a/.github/workflows/default_gems_list.yml b/.github/workflows/default_gems_list.yml index 1c7e2195c8..f6f8d820e5 100644 --- a/.github/workflows/default_gems_list.yml +++ b/.github/workflows/default_gems_list.yml @@ -23,7 +23,7 @@ jobs: if: ${{ github.repository == 'ruby/ruby' }} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 29adcab39a..55e4eecd94 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -61,7 +61,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 5c639ad48b..f2e3df51cb 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -166,7 +166,7 @@ jobs: [ ${#failed[@]} -eq 0 ] shell: sh - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/modgc.yml b/.github/workflows/modgc.yml index 1d14934df8..a00d878ba5 100644 --- a/.github/workflows/modgc.yml +++ b/.github/workflows/modgc.yml @@ -48,7 +48,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -61,7 +61,7 @@ jobs: uses: ./.github/actions/setup/ubuntu if: ${{ contains(matrix.os, 'ubuntu') }} - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/parse_y.yml b/.github/workflows/parse_y.yml index 87facc8a55..f87cc71dd8 100644 --- a/.github/workflows/parse_y.yml +++ b/.github/workflows/parse_y.yml @@ -51,14 +51,14 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github - uses: ./.github/actions/setup/ubuntu - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/post_push.yml b/.github/workflows/post_push.yml index 318444c0a2..95ac8b202f 100644 --- a/.github/workflows/post_push.yml +++ b/.github/workflows/post_push.yml @@ -28,7 +28,7 @@ jobs: REDMINE_SYS_API_KEY: ${{ secrets.REDMINE_SYS_API_KEY }} if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/ruby_') }} - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 500 # for notify-slack-commits token: ${{ secrets.MATZBOT_AUTO_UPDATE_TOKEN }} diff --git a/.github/workflows/pr-playground.yml b/.github/workflows/pr-playground.yml index f3c0556429..2391656277 100644 --- a/.github/workflows/pr-playground.yml +++ b/.github/workflows/pr-playground.yml @@ -25,7 +25,7 @@ jobs: && github.event.workflow_run.event == 'pull_request') }} steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3caeee9a3b..fa4af8c824 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: ruby/setup-ruby@v1 with: diff --git a/.github/workflows/rust-warnings.yml b/.github/workflows/rust-warnings.yml index a2e3208e52..df65245ca1 100644 --- a/.github/workflows/rust-warnings.yml +++ b/.github/workflows/rust-warnings.yml @@ -36,7 +36,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust run: rustup default beta diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index c607098997..b154476cb5 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -34,7 +34,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false diff --git a/.github/workflows/spec_guards.yml b/.github/workflows/spec_guards.yml index cf4661555c..856d6f61eb 100644 --- a/.github/workflows/spec_guards.yml +++ b/.github/workflows/spec_guards.yml @@ -46,9 +46,9 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: ${{ matrix.ruby }} bundler: none diff --git a/.github/workflows/sync_default_gems.yml b/.github/workflows/sync_default_gems.yml index 9ff97d5a4e..219c6aef83 100644 --- a/.github/workflows/sync_default_gems.yml +++ b/.github/workflows/sync_default_gems.yml @@ -31,12 +31,12 @@ jobs: if: ${{ github.repository == 'ruby/ruby' }} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 name: Check out ruby/ruby with: token: ${{ github.repository == 'ruby/ruby' && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.4' bundler: none diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 88c19b6fe6..2ed27e278b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -60,7 +60,7 @@ jobs: )}} steps: &make-steps - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -69,7 +69,7 @@ jobs: with: arch: ${{ matrix.arch }} - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none @@ -221,7 +221,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup/ubuntu @@ -237,7 +237,7 @@ jobs: - run: make install - name: Checkout ruby-bench - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ruby/ruby-bench path: ruby-bench diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 0d2a6f0545..2242f13e69 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -59,7 +59,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -98,7 +98,7 @@ jobs: run: | echo "WASI_SDK_PATH=/opt/wasi-sdk" >> $GITHUB_ENV - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1d44a5482c..f997ed56d5 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -59,14 +59,14 @@ jobs: - run: md build working-directory: - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: # windows-11-arm has only 3.4.1, 3.4.2, 3.4.3, head ruby-version: ${{ !endsWith(matrix.os, 'arm') && '3.1' || '3.4' }} bundler: none windows-toolchain: none - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml index a59b4d6508..d7509c8bf5 100644 --- a/.github/workflows/yjit-macos.yml +++ b/.github/workflows/yjit-macos.yml @@ -41,7 +41,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: RUST_BACKTRACE=1 cargo test working-directory: yjit @@ -83,7 +83,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 150f0b3275..d79468a412 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -36,7 +36,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # For now we can't run cargo test --offline because it complains about the # capstone dependency, even though the dependency is optional @@ -68,7 +68,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Check that we don't have linting errors in release mode, too - run: cargo clippy --all-targets --all-features @@ -121,14 +121,14 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github - uses: ./.github/actions/setup/ubuntu - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index a638907811..aa3c08b9d8 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -68,7 +68,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -172,7 +172,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup/macos @@ -192,7 +192,7 @@ jobs: run: echo "MAKEFLAGS=" >> "$GITHUB_ENV" - name: Checkout ruby-bench - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ruby/ruby-bench path: ruby-bench diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index 28bfec963e..f79c1c5b1d 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -41,7 +41,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: cargo clippy --all-targets --all-features working-directory: zjit @@ -104,14 +104,14 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout-cone-mode: false sparse-checkout: /.github - uses: ./.github/actions/setup/ubuntu - - uses: ruby/setup-ruby@b90be12699fdfcbee4440c2bba85f6f460446bb0 # v1.279.0 + - uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0 with: ruby-version: '3.1' bundler: none @@ -229,7 +229,7 @@ jobs: )}} steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup/ubuntu @@ -245,7 +245,7 @@ jobs: - run: make install - name: Checkout ruby-bench - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ruby/ruby-bench path: ruby-bench @@ -49,6 +49,7 @@ releases. * prism 1.8.0 * stringio 3.2.1.dev * strscan 3.1.7.dev +* syntax_suggest 2.0.3 ### The following bundled gems are updated. diff --git a/benchmark/file_basename.yml b/benchmark/file_basename.yml new file mode 100644 index 0000000000..fbd78785aa --- /dev/null +++ b/benchmark/file_basename.yml @@ -0,0 +1,6 @@ +prelude: | + # frozen_string_literal: true +benchmark: + long: File.basename("/Users/george/src/github.com/ruby/ruby/benchmark/file_dirname.yml") + long_name: File.basename("Users_george_src_github.com_ruby_ruby_benchmark_file_dirname.yml") + withext: File.basename("/Users/george/src/github.com/ruby/ruby/benchmark/file_dirname.yml", ".yml") diff --git a/benchmark/file_dirname.yml b/benchmark/file_dirname.yml new file mode 100644 index 0000000000..43a81c9371 --- /dev/null +++ b/benchmark/file_dirname.yml @@ -0,0 +1,6 @@ +prelude: | + # frozen_string_literal: true +benchmark: + long: File.dirname("/Users/george/src/github.com/ruby/ruby/benchmark/file_dirname.yml") + short: File.dirname("foo/bar") + n_4: File.dirname("/Users/george/src/github.com/ruby/ruby/benchmark/file_dirname.yml", 4) diff --git a/benchmark/file_extname.yml b/benchmark/file_extname.yml new file mode 100644 index 0000000000..fb16e55840 --- /dev/null +++ b/benchmark/file_extname.yml @@ -0,0 +1,6 @@ +prelude: | + # frozen_string_literal: true +benchmark: + long: File.extname("/Users/george/src/github.com/ruby/ruby/benchmark/file_dirname.yml") + long_name: File.extname("Users_george_src_github.com_ruby_ruby_benchmark_file_dirname.yml") + short: File.extname("foo/bar") diff --git a/doc/jit/zjit.md b/doc/jit/zjit.md index 38124cb737..a284fce811 100644 --- a/doc/jit/zjit.md +++ b/doc/jit/zjit.md @@ -1,3 +1,7 @@ +<p align="center"> + <img src="https://github.com/user-attachments/assets/27abfe03-3e96-4220-b6f1-278bb0c87684" width="400"> +</p> + # ZJIT: ADVANCED RUBY JIT PROTOTYPE ZJIT is a method-based just-in-time (JIT) compiler for Ruby. It uses profile diff --git a/doc/maintainers.md b/doc/maintainers.md index 46840343ca..d1855cdbcb 100644 --- a/doc/maintainers.md +++ b/doc/maintainers.md @@ -50,6 +50,10 @@ consensus on ruby-core/ruby-dev. * *No maintainer* +#### lib/pathname.rb + +* Tanaka Akira ([akr]) + #### lib/rubygems.rb, lib/rubygems/* * Hiroshi SHIBATA ([hsbt]) @@ -350,12 +354,6 @@ consensus on ruby-core/ruby-dev. * https://github.com/ruby/openssl * https://rubygems.org/gems/openssl -#### ext/pathname - -* Tanaka Akira ([akr]) -* https://github.com/ruby/pathname -* https://rubygems.org/gems/pathname - #### ext/psych * Aaron Patterson ([tenderlove]) diff --git a/doc/standard_library.md b/doc/standard_library.md index 7a477283a9..4057031b00 100644 --- a/doc/standard_library.md +++ b/doc/standard_library.md @@ -11,6 +11,7 @@ of each. - `MakeMakefile`: A module used to generate a Makefile for C extensions - `RbConfig`: Information about your Ruby configuration and build - `Gem`: A package management framework for Ruby +- `Pathname`: Representation of the name of a file or directory on the filesystem. Pathname is a core class, but only methods that depend on other libraries are provided as a library. ## Extensions @@ -74,7 +75,6 @@ of each. - IO#wait ([GitHub][io-wait]): Provides the feature for waiting until IO is readable or writable without blocking. - JSON ([GitHub][json]): Implements JavaScript Object Notation for Ruby - OpenSSL ([GitHub][openssl]): Provides SSL, TLS, and general-purpose cryptography for Ruby -- Pathname ([GitHub][pathname]): Representation of the name of a file or directory on the filesystem - Psych ([GitHub][psych]): A YAML parser and emitter for Ruby - StringIO ([GitHub][stringio]): Pseudo-I/O on String objects - StringScanner ([GitHub][strscan]): Provides lexical scanning operations on a String diff --git a/ext/coverage/coverage.c b/ext/coverage/coverage.c index 1fda8191cc..747f065fba 100644 --- a/ext/coverage/coverage.c +++ b/ext/coverage/coverage.c @@ -52,17 +52,29 @@ rb_coverage_supported(VALUE self, VALUE _mode) /* * call-seq: - * Coverage.setup => nil - * Coverage.setup(:all) => nil - * Coverage.setup(lines: bool, branches: bool, methods: bool, eval: bool) => nil - * Coverage.setup(oneshot_lines: true) => nil + * Coverage.setup -> nil + * Coverage.setup(type) -> nil + * Coverage.setup(lines: false, branches: false, methods: false, eval: false, oneshot_lines: false) -> nil * - * Set up the coverage measurement. + * Performs setup for coverage measurement, but does not start coverage measurement. + * To start coverage measurement, use Coverage.resume. * - * Note that this method does not start the measurement itself. - * Use Coverage.resume to start the measurement. + * To perform both setup and start coverage measurement, Coverage.start can be used. * - * You may want to use Coverage.start to setup and then start the measurement. + * With argument +type+ given and +type+ is symbol +:all+, enables all types of coverage + * (lines, branches, methods, and eval). + * + * Keyword arguments or hash +type+ can be given with each of the following keys: + * + * - +lines+: Enables line coverage that records the number of times each line was executed. + * If +lines+ is enabled, +oneshot_lines+ cannot be enabled. + * See {Lines Coverage}[rdoc-ref:Coverage@Lines+Coverage]. + * - +branches+: Enables branch coverage that records the number of times each + * branch in each conditional was executed. See {Branches Coverage}[rdoc-ref:Coverage@Branch+Coverage]. + * - +methods+: Enables method coverage that records the number of times each method was exectued. + * See {Methods Coverage}[rdoc-ref:Coverage@Methods+Coverage]. + * - +eval+: Enables coverage for evaluations (e.g. Kernel#eval, Module#class_eval). + * See {Eval Coverage}[rdoc-ref:Coverage@Eval+Coverage]. */ static VALUE rb_coverage_setup(int argc, VALUE *argv, VALUE klass) @@ -600,6 +612,62 @@ rb_coverage_running(VALUE klass) * 5. The ending line number the method appears on in the file. * 6. The ending column number the method appears on in the file. * + * == Eval \Coverage + * + * Eval coverage can be combined with the coverage types above to track + * coverage for eval. + * + * require "coverage" + * Coverage.start(eval: true, lines: true) + * + * eval(<<~RUBY, nil, "eval 1") + * ary = [] + * 10.times do |i| + * ary << "hello" * i + * end + * RUBY + * + * Coverage.result # => {"eval 1" => {lines: [1, 1, 10, nil]}} + * + * Note that the eval must have a filename assigned, otherwise coverage + * will not be measured. + * + * require "coverage" + * Coverage.start(eval: true, lines: true) + * + * eval(<<~RUBY) + * ary = [] + * 10.times do |i| + * ary << "hello" * i + * end + * RUBY + * + * Coverage.result # => {"(eval)" => {lines: [nil, nil, nil, nil]}} + * + * Also note that if a line number is assigned to the eval and it is not 1, + * then the resulting coverage will be padded with +nil+ if the line number is + * greater than 1, and truncated if the line number is less than 1. + * + * require "coverage" + * Coverage.start(eval: true, lines: true) + * + * eval(<<~RUBY, nil, "eval 1", 3) + * ary = [] + * 10.times do |i| + * ary << "hello" * i + * end + * RUBY + * + * eval(<<~RUBY, nil, "eval 2", -1) + * ary = [] + * 10.times do |i| + * ary << "hello" * i + * end + * RUBY + * + * Coverage.result + * # => {"eval 1" => {lines: [nil, nil, 1, 1, 10, nil]}, "eval 2" => {lines: [10, nil]}} + * * == All \Coverage Modes * * You can also run all modes of coverage simultaneously with this shortcut. diff --git a/ext/extmk.rb b/ext/extmk.rb index 8f847f4f3a..578e1cfa01 100755 --- a/ext/extmk.rb +++ b/ext/extmk.rb @@ -592,6 +592,7 @@ gem = #{@gemname} build_complete = $(TARGET_GEM_DIR)/gem.build_complete install-so: build_complete clean-so:: clean-build_complete +$(build_complete) $(OBJS): $(TARGET_SO_DIR_TIMESTAMP) build_complete: $(build_complete) $(build_complete): $(TARGET_SO) diff --git a/ext/json/lib/json.rb b/ext/json/lib/json.rb index f619d93252..2f6db44227 100644 --- a/ext/json/lib/json.rb +++ b/ext/json/lib/json.rb @@ -6,6 +6,15 @@ require 'json/common' # # \JSON is a lightweight data-interchange format. # +# \JSON is easy for us humans to read and write, +# and equally simple for machines to read (parse) and write (generate). +# +# \JSON is language-independent, making it an ideal interchange format +# for applications in differing programming languages +# and on differing operating systems. +# +# == \JSON Values +# # A \JSON value is one of the following: # - Double-quoted text: <tt>"foo"</tt>. # - Number: +1+, +1.0+, +2.0e2+. diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c index 457ffc2789..1143e4801d 100644 --- a/ext/objspace/objspace.c +++ b/ext/objspace/objspace.c @@ -137,6 +137,7 @@ memsize_of_all_m(int argc, VALUE *argv, VALUE self) if (argc > 0) { rb_scan_args(argc, argv, "01", &data.klass); + if (!NIL_P(data.klass)) rb_obj_is_kind_of(Qnil, data.klass); } each_object_with_flags(total_i, &data); @@ -214,15 +214,16 @@ file_path_convert(VALUE name) return name; } -static rb_encoding * +static void check_path_encoding(VALUE str) { - rb_encoding *enc = rb_enc_get(str); - if (!rb_enc_asciicompat(enc)) { - rb_raise(rb_eEncCompatError, "path name must be ASCII-compatible (%s): %"PRIsVALUE, - rb_enc_name(enc), rb_str_inspect(str)); + if (RB_UNLIKELY(!rb_str_enc_fastpath(str))) { + rb_encoding *enc = rb_str_enc_get(str); + if (!rb_enc_asciicompat(enc)) { + rb_raise(rb_eEncCompatError, "path name must be ASCII-compatible (%s): %"PRIsVALUE, + rb_enc_name(enc), rb_str_inspect(str)); + } } - return enc; } VALUE @@ -250,7 +251,7 @@ rb_get_path_check_convert(VALUE obj) rb_raise(rb_eArgError, "path name contains null byte"); } - return rb_str_new4(obj); + return rb_str_new_frozen(obj); } VALUE @@ -265,6 +266,19 @@ rb_get_path(VALUE obj) return rb_get_path_check_convert(rb_get_path_check_to_string(obj)); } +static inline VALUE +check_path(VALUE obj, const char **cstr) +{ + VALUE str = rb_get_path_check_convert(rb_get_path_check_to_string(obj)); +#if RUBY_DEBUG + str = rb_str_new_frozen(str); +#endif + *cstr = RSTRING_PTR(str); + return str; +} + +#define CheckPath(str, cstr) RB_GC_GUARD(str) = check_path(str, &cstr); + VALUE rb_str_encode_ospath(VALUE path) { @@ -3557,8 +3571,8 @@ static const char file_alt_separator[] = {FILE_ALT_SEPARATOR, '\0'}; # define isADS(x) 0 #endif -#define Next(p, e, enc) ((p) + rb_enc_mbclen((p), (e), (enc))) -#define Inc(p, e, enc) ((p) = Next((p), (e), (enc))) +#define Next(p, e, mb_enc, enc) ((p) + ((mb_enc) ? rb_enc_mbclen((p), (e), (enc)) : 1)) +#define Inc(p, e, mb_enc, enc) ((p) = Next((p), (e), (mb_enc), (enc))) #if defined(DOSISH_UNC) #define has_unc(buf) (isdirsep((buf)[0]) && isdirsep((buf)[1])) @@ -3622,7 +3636,7 @@ not_same_drive(VALUE path, int drive) #endif /* DOSISH_DRIVE_LETTER */ static inline char * -skiproot(const char *path, const char *end, rb_encoding *enc) +skiproot(const char *path, const char *end) { #ifdef DOSISH_DRIVE_LETTER if (path + 2 <= end && has_drive_letter(path)) path += 2; @@ -3631,31 +3645,37 @@ skiproot(const char *path, const char *end, rb_encoding *enc) return (char *)path; } -#define nextdirsep rb_enc_path_next -char * -rb_enc_path_next(const char *s, const char *e, rb_encoding *enc) +static inline char * +enc_path_next(const char *s, const char *e, bool mb_enc, rb_encoding *enc) { while (s < e && !isdirsep(*s)) { - Inc(s, e, enc); + Inc(s, e, mb_enc, enc); } return (char *)s; } +#define nextdirsep rb_enc_path_next +char * +rb_enc_path_next(const char *s, const char *e, rb_encoding *enc) +{ + return enc_path_next(s, e, true, enc); +} + #if defined(DOSISH_UNC) || defined(DOSISH_DRIVE_LETTER) -#define skipprefix rb_enc_path_skip_prefix +#define skipprefix enc_path_skip_prefix #else -#define skipprefix(path, end, enc) (path) +#define skipprefix(path, end, mb_enc, enc) (path) #endif -char * -rb_enc_path_skip_prefix(const char *path, const char *end, rb_encoding *enc) +static inline char * +enc_path_skip_prefix(const char *path, const char *end, bool mb_enc, rb_encoding *enc) { #if defined(DOSISH_UNC) || defined(DOSISH_DRIVE_LETTER) #ifdef DOSISH_UNC if (path + 2 <= end && isdirsep(path[0]) && isdirsep(path[1])) { path += 2; while (path < end && isdirsep(*path)) path++; - if ((path = rb_enc_path_next(path, end, enc)) < end && path[0] && path[1] && !isdirsep(path[1])) - path = rb_enc_path_next(path + 1, end, enc); + if ((path = enc_path_next(path, end, mb_enc, enc)) < end && path[0] && path[1] && !isdirsep(path[1])) + path = enc_path_next(path + 1, end, mb_enc, enc); return (char *)path; } #endif @@ -3667,19 +3687,24 @@ rb_enc_path_skip_prefix(const char *path, const char *end, rb_encoding *enc) return (char *)path; } +char * +rb_enc_path_skip_prefix(const char *path, const char *end, rb_encoding *enc) +{ + return enc_path_skip_prefix(path, end, true, enc); +} + static inline char * skipprefixroot(const char *path, const char *end, rb_encoding *enc) { #if defined(DOSISH_UNC) || defined(DOSISH_DRIVE_LETTER) - char *p = skipprefix(path, end, enc); + char *p = skipprefix(path, end, true, enc); while (isdirsep(*p)) p++; return p; #else - return skiproot(path, end, enc); + return skiproot(path, end); #endif } -#define strrdirsep rb_enc_path_last_separator char * rb_enc_path_last_separator(const char *path, const char *end, rb_encoding *enc) { @@ -3692,14 +3717,39 @@ rb_enc_path_last_separator(const char *path, const char *end, rb_encoding *enc) last = (char *)tmp; } else { - Inc(path, end, enc); + Inc(path, end, true, enc); } } return last; } +static inline char * +strrdirsep(const char *path, const char *end, bool mb_enc, rb_encoding *enc) +{ + if (RB_UNLIKELY(mb_enc)) { + return rb_enc_path_last_separator(path, end, enc); + } + + const char *cursor = end - 1; + + while (isdirsep(cursor[0])) { + cursor--; + } + + while (cursor >= path) { + if (isdirsep(cursor[0])) { + while (cursor > path && isdirsep(cursor[-1])) { + cursor--; + } + return (char *)cursor; + } + cursor--; + } + return NULL; +} + static char * -chompdirsep(const char *path, const char *end, rb_encoding *enc) +chompdirsep(const char *path, const char *end, bool mb_enc, rb_encoding *enc) { while (path < end) { if (isdirsep(*path)) { @@ -3708,7 +3758,7 @@ chompdirsep(const char *path, const char *end, rb_encoding *enc) if (path >= end) return (char *)last; } else { - Inc(path, end, enc); + Inc(path, end, mb_enc, enc); } } return (char *)path; @@ -3718,7 +3768,7 @@ char * rb_enc_path_end(const char *path, const char *end, rb_encoding *enc) { if (path < end && isdirsep(*path)) path++; - return chompdirsep(path, end, enc); + return chompdirsep(path, end, true, enc); } static rb_encoding * @@ -3753,7 +3803,7 @@ ntfs_tail(const char *path, const char *end, rb_encoding *enc) if (isADS(*path)) path++; } else { - Inc(path, end, enc); + Inc(path, end, true, enc); } } return (char *)path; @@ -3815,7 +3865,7 @@ copy_home_path(VALUE result, const char *dir) rb_enc_associate_index(result, encidx); #if defined DOSISH || defined __CYGWIN__ enc = rb_enc_from_index(encidx); - for (bend = (p = buf) + dirlen; p < bend; Inc(p, bend, enc)) { + for (bend = (p = buf) + dirlen; p < bend; Inc(p, bend, true, enc)) { if (*p == '\\') { *p = '/'; } @@ -4038,7 +4088,7 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na rb_enc_associate(result, enc = fs_enc_check(result, fname)); p = pend; } - p = chompdirsep(skiproot(buf, p, enc), p, enc); + p = chompdirsep(skiproot(buf, p), p, true, enc); s += 2; } } @@ -4059,11 +4109,11 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na if (isdirsep(*s)) { /* specified full path, but not drive letter nor UNC */ /* we need to get the drive letter or UNC share name */ - p = skipprefix(buf, p, enc); + p = skipprefix(buf, p, true, enc); } else #endif /* defined DOSISH || defined __CYGWIN__ */ - p = chompdirsep(skiproot(buf, p, enc), p, enc); + p = chompdirsep(skiproot(buf, p), p, true, enc); } else { size_t len; @@ -4087,7 +4137,7 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na rb_str_set_len(result, p-buf+1); BUFCHECK(bdiff + 1 >= buflen); p[1] = 0; - root = skipprefix(buf, p+1, enc); + root = skipprefix(buf, p+1, true, enc); b = s; while (*s) { @@ -4103,7 +4153,7 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na /* We must go back to the parent */ char *n; *p = '\0'; - if (!(n = strrdirsep(root, p, enc))) { + if (!(n = strrdirsep(root, p, true, enc))) { *p = '/'; } else { @@ -4166,7 +4216,7 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na } } #endif /* __APPLE__ */ - Inc(s, fend, enc); + Inc(s, fend, true, enc); break; } } @@ -4194,7 +4244,7 @@ rb_file_expand_path_internal(VALUE fname, VALUE dname, int abs_mode, int long_na BUFCOPY(b, s-b); rb_str_set_len(result, p-buf); } - if (p == skiproot(buf, p + !!*p, enc) - 1) p++; + if (p == skiproot(buf, p + !!*p) - 1) p++; #if USE_NTFS *p = '\0'; @@ -4466,7 +4516,7 @@ realpath_rec(long *prefixlenp, VALUE *resolvedp, const char *unresolved, VALUE f if (*prefixlenp < RSTRING_LEN(*resolvedp)) { const char *resolved_str = RSTRING_PTR(*resolvedp); const char *resolved_names = resolved_str + *prefixlenp; - const char *lastsep = strrdirsep(resolved_names, resolved_str + RSTRING_LEN(*resolvedp), enc); + const char *lastsep = strrdirsep(resolved_names, resolved_str + RSTRING_LEN(*resolvedp), true, enc); long len = lastsep ? lastsep - resolved_names : 0; rb_str_resize(*resolvedp, *prefixlenp + len); } @@ -4606,7 +4656,7 @@ rb_check_realpath_emulate(VALUE basedir, VALUE path, rb_encoding *origenc, enum root_found: RSTRING_GETMEM(resolved, prefixptr, prefixlen); pend = prefixptr + prefixlen; - ptr = chompdirsep(prefixptr, pend, enc); + ptr = chompdirsep(prefixptr, pend, true, enc); if (ptr < pend) { prefixlen = ++ptr - prefixptr; rb_str_set_len(resolved, prefixlen); @@ -4616,7 +4666,7 @@ rb_check_realpath_emulate(VALUE basedir, VALUE path, rb_encoding *origenc, enum if (*prefixptr == FILE_ALT_SEPARATOR) { *prefixptr = '/'; } - Inc(prefixptr, pend, enc); + Inc(prefixptr, pend, true, enc); } #endif @@ -4860,8 +4910,8 @@ rmext(const char *p, long l0, long l1, const char *e, long l2, rb_encoding *enc) return 0; } -const char * -ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encoding *enc) +static inline const char * +enc_find_basename(const char *name, long *baselen, long *alllen, bool mb_enc, rb_encoding *enc) { const char *p, *q, *e, *end; #if defined DOSISH_DRIVE_LETTER || defined DOSISH_UNC @@ -4869,13 +4919,22 @@ ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encodin #endif long f = 0, n = -1; - end = name + (alllen ? (size_t)*alllen : strlen(name)); - name = skipprefix(name, end, enc); + long len = (alllen ? (size_t)*alllen : strlen(name)); + + if (len <= 0) { + return name; + } + + end = name + len; + name = skipprefix(name, end, mb_enc, enc); #if defined DOSISH_DRIVE_LETTER || defined DOSISH_UNC root = name; #endif - while (isdirsep(*name)) + + while (isdirsep(*name)) { name++; + } + if (!*name) { p = name - 1; f = 1; @@ -4897,32 +4956,47 @@ ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encodin #endif /* defined DOSISH_DRIVE_LETTER || defined DOSISH_UNC */ } else { - if (!(p = strrdirsep(name, end, enc))) { + p = strrdirsep(name, end, mb_enc, enc); + if (!p) { p = name; } else { - while (isdirsep(*p)) p++; /* skip last / */ + while (isdirsep(*p)) { + p++; /* skip last / */ + } } #if USE_NTFS n = ntfs_tail(p, end, enc) - p; #else - n = chompdirsep(p, end, enc) - p; + n = chompdirsep(p, end, mb_enc, enc) - p; #endif for (q = p; q - p < n && *q == '.'; q++); - for (e = 0; q - p < n; Inc(q, end, enc)) { + for (e = 0; q - p < n; Inc(q, end, mb_enc, enc)) { if (*q == '.') e = q; } - if (e) f = e - p; - else f = n; + if (e) { + f = e - p; + } + else { + f = n; + } } - if (baselen) + if (baselen) { *baselen = f; - if (alllen) + } + if (alllen) { *alllen = n; + } return p; } +const char * +ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encoding *enc) +{ + return enc_find_basename(name, baselen, alllen, true, enc); +} + /* * call-seq: * File.basename(file_name [, suffix] ) -> base_name @@ -4943,7 +5017,7 @@ ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encodin static VALUE rb_file_s_basename(int argc, VALUE *argv, VALUE _) { - VALUE fname, fext, basename; + VALUE fname, fext; const char *name, *p; long f, n; rb_encoding *enc; @@ -4952,18 +5026,23 @@ rb_file_s_basename(int argc, VALUE *argv, VALUE _) if (rb_check_arity(argc, 1, 2) == 2) { fext = argv[1]; StringValue(fext); - enc = check_path_encoding(fext); + check_path_encoding(fext); + enc = rb_str_enc_get(fext); } fname = argv[0]; - FilePathStringValue(fname); + CheckPath(fname, name); if (NIL_P(fext) || !(enc = rb_enc_compatible(fname, fext))) { - enc = rb_enc_get(fname); + enc = rb_str_enc_get(fname); fext = Qnil; } - if ((n = RSTRING_LEN(fname)) == 0 || !*(name = RSTRING_PTR(fname))) - return rb_str_new_shared(fname); - p = ruby_enc_find_basename(name, &f, &n, enc); + n = RSTRING_LEN(fname); + if (n == 0 || !*name) { + rb_enc_str_new(0, 0, enc); + } + + bool mb_enc = !rb_str_encindex_fastpath(rb_enc_to_index(enc)); + p = enc_find_basename(name, &f, &n, mb_enc, enc); if (n >= 0) { if (NIL_P(fext)) { f = n; @@ -4976,12 +5055,12 @@ rb_file_s_basename(int argc, VALUE *argv, VALUE _) } RB_GC_GUARD(fext); } - if (f == RSTRING_LEN(fname)) return rb_str_new_shared(fname); + if (f == RSTRING_LEN(fname)) { + return rb_str_new_shared(fname); + } } - basename = rb_str_new(p, f); - rb_enc_copy(basename, fname); - return basename; + return rb_enc_str_new(p, f, enc); } static VALUE rb_file_dirname_n(VALUE fname, int n); @@ -5026,19 +5105,18 @@ rb_file_dirname_n(VALUE fname, int n) { const char *name, *root, *p, *end; VALUE dirname; - rb_encoding *enc; - VALUE sepsv = 0; - const char **seps; if (n < 0) rb_raise(rb_eArgError, "negative level: %d", n); - FilePathStringValue(fname); - name = StringValueCStr(fname); + CheckPath(fname, name); end = name + RSTRING_LEN(fname); - enc = rb_enc_get(fname); - root = skiproot(name, end, enc); + + bool mb_enc = !rb_str_enc_fastpath(fname); + rb_encoding *enc = rb_str_enc_get(fname); + + root = skiproot(name, end); #ifdef DOSISH_UNC if (root > name + 1 && isdirsep(*name)) - root = skipprefix(name = root - 2, end, enc); + root = skipprefix(name = root - 2, end, mb_enc, enc); #else if (root > name + 1) name = root - 1; @@ -5047,75 +5125,41 @@ rb_file_dirname_n(VALUE fname, int n) p = root; } else { - int i; - switch (n) { - case 0: - p = end; - break; - case 1: - if (!(p = strrdirsep(root, end, enc))) p = root; - break; - default: - seps = ALLOCV_N(const char *, sepsv, n); - for (i = 0; i < n; ++i) seps[i] = root; - i = 0; - for (p = root; p < end; ) { - if (isdirsep(*p)) { - const char *tmp = p++; - while (p < end && isdirsep(*p)) p++; - if (p >= end) break; - seps[i++] = tmp; - if (i == n) i = 0; - } - else { - Inc(p, end, enc); - } + p = end; + while (n) { + if (!(p = strrdirsep(root, p, mb_enc, enc))) { + p = root; + break; } - p = seps[i]; - ALLOCV_END(sepsv); - break; + n--; } } + if (p == name) { - dirname = rb_str_new(".", 1); - rb_enc_copy(dirname, fname); - return dirname; + return rb_enc_str_new(".", 1, enc); } #ifdef DOSISH_DRIVE_LETTER if (has_drive_letter(name) && isdirsep(*(name + 2))) { - const char *top = skiproot(name + 2, end, enc); - dirname = rb_str_new(name, 3); + const char *top = skiproot(name + 2, end); + dirname = rb_enc_str_new(name, 3, enc); rb_str_cat(dirname, top, p - top); } else #endif - dirname = rb_str_new(name, p - name); + dirname = rb_enc_str_new(name, p - name, enc); #ifdef DOSISH_DRIVE_LETTER if (has_drive_letter(name) && root == name + 2 && p - name == 2) rb_str_cat(dirname, ".", 1); #endif - rb_enc_copy(dirname, fname); return dirname; } -/* - * accept a String, and return the pointer of the extension. - * if len is passed, set the length of extension to it. - * returned pointer is in ``name'' or NULL. - * returns *len - * no dot NULL 0 - * dotfile top 0 - * end with dot dot 1 - * .ext dot len of .ext - * .ext:stream dot len of .ext without :stream (NTFS only) - * - */ -const char * -ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc) +static inline const char * +enc_find_extname(const char *name, long *len, bool mb_enc, rb_encoding *enc) { const char *p, *e, *end = name + (len ? *len : (long)strlen(name)); - p = strrdirsep(name, end, enc); /* get the last path component */ + p = strrdirsep(name, end, mb_enc, enc); /* get the last path component */ if (!p) p = name; else @@ -5148,7 +5192,7 @@ ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc) #endif else if (isdirsep(*p)) break; - Inc(p, end, enc); + Inc(p, end, mb_enc, enc); } if (len) { @@ -5164,6 +5208,24 @@ ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc) } /* + * accept a String, and return the pointer of the extension. + * if len is passed, set the length of extension to it. + * returned pointer is in ``name'' or NULL. + * returns *len + * no dot NULL 0 + * dotfile top 0 + * end with dot dot 1 + * .ext dot len of .ext + * .ext:stream dot len of .ext without :stream (NTFS only) + * + */ +const char * +ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc) +{ + return enc_find_extname(name, len, true, enc); +} + +/* * call-seq: * File.extname(path) -> string * @@ -5192,18 +5254,19 @@ ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc) static VALUE rb_file_s_extname(VALUE klass, VALUE fname) { - const char *name, *e; - long len; - VALUE extname; + const char *name; + CheckPath(fname, name); + long len = RSTRING_LEN(fname); - FilePathStringValue(fname); - name = StringValueCStr(fname); - len = RSTRING_LEN(fname); - e = ruby_enc_find_extname(name, &len, rb_enc_get(fname)); - if (len < 1) - return rb_str_new(0, 0); - extname = rb_str_subseq(fname, e - name, len); /* keep the dot, too! */ - return extname; + if (len < 1) { + return rb_enc_str_new(0, 0, rb_str_enc_get(fname)); + } + + bool mb_enc = !rb_str_enc_fastpath(fname); + rb_encoding *enc = rb_str_enc_get(fname); + + const char *ext = enc_find_extname(name, &len, mb_enc, enc); + return rb_enc_str_new(ext, len, enc); } /* @@ -5315,7 +5378,7 @@ rb_file_join_ary(VALUE ary) rb_enc_copy(result, tmp); } else { - tail = chompdirsep(name, name + len, rb_enc_get(result)); + tail = chompdirsep(name, name + len, true, rb_enc_get(result)); if (RSTRING_PTR(tmp) && isdirsep(RSTRING_PTR(tmp)[0])) { rb_str_set_len(result, tail - name); } diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index b9fccd6b4c..b8af39cd99 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -439,16 +439,15 @@ rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data) } static void -rb_mmtk_update_global_tables(int table) +rb_mmtk_update_global_tables(int table, bool moving) { MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT); - // TODO: set weak_only to true for non-moving GC rb_gc_vm_weak_table_foreach( rb_mmtk_update_global_tables_i, rb_mmtk_update_global_tables_replace_i, NULL, - false, + !moving, (enum rb_gc_vm_weak_tables)table ); } diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h index 21a5bf9415..4cef1668a4 100644 --- a/gc/mmtk/mmtk.h +++ b/gc/mmtk/mmtk.h @@ -74,7 +74,7 @@ typedef struct MMTk_RubyUpcalls { void (*handle_weak_references)(MMTk_ObjectReference object, bool moving); void (*call_obj_free)(MMTk_ObjectReference object); size_t (*vm_live_bytes)(void); - void (*update_global_tables)(int tbl_idx); + void (*update_global_tables)(int tbl_idx, bool moving); int (*global_tables_count)(void); void (*update_finalizer_table)(void); bool (*special_const_p)(MMTk_ObjectReference object); diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs index 255b2b1e56..2a0e9113fa 100644 --- a/gc/mmtk/src/abi.rs +++ b/gc/mmtk/src/abi.rs @@ -318,7 +318,7 @@ pub struct RubyUpcalls { pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool), pub call_obj_free: extern "C" fn(object: ObjectReference), pub vm_live_bytes: extern "C" fn() -> usize, - pub update_global_tables: extern "C" fn(tbl_idx: c_int), + pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool), pub global_tables_count: extern "C" fn() -> c_int, pub update_finalizer_table: extern "C" fn(), pub special_const_p: extern "C" fn(object: ObjectReference) -> bool, diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs index 19dc6a0ee1..d0a54f01bf 100644 --- a/gc/mmtk/src/weak_proc.rs +++ b/gc/mmtk/src/weak_proc.rs @@ -270,7 +270,10 @@ struct UpdateGlobalTables { } impl GlobalTableProcessingWork for UpdateGlobalTables { fn process_table(&mut self) { - (crate::upcalls().update_global_tables)(self.idx) + (crate::upcalls().update_global_tables)( + self.idx, + crate::mmtk().get_plan().current_gc_may_move_object(), + ) } } impl GCWork<Ruby> for UpdateGlobalTables { @@ -2923,7 +2923,7 @@ NOINSERT_UPDATE_CALLBACK(hash_aset_str) * h = {foo: 0, bar: 1} * h[:baz] = 2 # => 2 * h[:baz] # => 2 - * h # => {:foo=>0, :bar=>1, :baz=>2} + * h # => {foo: 0, bar: 1, baz: 2} * * Related: #[]; see also {Methods for Assigning}[rdoc-ref:Hash@Methods+for+Assigning]. */ diff --git a/internal/string.h b/internal/string.h index cd1e8d7929..9212ce8986 100644 --- a/internal/string.h +++ b/internal/string.h @@ -33,7 +33,13 @@ enum ruby_rstring_private_flags { static inline bool rb_str_encindex_fastpath(int encindex) { - // The overwhelming majority of strings are in one of these 3 encodings. + // The overwhelming majority of strings are in one of these 3 encodings, + // which are all either ASCII or perfect ASCII supersets. + // Hence you can use fast, single byte algorithms on them, such as `memchr` etc, + // without all the overhead of fetching the rb_encoding and using functions such as + // rb_enc_mbminlen etc. + // Many other encodings could qualify, but they are expected to be rare occurences, + // so it's better to keep that list small. switch (encindex) { case ENCINDEX_ASCII_8BIT: case ENCINDEX_UTF_8: @@ -50,6 +56,13 @@ rb_str_enc_fastpath(VALUE str) return rb_str_encindex_fastpath(ENCODING_GET_INLINED(str)); } +static inline rb_encoding * +rb_str_enc_get(VALUE str) +{ + RUBY_ASSERT(RB_TYPE_P(str, T_STRING)); + return rb_enc_from_index(ENCODING_GET(str)); +} + /* string.c */ VALUE rb_str_dup_m(VALUE str); VALUE rb_fstring(VALUE); diff --git a/lib/prism.rb b/lib/prism.rb index d809557fce..dab3420377 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -61,8 +61,7 @@ module Prism # Prism::lex_compat(source, **options) -> LexCompat::Result # # Returns a parse result whose value is an array of tokens that closely - # resembles the return value of Ripper::lex. The main difference is that the - # `:on_sp` token is not emitted. + # resembles the return value of Ripper::lex. # # For supported options, see Prism::parse. def self.lex_compat(source, **options) @@ -72,9 +71,8 @@ module Prism # :call-seq: # Prism::lex_ripper(source) -> Array # - # This lexes with the Ripper lex. It drops any space events but otherwise - # returns the same tokens. Raises SyntaxError if the syntax in source is - # invalid. + # This wraps the result of Ripper.lex. It produces almost exactly the + # same tokens. Raises SyntaxError if the syntax in source is invalid. def self.lex_ripper(source) LexRipper.new(source).result # steep:ignore end diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index f7b9a0effc..597e63c73e 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -226,7 +226,7 @@ module Prism end # Tokens where state should be ignored - # used for :on_comment, :on_heredoc_end, :on_embexpr_end + # used for :on_sp, :on_comment, :on_heredoc_end, :on_embexpr_end class IgnoreStateToken < Token def ==(other) # :nodoc: self[0...-1] == other[0...-1] @@ -611,10 +611,10 @@ module Prism BOM_FLUSHED = RUBY_VERSION >= "3.3.0" private_constant :BOM_FLUSHED - attr_reader :source, :options + attr_reader :options - def initialize(source, **options) - @source = source + def initialize(code, **options) + @code = code @options = options end @@ -624,12 +624,14 @@ module Prism state = :default heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]] - result = Prism.lex(source, **options) + result = Prism.lex(@code, **options) + source = result.source result_value = result.value previous_state = nil #: State? last_heredoc_end = nil #: Integer? + eof_token = nil - bom = source.byteslice(0..2) == "\xEF\xBB\xBF" + bom = source.slice(0, 3) == "\xEF\xBB\xBF" result_value.each_with_index do |(token, lex_state), index| lineno = token.location.start_line @@ -741,6 +743,7 @@ module Prism Token.new([[lineno, column], event, value, lex_state]) when :on_eof + eof_token = token previous_token = result_value[index - 1][0] # If we're at the end of the file and the previous token was a @@ -763,7 +766,7 @@ module Prism end_offset += 3 end - tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state]) + tokens << Token.new([[lineno, 0], :on_nl, source.slice(start_offset, end_offset - start_offset), lex_state]) end end @@ -857,7 +860,89 @@ module Prism # We sort by location to compare against Ripper's output tokens.sort_by!(&:location) - Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source)) + # Add :on_sp tokens + tokens = add_on_sp_tokens(tokens, source, result.data_loc, bom, eof_token) + + Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source) + end + + def add_on_sp_tokens(tokens, source, data_loc, bom, eof_token) + new_tokens = [] + + prev_token_state = Translation::Ripper::Lexer::State.cached(Translation::Ripper::EXPR_BEG) + prev_token_end = bom ? 3 : 0 + + tokens.each do |token| + line, column = token.location + start_offset = source.line_to_byte_offset(line) + column + # Ripper reports columns on line 1 without counting the BOM, so we adjust to get the real offset + start_offset += 3 if line == 1 && bom + + if start_offset > prev_token_end + sp_value = source.slice(prev_token_end, start_offset - prev_token_end) + sp_line = source.line(prev_token_end) + sp_column = source.column(prev_token_end) + # Ripper reports columns on line 1 without counting the BOM + sp_column -= 3 if sp_line == 1 && bom + continuation_index = sp_value.byteindex("\\") + + # ripper emits up to three :on_sp tokens when line continuations are used + if continuation_index + next_whitespace_index = continuation_index + 1 + next_whitespace_index += 1 if sp_value.byteslice(next_whitespace_index) == "\r" + next_whitespace_index += 1 + first_whitespace = sp_value[0...continuation_index] + continuation = sp_value[continuation_index...next_whitespace_index] + second_whitespace = sp_value[next_whitespace_index..] + + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column], + :on_sp, + first_whitespace, + prev_token_state + ]) unless first_whitespace.empty? + + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column + continuation_index], + :on_sp, + continuation, + prev_token_state + ]) + + new_tokens << IgnoreStateToken.new([ + [sp_line + 1, 0], + :on_sp, + second_whitespace, + prev_token_state + ]) unless second_whitespace.empty? + else + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column], + :on_sp, + sp_value, + prev_token_state + ]) + end + end + + new_tokens << token + prev_token_state = token.state + prev_token_end = start_offset + token.value.bytesize + end + + unless data_loc # no trailing :on_sp with __END__ as it is always preceded by :on_nl + end_offset = eof_token.location.end_offset + if prev_token_end < end_offset + new_tokens << IgnoreStateToken.new([ + [source.line(prev_token_end), source.column(prev_token_end)], + :on_sp, + source.slice(prev_token_end, end_offset - prev_token_end), + prev_token_state + ]) + end + end + + new_tokens end end diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 4b5c3b77fd..2054cf55ac 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -19,8 +19,6 @@ module Prism lex(source).each do |token| case token[1] - when :on_sp - # skip when :on_tstring_content if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) previous[2] << token[2] diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index c8f9fa7731..735217d2e0 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -832,7 +832,7 @@ module Prism # foo(bar) # ^^^ def visit_arguments_node(node) - arguments, _ = visit_call_node_arguments(node, nil, false) + arguments, _, _ = visit_call_node_arguments(node, nil, false) arguments end @@ -1042,16 +1042,16 @@ module Prism case node.name when :[] receiver = visit(node.receiver) - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) call = on_aref(receiver, arguments) - if block.nil? - call - else + if has_ripper_block bounds(node.location) on_method_add_block(call, block) + else + call end when :[]= receiver = visit(node.receiver) @@ -1110,9 +1110,9 @@ module Prism if node.variable_call? on_vcall(message) else - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) call = - if node.opening_loc.nil? && arguments&.any? + if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any? bounds(node.location) on_command(message, arguments) elsif !node.opening_loc.nil? @@ -1123,11 +1123,11 @@ module Prism on_method_add_arg(on_fcall(message), on_args_new) end - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end end @@ -1151,7 +1151,7 @@ module Prism bounds(node.location) on_assign(on_field(receiver, call_operator, message), value) else - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) call = if node.opening_loc.nil? bounds(node.location) @@ -1169,27 +1169,35 @@ module Prism on_method_add_arg(on_call(receiver, call_operator, message), arguments) end - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end end end - # Visit the arguments and block of a call node and return the arguments - # and block as they should be used. - private def visit_call_node_arguments(arguments_node, block_node, trailing_comma) + # Extract the arguments and block Ripper-style, which means if the block + # is like `&b` then it's moved to arguments. + private def get_arguments_and_block(arguments_node, block_node) arguments = arguments_node&.arguments || [] block = block_node if block.is_a?(BlockArgumentNode) - arguments << block + arguments += [block] block = nil end + [arguments, block] + end + + # Visit the arguments and block of a call node and return the arguments + # and block as they should be used. + private def visit_call_node_arguments(arguments_node, block_node, trailing_comma) + arguments, block = get_arguments_and_block(arguments_node, block_node) + [ if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode) visit(arguments.first) @@ -1203,7 +1211,8 @@ module Prism on_args_add_block(args, false) end end, - visit(block) + visit(block), + block != nil, ] end @@ -1640,10 +1649,10 @@ module Prism end bounds(node.location) - if receiver.nil? - on_def(name, parameters, bodystmt) - else + if receiver on_defs(receiver, operator, name, parameters, bodystmt) + else + on_def(name, parameters, bodystmt) end end @@ -2041,7 +2050,7 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_index_operator_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2058,7 +2067,7 @@ module Prism # ^^^^^^^^^^^^^^^^ def visit_index_and_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2075,7 +2084,7 @@ module Prism # ^^^^^^^^^^^^^^^^ def visit_index_or_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2092,7 +2101,7 @@ module Prism # ^^^^^^^^ def visit_index_target_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) on_aref_field(receiver, arguments) @@ -3122,7 +3131,7 @@ module Prism # super(foo) # ^^^^^^^^^^ def visit_super_node(node) - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) if !node.lparen_loc.nil? bounds(node.lparen_loc) @@ -3132,11 +3141,11 @@ module Prism bounds(node.location) call = on_super(arguments) - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end @@ -3446,12 +3455,12 @@ module Prism # :stopdoc: def _dispatch_0; end - def _dispatch_1(_); end - def _dispatch_2(_, _); end - def _dispatch_3(_, _, _); end - def _dispatch_4(_, _, _, _); end - def _dispatch_5(_, _, _, _, _); end - def _dispatch_7(_, _, _, _, _, _, _); end + def _dispatch_1(arg); arg end + def _dispatch_2(arg, _); arg end + def _dispatch_3(arg, _, _); arg end + def _dispatch_4(arg, _, _, _); arg end + def _dispatch_5(arg, _, _, _, _); arg end + def _dispatch_7(arg, _, _, _, _, _, _); arg end # :startdoc: # diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index 350daf1e16..62d36bcf48 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -163,8 +163,6 @@ class Gem::Ext::Builder @gem_dir = spec.full_gem_path @target_rbconfig = target_rbconfig @build_jobs = build_jobs - - @ran_rake = false end ## @@ -177,7 +175,6 @@ class Gem::Ext::Builder when /configure/ then Gem::Ext::ConfigureBuilder when /rakefile/i, /mkrf_conf/i then - @ran_rake = true Gem::Ext::RakeBuilder when /CMakeLists.txt/ then Gem::Ext::CmakeBuilder.new @@ -250,8 +247,6 @@ EOF FileUtils.rm_f @spec.gem_build_complete_path @spec.extensions.each do |extension| - break if @ran_rake - build_extension extension, dest_path end diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index e5008a24db..0fcb635394 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -436,6 +436,7 @@ or set it to nil if you don't want to specify a license. warning "deprecated autorequire specified" if @specification.autorequire @specification.executables.each do |executable| + validate_executable(executable) validate_shebang_line_in(executable) end @@ -449,6 +450,13 @@ or set it to nil if you don't want to specify a license. warning("no #{attribute} specified") if value.nil? || value.empty? end + def validate_executable(executable) + separators = [File::SEPARATOR, File::ALT_SEPARATOR, File::PATH_SEPARATOR].compact.map {|sep| Regexp.escape(sep) }.join + return unless executable.match?(/[\s#{separators}]/) + + error "executable \"#{executable}\" contains invalid characters" + end + def validate_shebang_line_in(executable) executable_path = File.join(@specification.bindir, executable) return if File.read(executable_path, 2) == "#!" diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 58197e95d0..76ca892ac3 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -180,18 +180,17 @@ module SyntaxSuggest # EOM # expect(lines.first.trailing_slash?).to eq(true) # - if SyntaxSuggest.use_prism_parser? - def trailing_slash? - last = @lex.last - last&.type == :on_tstring_end - end - else - def trailing_slash? - last = @lex.last - return false unless last - return false unless last.type == :on_sp + def trailing_slash? + last = @lex.last + # Older versions of prism diverged slightly from Ripper in compatibility mode + case last&.type + when :on_sp last.token == TRAILING_SLASH + when :on_tstring_end + true + else + false end end diff --git a/lib/syntax_suggest/version.rb b/lib/syntax_suggest/version.rb index 1aa908f4e5..db50a1a89a 100644 --- a/lib/syntax_suggest/version.rb +++ b/lib/syntax_suggest/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxSuggest - VERSION = "2.0.2" + VERSION = "2.0.3" end diff --git a/spec/ruby/core/file/basename_spec.rb b/spec/ruby/core/file/basename_spec.rb index 989409d76b..87695ab97b 100644 --- a/spec/ruby/core/file/basename_spec.rb +++ b/spec/ruby/core/file/basename_spec.rb @@ -151,8 +151,34 @@ describe "File.basename" do File.basename("c:\\bar.txt", ".*").should == "bar" File.basename("c:\\bar.txt.exe", ".*").should == "bar.txt" end + + it "handles Shift JIS 0x5C (\\) as second byte of a multi-byte sequence" do + # dir\fileソname.txt + path = "dir\\file\x83\x5cname.txt".b.force_encoding(Encoding::SHIFT_JIS) + path.valid_encoding?.should be_true + File.basename(path).should == "file\x83\x5cname.txt".b.force_encoding(Encoding::SHIFT_JIS) + end end + it "rejects strings encoded with non ASCII-compatible encodings" do + Encoding.list.reject(&:ascii_compatible?).reject(&:dummy?).each do |enc| + begin + path = "/foo/bar".encode(enc) + rescue Encoding::ConverterNotFoundError + next + end + + -> { + File.basename(path) + }.should raise_error(Encoding::CompatibilityError) + end + end + + it "works with all ASCII-compatible encodings" do + Encoding.list.select(&:ascii_compatible?).each do |enc| + File.basename("/foo/bar".encode(enc)).should == "bar".encode(enc) + end + end it "returns the extension for a multibyte filename" do File.basename('/path/Офис.m4a').should == "Офис.m4a" @@ -2880,7 +2880,7 @@ str_null_check(VALUE str, int *w) int minlen = 1; if (RB_UNLIKELY(!rb_str_enc_fastpath(str))) { - rb_encoding *enc = rb_enc_get(str); + rb_encoding *enc = rb_str_enc_get(str); minlen = rb_enc_mbminlen(enc); if (minlen > 1) { diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index 78947a095b..8b369c3894 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -54,7 +54,11 @@ class TestObjSpace < Test::Unit::TestCase assert_operator(a, :>, b) assert_operator(a, :>, 0) assert_operator(b, :>, 0) - assert_raise(TypeError) {ObjectSpace.memsize_of_all('error')} + assert_kind_of(Integer, ObjectSpace.memsize_of_all(Enumerable)) + end + + def test_memsize_of_all_with_wrong_type + assert_raise(TypeError) { ObjectSpace.memsize_of_all(Object.new) } end def test_count_objects_size diff --git a/test/prism/fixtures/bom_leading_space.txt b/test/prism/fixtures/bom_leading_space.txt new file mode 100644 index 0000000000..48d3ee50ea --- /dev/null +++ b/test/prism/fixtures/bom_leading_space.txt @@ -0,0 +1 @@ + p (42) diff --git a/test/prism/fixtures/bom_spaces.txt b/test/prism/fixtures/bom_spaces.txt new file mode 100644 index 0000000000..c18ad4c21a --- /dev/null +++ b/test/prism/fixtures/bom_spaces.txt @@ -0,0 +1 @@ +p ( 42 ) diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 2a0504c19f..6e9dcee4c9 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -39,6 +39,8 @@ module Prism # Skip these tests that we haven't implemented yet. omitted_sexp_raw = [ + "bom_leading_space.txt", + "bom_spaces.txt", "dos_endings.txt", "heredocs_with_fake_newlines.txt", "heredocs_with_ignored_newlines.txt", @@ -84,6 +86,56 @@ module Prism define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) } end + module Events + attr_reader :events + + def initialize(...) + super + @events = [] + end + + Prism::Translation::Ripper::PARSER_EVENTS.each do |event| + define_method(:"on_#{event}") do |*args| + @events << [event, *args] + super(*args) + end + end + end + + class RipperEvents < Ripper + include Events + end + + class PrismEvents < Translation::Ripper + include Events + end + + class ObjectEvents < Translation::Ripper + OBJECT = BasicObject.new + Prism::Translation::Ripper::PARSER_EVENTS.each do |event| + define_method(:"on_#{event}") { |*args| OBJECT } + end + end + + Fixture.each_for_current_ruby(except: incorrect) do |fixture| + define_method("#{fixture.test_name}_events") do + source = fixture.read + # Similar to test/ripper/assert_parse_files.rb in CRuby + object_events = ObjectEvents.new(source) + assert_nothing_raised { object_events.parse } + end + end + + def test_events + source = "1 rescue 2" + ripper = RipperEvents.new(source) + prism = PrismEvents.new(source) + ripper.parse + prism.parse + # This makes sure that the content is the same. Ordering is not correct for now. + assert_equal(ripper.events.sort, prism.events.sort) + end + def test_lexer lexer = Translation::Ripper::Lexer.new("foo") expected = [[1, 0], :on_ident, "foo", Translation::Ripper::EXPR_CMDARG] @@ -92,7 +144,7 @@ module Prism assert_equal(expected, lexer.parse[0].to_a) assert_equal(lexer.parse[0].to_a, lexer.scan[0].to_a) - assert_equal(%i[on_int on_op], Translation::Ripper::Lexer.new("1 +").lex.map(&:event)) + assert_equal(%i[on_int on_sp on_op], Translation::Ripper::Lexer.new("1 +").lex.map(&:event)) assert_raise(SyntaxError) { Translation::Ripper::Lexer.new("1 +").lex(raise_errors: true) } end @@ -121,15 +173,17 @@ module Prism def assert_ripper_lex(source) prism = Translation::Ripper.lex(source) ripper = Ripper.lex(source) - ripper.reject! { |elem| elem[1] == :on_sp } # Prism doesn't emit on_sp - ripper.sort_by! { |elem| elem[0] } # Prism emits tokens by their order in the code, not in parse order + + # Prism emits tokens by their order in the code, not in parse order + ripper.sort_by! { |elem| elem[0] } [prism.size, ripper.size].max.times do |i| expected = ripper[i] actual = prism[i] + # Since tokens related to heredocs are not emitted in the same order, # the state also doesn't line up. - if expected[1] == :on_heredoc_end && actual[1] == :on_heredoc_end + if expected && actual && expected[1] == :on_heredoc_end && actual[1] == :on_heredoc_end expected[3] = actual[3] = nil end diff --git a/test/psych/test_data.rb b/test/psych/test_data.rb index 57c3478193..5e340c580a 100644 --- a/test/psych/test_data.rb +++ b/test/psych/test_data.rb @@ -83,12 +83,11 @@ module Psych # completely different members TestData.send :remove_const, :D - TestData.const_set :D, Data.define(:foo, :bar) + TestData.const_set :D, Data.define(:a, :c) e = assert_raise(ArgumentError) { Psych.unsafe_load d } - assert_equal 'unknown keywords: :a, :b', e.message + assert_include e.message, 'keyword:' ensure TestData.send :remove_const, :D end end end - diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index 2a61fc3450..7f5dbe9155 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -1595,7 +1595,7 @@ q.pop # [Bug #21342] def test_unlock_locked_mutex_with_collected_fiber bug21127 = '[ruby-core:120930] [Bug #21127]' - assert_ruby_status([], "#{<<~"begin;"}\n#{<<~'end;'}") + assert_ruby_status([], "#{<<~"begin;"}\n#{<<~'end;'}", bug21127) begin; 5.times do m = Mutex.new diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index ad2df806d5..2066610cb2 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -470,6 +470,42 @@ class TestZJIT < Test::Unit::TestCase }, insns: [:getblockparamproxy] end + def test_getblockparam + assert_compiles '2', %q{ + def test(&blk) + blk + end + test { 2 }.call + test { 2 }.call + }, insns: [:getblockparam] + end + + def test_getblockparam_proxy_side_exit_restores_block_local + assert_compiles '2', %q{ + def test(&block) + b = block + # sideexits here + raise "test" unless block + b ? 2 : 3 + end + test {} + test {} + }, insns: [:getblockparam, :getblockparamproxy] + end + + def test_getblockparam_used_twice_in_args + assert_compiles '1', %q{ + def f(*args) = args + def test(&blk) + b = blk + f(*[1], blk) + blk + end + test {1}.call + test {1}.call + }, insns: [:getblockparam] + end + def test_optimized_method_call_proc_call assert_compiles '2', %q{ p = proc { |x| x * 2 } @@ -833,6 +869,61 @@ class TestZJIT < Test::Unit::TestCase }, call_threshold: 2 end + def test_pos_optional_with_maybe_too_many_args + assert_compiles '[[1, 2, 3, 4, 5, 6], [10, 20, 30, 4, 5, 6], [10, 20, 30, 40, 50, 60]]', %q{ + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + }, call_threshold: 2 + end + + def test_send_kwarg_partial_optional + assert_compiles '[[1, 2, 3], [1, 20, 3], [10, 2, 30]]', %q{ + def test(a: 1, b: 2, c: 3) = [a, b, c] + def entry = [test, test(b: 20), test(c: 30, a: 10)] + entry + entry + }, call_threshold: 2 + end + + def test_send_kwarg_optional_a_lot + assert_compiles '[[1, 2, 3, 4, 5, 6], [1, 2, 3, 7, 8, 9], [2, 4, 6, 8, 10, 12]]', %q{ + def test(a: 1, b: 2, c: 3, d: 4, e: 5, f: 6) = [a, b, c, d, e, f] + def entry = [test, test(d: 7, f: 9, e: 8), test(f: 12, e: 10, d: 8, c: 6, b: 4, a: 2)] + entry + entry + }, call_threshold: 2 + end + + def test_send_kwarg_non_constant_default + assert_compiles '[[1, 2], [10, 2]]', %q{ + def make_default = 2 + def test(a: 1, b: make_default) = [a, b] + def entry = [test, test(a: 10)] + entry + entry + }, call_threshold: 2 + end + + def test_send_kwarg_optional_static_with_side_exit + # verify frame reconstruction with synthesized keyword defaults is correct + assert_compiles '[10, 2, 10]', %q{ + def callee(a: 1, b: 2) + # use binding to force side-exit + x = binding.local_variable_get(:a) + [a, b, x] + end + + def entry + callee(a: 10) # b should get default value + end + + entry + entry + }, call_threshold: 2 + end + def test_send_all_arg_types assert_compiles '[:req, :opt, :post, :kwr, :kwo, true]', %q{ def test(a, b = :opt, c, d:, e: :kwo) = [a, b, c, d, e, block_given?] @@ -1388,6 +1479,190 @@ class TestZJIT < Test::Unit::TestCase }, call_threshold: 2 end + def test_invokesuper_with_optional_keyword_args + assert_compiles '[1, 2, 3]', %q{ + class Parent + def foo(a, b: 2, c: 3) = [a, b, c] + end + + class Child < Parent + def foo(a) = super(a) + end + + def test = Child.new.foo(1) + + test + test + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default + assert_compiles '[[2, 4, 16], [10, 4, 16], [2, 20, 16], [2, 4, 30], [10, 20, 30]]', %q{ + def dbl(x = 1) = x * 2 + + def foo(a: dbl, b: dbl(2), c: dbl(2 ** 3)) + [a, b, c] + end + + def test + [ + foo, + foo(a: 10), + foo(b: 20), + foo(c: 30), + foo(a: 10, b: 20, c: 30) + ] + end + + test + test + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default_not_evaluated_when_provided + assert_compiles '[1, 2, 3]', %q{ + def foo(a: raise, b: raise, c: raise) + [a, b, c] + end + + def test + foo(a: 1, b: 2, c: 3) + end + + test + test + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default_evaluated_when_not_provided + assert_compiles '["a", "b", "c"]', %q{ + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default_jit_to_jit + # Test that kw_bits passing works correctly in JIT-to-JIT calls + assert_compiles '[2, 4, 6]', %q{ + def make_default(x) = x * 2 + + def callee(a: make_default(1), b: make_default(2), c: make_default(3)) + [a, b, c] + end + + def caller_method + callee + end + + # Warm up callee first so it gets JITted + callee + callee + + # Now warm up caller - this creates JIT-to-JIT call + caller_method + caller_method + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default_side_exit + # Verify frame reconstruction includes correct values for non-constant defaults + assert_compiles '[10, 2, 30]', %q{ + def make_b = 2 + + def callee(a: 1, b: make_b, c: 3) + x = binding.local_variable_get(:a) + y = binding.local_variable_get(:b) + z = binding.local_variable_get(:c) + [x, y, z] + end + + def test + callee(a: 10, c: 30) + end + + test + test + }, call_threshold: 2 + end + + def test_send_with_non_constant_keyword_default_evaluation_order + # Verify defaults are evaluated left-to-right and only when not provided + assert_compiles '[["a", "b", "c"], ["b", "c"], ["a", "c"], ["a", "b"]]', %q{ + def log(x) + $order << x + x + end + + def foo(a: log("a"), b: log("b"), c: log("c")) + [a, b, c] + end + + def test + results = [] + + $order = [] + foo + results << $order.dup + + $order = [] + foo(a: "A") + results << $order.dup + + $order = [] + foo(b: "B") + results << $order.dup + + $order = [] + foo(c: "C") + results << $order.dup + + results + end + + test + test + }, call_threshold: 2 + end + + def test_send_with_too_many_non_constant_keyword_defaults + assert_compiles '35', %q{ + def many_kwargs( k1: 1, k2: 2, k3: 3, k4: 4, k5: 5, k6: 6, k7: 7, k8: 8, k9: 9, k10: 10, k11: 11, k12: 12, k13: 13, k14: 14, k15: 15, k16: 16, k17: 17, k18: 18, k19: 19, k20: 20, k21: 21, k22: 22, k23: 23, k24: 24, k25: 25, k26: 26, k27: 27, k28: 28, k29: 29, k30: 30, k31: 31, k32: 32, k33: 33, k34: k33 + 1) = k1 + k34 + def t = many_kwargs + t + t + }, call_threshold: 2 + end + def test_invokebuiltin # Not using assert_compiles due to register spill assert_runs '["."]', %q{ @@ -4417,6 +4692,82 @@ class TestZJIT < Test::Unit::TestCase }, call_threshold: 14, num_profiles: 5 end + def test_is_a_string_special_case + assert_compiles '[true, false, false, false, false, false]', %q{ + def test(x) + x.is_a?(String) + end + test("foo") + [test("bar"), test(1), test(false), test(:foo), test([]), test({})] + } + end + + def test_is_a_array_special_case + assert_compiles '[true, true, false, false, false, false, false]', %q{ + def test(x) + x.is_a?(Array) + end + test([]) + [test([1,2,3]), test([]), test(1), test(false), test(:foo), test("foo"), test({})] + } + end + + def test_is_a_hash_special_case + assert_compiles '[true, true, false, false, false, false, false]', %q{ + def test(x) + x.is_a?(Hash) + end + test({}) + [test({:a => "b"}), test({}), test(1), test(false), test(:foo), test([]), test("foo")] + } + end + + def test_is_a_hash_subclass + assert_compiles 'true', %q{ + class MyHash < Hash + end + def test(x) + x.is_a?(Hash) + end + test({}) + test(MyHash.new) + } + end + + def test_is_a_normal_case + assert_compiles '[true, false]', %q{ + class MyClass + end + def test(x) + x.is_a?(MyClass) + end + test("a") + [test(MyClass.new), test("a")] + } + end + + def test_exit_tracing + # This is a very basic smoke test. The StackProf format + # this option generates is external to us. + Dir.mktmpdir("zjit_test_exit_tracing") do |tmp_dir| + assert_compiles('true', <<~RUBY, extra_args: ['-C', tmp_dir, '--zjit-trace-exits']) + def test(object) = object.itself + + # induce an exit just for good measure + array = [] + test(array) + test(array) + def array.itself = :not_itself + test(array) + + RubyVM::ZJIT.exit_locations.is_a?(Hash) + RUBY + dump_files = Dir.glob('zjit_exits_*.dump', base: tmp_dir) + assert_equal(1, dump_files.length) + refute(File.empty?(File.join(tmp_dir, dump_files.first))) + end + end + private # Assert that every method call in `test_script` can be compiled by ZJIT @@ -4477,10 +4828,11 @@ class TestZJIT < Test::Unit::TestCase stats: false, debug: true, allowed_iseqs: nil, + extra_args: nil, timeout: 1000, pipe_fd: nil ) - args = ["--disable-gems"] + args = ["--disable-gems", *extra_args] if zjit args << "--zjit-call-threshold=#{call_threshold}" args << "--zjit-num-profiles=#{num_profiles}" diff --git a/test/rubygems/test_gem_ext_builder.rb b/test/rubygems/test_gem_ext_builder.rb index 34f85e6b75..5fcbc3e2ac 100644 --- a/test/rubygems/test_gem_ext_builder.rb +++ b/test/rubygems/test_gem_ext_builder.rb @@ -18,7 +18,7 @@ class TestGemExtBuilder < Gem::TestCase @spec = util_spec "a" - @builder = Gem::Ext::Builder.new @spec, "" + @builder = Gem::Ext::Builder.new @spec end def teardown @@ -201,6 +201,57 @@ install: Gem.configuration.install_extension_in_lib = @orig_install_extension_in_lib end + def test_build_multiple_extensions + pend if RUBY_ENGINE == "truffleruby" + pend "terminates on ruby/ruby" if ruby_repo? + + extension_in_lib do + @spec.extensions << "ext/Rakefile" + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + + FileUtils.mkdir_p ext_dir + + extconf_rb = File.join ext_dir, "extconf.rb" + rakefile = File.join ext_dir, "Rakefile" + + File.open extconf_rb, "w" do |f| + f.write <<-'RUBY' + require 'mkmf' + + create_makefile 'a' + RUBY + end + + File.open rakefile, "w" do |f| + f.write <<-RUBY + task :default do + FileUtils.touch File.join "#{ext_dir}", 'foo' + end + RUBY + end + + ext_lib_dir = File.join ext_dir, "lib" + FileUtils.mkdir ext_lib_dir + FileUtils.touch File.join ext_lib_dir, "a.rb" + FileUtils.mkdir File.join ext_lib_dir, "a" + FileUtils.touch File.join ext_lib_dir, "a", "b.rb" + + use_ui @ui do + @builder.build_extensions + end + + assert_path_exist @spec.extension_dir + assert_path_exist @spec.gem_build_complete_path + assert_path_exist File.join @spec.gem_dir, "ext", "foo" + assert_path_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_exist File.join @spec.extension_dir, "a.rb" + assert_path_exist File.join @spec.gem_dir, "lib", "a.rb" + assert_path_exist File.join @spec.gem_dir, "lib", "a", "b.rb" + end + end + def test_build_extensions_none use_ui @ui do @builder.build_extensions diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index e8c2c0eb47..7675ade415 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -3013,6 +3013,65 @@ duplicate dependency on c (>= 1.2.3, development), (~> 1.2) use: assert_match "#{w}: bin/exec is missing #! line\n", @ui.error, "error" end + def test_validate_executables_with_space + util_setup_validate + + FileUtils.mkdir_p File.join(@tempdir, "bin") + File.write File.join(@tempdir, "bin", "echo hax"), "#!/usr/bin/env ruby\n" + + @a1.executables = ["echo hax"] + + e = assert_raise Gem::InvalidSpecificationException do + use_ui @ui do + Dir.chdir @tempdir do + @a1.validate + end + end + end + + assert_match "executable \"echo hax\" contains invalid characters", e.message + end + + def test_validate_executables_with_path_separator + util_setup_validate + + FileUtils.mkdir_p File.join(@tempdir, "bin") + File.write File.join(@tempdir, "exe"), "#!/usr/bin/env ruby\n" + + @a1.executables = Gem.win_platform? ? ["..\\exe"] : ["../exe"] + + e = assert_raise Gem::InvalidSpecificationException do + use_ui @ui do + Dir.chdir @tempdir do + @a1.validate + end + end + end + + assert_match "executable \"#{Gem.win_platform? ? "..\\exe" : "../exe"}\" contains invalid characters", e.message + end + + def test_validate_executables_with_path_list_separator + sep = Gem.win_platform? ? ";" : ":" + + util_setup_validate + + FileUtils.mkdir_p File.join(@tempdir, "bin") + File.write File.join(@tempdir, "bin", "foo#{sep}bar"), "#!/usr/bin/env ruby\n" + + @a1.executables = ["foo#{sep}bar"] + + e = assert_raise Gem::InvalidSpecificationException do + use_ui @ui do + Dir.chdir @tempdir do + @a1.validate + end + end + end + + assert_match "executable \"foo#{sep}bar\" contains invalid characters", e.message + end + def test_validate_empty_require_paths util_setup_validate diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 14d7a3893d..477cc75546 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -398,6 +398,10 @@ module SyncDefaultGems upstream = File.join("..", "..", config.upstream) + unless File.exist?(upstream) + abort %[Expected '#{upstream}' (#{File.expand_path("#{upstream}")}) to be a directory, but it didn't exist.] + end + config.mappings.each do |src, dst| rm_rf(dst) end @@ -798,26 +802,6 @@ module SyncDefaultGems return true end - def sync_lib(repo, upstream = nil) - unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") - abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] - end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" - else - "lib/#{repo}.gemspec" - end - cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") - end - def update_default_gems(gem, release: false) config = REPOSITORIES[gem] author, repository = config.upstream.split('/') diff --git a/vcpkg.json b/vcpkg.json index b31cf27ba8..0bb6deaa11 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -7,5 +7,5 @@ "openssl", "zlib" ], - "builtin-baseline": "84bab45d415d22042bd0b9081aea57f362da3f35" + "builtin-baseline": "66c0373dc7fca549e5803087b9487edfe3aca0a1" }
\ No newline at end of file @@ -3682,8 +3682,9 @@ rb_execution_context_mark(const rb_execution_context_t *ec) rb_control_frame_t *cfp = ec->cfp; rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size); - VM_ASSERT(sp == ec->cfp->sp); - rb_gc_mark_vm_stack_values((long)(sp - p), p); + for (long i = 0; i < (long)(sp - p); i++) { + rb_gc_mark_movable(p[i]); + } while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index 9049624529..1e8f3414ec 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -20,7 +20,7 @@ pub mod arm64; pub struct Label(pub usize); /// The object that knows how to encode the branch instruction. -type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64)>; +type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64) -> Result<(), ()>>; /// Reference to an ASM label pub struct LabelRef { @@ -233,7 +233,7 @@ impl CodeBlock { } // Add a label reference at the current write position - pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) + 'static) { + pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) -> Result<(), ()> + 'static) { assert!(label.0 < self.label_addrs.len()); // Keep track of the reference @@ -248,8 +248,9 @@ impl CodeBlock { } // Link internal label references - pub fn link_labels(&mut self) { + pub fn link_labels(&mut self) -> Result<(), ()> { let orig_pos = self.write_pos; + let mut link_result = Ok(()); // For each label reference for label_ref in mem::take(&mut self.label_refs) { @@ -261,11 +262,14 @@ impl CodeBlock { assert!(label_addr < self.mem_size); self.write_pos = ref_pos; - (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + let encode_result = (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + link_result = link_result.and(encode_result); - // Assert that we've written the same number of bytes that we - // expected to have written. - assert!(self.write_pos == ref_pos + label_ref.num_bytes); + // Verify number of bytes written when the callback returns Ok + if encode_result.is_ok() { + assert_eq!(self.write_pos, ref_pos + label_ref.num_bytes, "label_ref \ + callback didn't write number of bytes it claimed to write upfront"); + } } self.write_pos = orig_pos; @@ -274,6 +278,8 @@ impl CodeBlock { self.label_addrs.clear(); self.label_names.clear(); assert!(self.label_refs.is_empty()); + + link_result } /// Convert a Label to CodePtr diff --git a/zjit/src/asm/x86_64/mod.rs b/zjit/src/asm/x86_64/mod.rs index cfedca4540..0eeaae59dd 100644 --- a/zjit/src/asm/x86_64/mod.rs +++ b/zjit/src/asm/x86_64/mod.rs @@ -679,6 +679,7 @@ pub fn call_label(cb: &mut CodeBlock, label: Label) { cb.label_ref(label, 5, |cb, src_addr, dst_addr| { cb.write_byte(0xE8); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } @@ -795,6 +796,7 @@ fn write_jcc<const OP: u8>(cb: &mut CodeBlock, label: Label) { cb.write_byte(0x0F); cb.write_byte(OP); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } @@ -834,6 +836,7 @@ pub fn jmp_label(cb: &mut CodeBlock, label: Label) { cb.label_ref(label, 5, |cb, src_addr, dst_addr| { cb.write_byte(0xE9); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } diff --git a/zjit/src/asm/x86_64/tests.rs b/zjit/src/asm/x86_64/tests.rs index d574bdb034..268fe6b1c0 100644 --- a/zjit/src/asm/x86_64/tests.rs +++ b/zjit/src/asm/x86_64/tests.rs @@ -136,7 +136,7 @@ fn test_call_label() { let cb = compile(|cb| { let label_idx = cb.new_label("fn".to_owned()); call_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); assert_snapshot!(cb.hexdump(), @"e8fbffffff"); @@ -255,7 +255,7 @@ fn test_jge_label() { let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jge_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); assert_disasm_snapshot!(cb.disasm(), @" 0x0: jge 0"); assert_snapshot!(cb.hexdump(), @"0f8dfaffffff"); @@ -268,14 +268,14 @@ fn test_jmp_label() { let label_idx = cb.new_label("next".to_owned()); jmp_label(cb, label_idx); cb.write_label(label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); // Backwards jump let cb2 = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); cb.write_label(label_idx); jmp_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); assert_disasm_snapshot!(disasms!(cb1, cb2), @r" @@ -301,7 +301,7 @@ fn test_jo_label() { let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jo_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); assert_disasm_snapshot!(cb.disasm(), @" 0x0: jo 0"); diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 0ef22be631..d06e84536f 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -694,7 +694,8 @@ impl Assembler { /// VRegs, most splits should happen in [`Self::arm64_split`]. However, some instructions /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so this /// splits them and uses scratch registers for it. - fn arm64_scratch_split(mut self) -> Assembler { + /// Linearizes all blocks into a single giant block. + fn arm64_scratch_split(self) -> Assembler { /// If opnd is Opnd::Mem with a too large disp, make the disp smaller using lea. fn split_large_disp(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { match opnd { @@ -750,12 +751,23 @@ impl Assembler { // Prepare StackState to lower MemBase::Stack let stack_state = StackState::new(self.stack_base_idx); - let mut asm_local = Assembler::new_with_asm(&self); + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + let asm = &mut asm_local; - asm.accept_scratch_reg = true; - let iterator = &mut self.instruction_iterator(); - while let Some((_, mut insn)) = iterator.next(asm) { + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for (idx, insn) in linearized_insns.iter().enumerate() { + let mut insn = insn.clone(); match &mut insn { Insn::Add { left, right, out } | Insn::Sub { left, right, out } | @@ -795,7 +807,7 @@ impl Assembler { }; // If the next instruction is JoMul - if matches!(iterator.peek(), Some((_, Insn::JoMul(_)))) { + if idx + 1 < linearized_insns.len() && matches!(linearized_insns[idx + 1], Insn::JoMul(_)) { // Produce a register that is all zeros or all ones // Based on the sign bit of the 64-bit mul result asm.push_insn(Insn::RShift { out: SCRATCH0_OPND, opnd: reg_out, shift: Opnd::UImm(63) }); @@ -940,7 +952,7 @@ impl Assembler { /// Emit a conditional jump instruction to a specific target. This is /// called when lowering any of the conditional jump instructions. - fn emit_conditional_jump<const CONDITION: u8>(cb: &mut CodeBlock, target: Target) { + fn emit_conditional_jump<const CONDITION: u8>(asm: &Assembler, cb: &mut CodeBlock, target: Target) { fn generate_branch<const CONDITION: u8>(cb: &mut CodeBlock, src_addr: i64, dst_addr: i64) { let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { // If the jump offset fits into the conditional jump as @@ -991,23 +1003,31 @@ impl Assembler { (num_insns..cb.conditional_jump_insns()).for_each(|_| nop(cb)); } - match target { + let label = match target { Target::CodePtr(dst_ptr) => { let dst_addr = dst_ptr.as_offset(); let src_addr = cb.get_write_ptr().as_offset(); generate_branch::<CONDITION>(cb, src_addr, dst_addr); + return; }, - Target::Label(label_idx) => { - // We save `cb.conditional_jump_insns` number of bytes since we may use up to that amount - // `generate_branch` will pad the emitted branch instructions with `nop`s for each unused byte. - cb.label_ref(label_idx, (cb.conditional_jump_insns() * 4) as usize, |cb, src_addr, dst_addr| { - generate_branch::<CONDITION>(cb, src_addr - (cb.conditional_jump_insns() * 4) as i64, dst_addr); - }); - }, + Target::Label(l) => l, + Target::Block(ref edge) => asm.block_label(edge.target), Target::SideExit { .. } => { unreachable!("Target::SideExit should have been compiled by compile_exits") }, }; + // Try to use a single B.cond instruction + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if bcond_offset_fits_bits(offset) { + bcond(cb, CONDITION, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); } /// Emit a CBZ or CBNZ which branches when a register is zero or non-zero @@ -1110,8 +1130,13 @@ impl Assembler { let (_hook, mut hook_insn_idx) = AssemblerPanicHook::new(self, 0); // For each instruction + // NOTE: At this point, the assembler should have been linearized into a single giant block + // by either resolve_parallel_mov_pass() or arm64_scratch_split(). let mut insn_idx: usize = 0; - while let Some(insn) = self.insns.get(insn_idx) { + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { // Update insn_idx that is shown on panic hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); @@ -1215,7 +1240,7 @@ impl Assembler { }, Insn::Mul { left, right, out } => { // If the next instruction is JoMul with RShift created by arm64_scratch_split - match (self.insns.get(insn_idx + 1), self.insns.get(insn_idx + 2)) { + match (insns.get(insn_idx + 1), insns.get(insn_idx + 2)) { (Some(Insn::RShift { out: out_sign, opnd: out_opnd, shift: out_shift }), Some(Insn::JoMul(_))) => { // Compute the high 64 bits smulh(cb, Self::EMIT_OPND, left.into(), right.into()); @@ -1399,6 +1424,7 @@ impl Assembler { // Set output to the raw address of the label cb.label_ref(*label_idx, 4, |cb, end_addr, dst_addr| { adr(cb, Self::EMIT_OPND, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + Ok(()) }); mov(cb, out.into(), Self::EMIT_OPND); @@ -1412,33 +1438,19 @@ impl Assembler { Insn::CPush(opnd) => { emit_push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + // Second operand ends up at the lower stack address + stp_pre(cb, opnd1.into(), opnd0.into(), A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + }, Insn::CPop { out } => { emit_pop(cb, out.into()); }, Insn::CPopInto(opnd) => { emit_pop(cb, opnd.into()); }, - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - emit_push(cb, A64Opnd::Reg(reg)); - } - - // Push the flags/state register - mrs(cb, Self::EMIT_OPND, SystemRegister::NZCV); - emit_push(cb, Self::EMIT_OPND); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - // Pop the state/flags register - msr(cb, SystemRegister::NZCV, Self::EMIT_OPND); - emit_pop(cb, Self::EMIT_OPND); - - for reg in regs.into_iter().rev() { - emit_pop(cb, A64Opnd::Reg(reg)); - } + Insn::CPopPairInto(opnd0, opnd1) => { + // First operand is popped from the lower stack address + ldp_post(cb, opnd0.into(), opnd1.into(), A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); }, Insn::CCall { fptr, .. } => { match fptr { @@ -1480,14 +1492,31 @@ impl Assembler { emit_jmp_ptr(cb, dst_ptr, true); }, Target::Label(label_idx) => { - // Here we're going to save enough space for - // ourselves and then come back and write the - // instruction once we know the offset. We're going - // to assume we can fit into a single b instruction. - // It will panic otherwise. + // Reserve space for a single B instruction cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { - let bytes: i32 = (dst_addr - (src_addr - 4)).try_into().unwrap(); - b(cb, InstructionOffset::from_bytes(bytes)); + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::Block(ref edge) => { + let label = self.block_label(edge.target); + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } }); }, Target::SideExit { .. } => { @@ -1496,28 +1525,28 @@ impl Assembler { }; }, Insn::Je(target) | Insn::Jz(target) => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.clone()); + emit_conditional_jump::<{Condition::EQ}>(self, cb, target.clone()); }, Insn::Jne(target) | Insn::Jnz(target) | Insn::JoMul(target) => { - emit_conditional_jump::<{Condition::NE}>(cb, target.clone()); + emit_conditional_jump::<{Condition::NE}>(self, cb, target.clone()); }, Insn::Jl(target) => { - emit_conditional_jump::<{Condition::LT}>(cb, target.clone()); + emit_conditional_jump::<{Condition::LT}>(self, cb, target.clone()); }, Insn::Jg(target) => { - emit_conditional_jump::<{Condition::GT}>(cb, target.clone()); + emit_conditional_jump::<{Condition::GT}>(self, cb, target.clone()); }, Insn::Jge(target) => { - emit_conditional_jump::<{Condition::GE}>(cb, target.clone()); + emit_conditional_jump::<{Condition::GE}>(self, cb, target.clone()); }, Insn::Jbe(target) => { - emit_conditional_jump::<{Condition::LS}>(cb, target.clone()); + emit_conditional_jump::<{Condition::LS}>(self, cb, target.clone()); }, Insn::Jb(target) => { - emit_conditional_jump::<{Condition::CC}>(cb, target.clone()); + emit_conditional_jump::<{Condition::CC}>(self, cb, target.clone()); }, Insn::Jo(target) => { - emit_conditional_jump::<{Condition::VS}>(cb, target.clone()); + emit_conditional_jump::<{Condition::VS}>(self, cb, target.clone()); }, Insn::Joz(opnd, target) => { emit_cmp_zero_jump(cb, opnd.into(), true, target.clone()); @@ -1540,8 +1569,8 @@ impl Assembler { let Some(Insn::Cmp { left: status_reg @ Opnd::Reg(_), right: Opnd::UImm(_) | Opnd::Imm(_), - }) = self.insns.get(insn_idx + 1) else { - panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", self.insns.get(insn_idx + 1)); + }) = insns.get(insn_idx + 1) else { + panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", insns.get(insn_idx + 1)); }; // Attempt to increment a counter @@ -1590,7 +1619,7 @@ impl Assembler { } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { - if let Insn::PosMarker(callback) = self.insns.get(insn_idx).unwrap() { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { callback(pos, cb); } else { panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); @@ -1620,6 +1649,10 @@ impl Assembler { if use_scratch_reg { asm = asm.arm64_scratch_split(); asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = asm.resolve_parallel_mov_pass(); + asm_dump!(asm, resolve_parallel_mov); } // Create label instances in the code block @@ -1632,7 +1665,7 @@ impl Assembler { let gc_offsets = asm.arm64_emit(cb); if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { - cb.link_labels(); + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; // Invalidate icache for newly written out region so we don't run stale code. unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; @@ -1684,12 +1717,15 @@ mod tests { use super::*; use insta::assert_snapshot; + use crate::hir; static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; fn setup_asm() -> (Assembler, CodeBlock) { crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler - (Assembler::new(), CodeBlock::new_dummy()) + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) } #[test] @@ -1697,6 +1733,7 @@ mod tests { use crate::hir::SideExitReason; let mut asm = Assembler::new(); + asm.new_block_without_id(); asm.stack_base_idx = 1; let label = asm.new_label("bb0"); @@ -1749,6 +1786,29 @@ mod tests { } #[test] + fn test_conditional_branch_to_label() { + let (mut asm, mut cb) = setup_asm(); + let start = asm.new_label("start"); + let forward = asm.new_label("forward"); + + let value = asm.load(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0)); + asm.write_label(start.clone()); + asm.cmp(value, 0.into()); + asm.jg(forward.clone()); + asm.jl(start.clone()); + asm.write_label(forward); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [sp] + 0x4: cmp x0, #0 + 0x8: b.gt #0x10 + 0xc: b.lt #4 + "); + assert_snapshot!(cb.hexdump(), @"e00340f81f0000f14c000054cbffff54"); + } + + #[test] fn sp_movements_are_single_instruction() { let (mut asm, mut cb) = setup_asm(); @@ -1848,50 +1908,6 @@ mod tests { } #[test] - fn test_emit_cpush_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpush_all(); - asm.compile_with_num_regs(&mut cb, 0); - - assert_disasm_snapshot!(cb.disasm(), @r" - 0x0: str x1, [sp, #-0x10]! - 0x4: str x9, [sp, #-0x10]! - 0x8: str x10, [sp, #-0x10]! - 0xc: str x11, [sp, #-0x10]! - 0x10: str x12, [sp, #-0x10]! - 0x14: str x13, [sp, #-0x10]! - 0x18: str x14, [sp, #-0x10]! - 0x1c: str x15, [sp, #-0x10]! - 0x20: mrs x16, nzcv - 0x24: str x16, [sp, #-0x10]! - "); - assert_snapshot!(cb.hexdump(), @"e10f1ff8e90f1ff8ea0f1ff8eb0f1ff8ec0f1ff8ed0f1ff8ee0f1ff8ef0f1ff810423bd5f00f1ff8"); - } - - #[test] - fn test_emit_cpop_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpop_all(); - asm.compile_with_num_regs(&mut cb, 0); - - assert_disasm_snapshot!(cb.disasm(), @r" - 0x0: msr nzcv, x16 - 0x4: ldr x16, [sp], #0x10 - 0x8: ldr x15, [sp], #0x10 - 0xc: ldr x14, [sp], #0x10 - 0x10: ldr x13, [sp], #0x10 - 0x14: ldr x12, [sp], #0x10 - 0x18: ldr x11, [sp], #0x10 - 0x1c: ldr x10, [sp], #0x10 - 0x20: ldr x9, [sp], #0x10 - 0x24: ldr x1, [sp], #0x10 - "); - assert_snapshot!(cb.hexdump(), @"10421bd5f00741f8ef0741f8ee0741f8ed0741f8ec0741f8eb0741f8ea0741f8e90741f8e10741f8"); - } - - #[test] fn test_emit_frame() { let (mut asm, mut cb) = setup_asm(); @@ -2131,6 +2147,7 @@ mod tests { #[test] fn test_store_with_valid_scratch_reg() { let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); let mut cb = CodeBlock::new_dummy(); asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); @@ -2571,7 +2588,7 @@ mod tests { } #[test] - fn test_label_branch_generate_bounds() { + fn test_exceeding_label_branch_generate_bounds() { // The immediate in a conditional branch is a 19 bit unsigned integer // which has a max value of 2^18 - 1. const IMMEDIATE_MAX_VALUE: usize = 2usize.pow(18) - 1; @@ -2582,7 +2599,9 @@ mod tests { let page_size = unsafe { rb_jit_get_page_size() } as usize; let memory_required = (IMMEDIATE_MAX_VALUE + 8) * 4 + page_size; + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler let mut asm = Assembler::new(); + asm.new_block_without_id(); let mut cb = CodeBlock::new_dummy_sized(memory_required); let far_label = asm.new_label("far"); @@ -2595,7 +2614,7 @@ mod tests { }); asm.write_label(far_label.clone()); - asm.compile_with_num_regs(&mut cb, 1); + assert_eq!(Err(CompileError::LabelLinkingFailure), asm.compile(&mut cb)); } #[test] @@ -2694,6 +2713,76 @@ mod tests { } #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, x0 + 0x14: stp x2, x1, [sp, #-0x10]! + 0x18: stp x4, x3, [sp, #-0x10]! + 0x1c: mov x16, #0 + 0x20: blr x16 + 0x24: ldp x4, x3, [sp], #0x10 + 0x28: ldp x2, x1, [sp], #0x10 + 0x2c: adds x4, x4, x1 + 0x30: adds x2, x2, x3 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2e40300aae207bfa9e40fbfa9100080d200023fd6e40fc1a8e207c1a8840001ab420003ab"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, #5 + 0x14: mov x5, x0 + 0x18: stp x2, x1, [sp, #-0x10]! + 0x1c: stp x4, x3, [sp, #-0x10]! + 0x20: str x5, [sp, #-0x10]! + 0x24: mov x16, #0 + 0x28: blr x16 + 0x2c: ldr x5, [sp], #0x10 + 0x30: ldp x4, x3, [sp], #0x10 + 0x34: ldp x2, x1, [sp], #0x10 + 0x38: adds x5, x5, x1 + 0x3c: adds x0, x2, x3 + 0x40: adds x2, x2, x4 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2a40080d2e50300aae207bfa9e40fbfa9e50f1ff8100080d200023fd6e50741f8e40fc1a8e207c1a8a50001ab400003ab420004ab"); + } + + #[test] fn test_ccall_resolve_parallel_moves_large_cycle() { let (mut asm, mut cb) = setup_asm(); @@ -2717,6 +2806,38 @@ mod tests { } #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: stp x1, x0, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e103bfa9"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: ldp x0, x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e007c1a8"); + } + + #[test] fn test_split_spilled_lshift() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index d8d82a09ca..f2f7bc6165 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex}; use crate::codegen::local_size_and_idx_to_ep_offset; use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; use crate::hir::{Invariant, SideExitReason}; +use crate::hir; use crate::options::{TraceExits, debug, get_option}; use crate::cruby::VALUE; use crate::payload::IseqVersionRef; @@ -15,6 +16,104 @@ use crate::virtualmem::CodePtr; use crate::asm::{CodeBlock, Label}; use crate::state::rb_zjit_record_exit_stack; +/// LIR Block ID. Unique ID for each block, and also defined in LIR so +/// we can differentiate it from HIR block ids. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "l{}", self.0) + } +} + +/// Dummy HIR block ID used when creating test or invalid LIR blocks +const DUMMY_HIR_BLOCK_ID: usize = usize::MAX; +/// Dummy RPO index used when creating test or invalid LIR blocks +const DUMMY_RPO_INDEX: usize = usize::MAX; + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<Opnd>, +} + +#[derive(Clone, Debug)] +pub struct BasicBlock { + // Unique id for this block + pub id: BlockId, + + // HIR block this LIR block was lowered from. Not injective: multiple LIR blocks may share + // the same hir_block_id because we split HIR blocks into multiple LIR blocks during lowering. + pub hir_block_id: hir::BlockId, + + pub is_entry: bool, + + // Instructions in this basic block + pub insns: Vec<Insn>, + + // Input parameters for this block + pub parameters: Vec<Opnd>, + + // RPO position of the source HIR block + pub rpo_index: usize, +} + +pub struct EdgePair(Option<BranchEdge>, Option<BranchEdge>); + +impl BasicBlock { + fn new(id: BlockId, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> Self { + Self { + id, + hir_block_id, + is_entry, + insns: vec![], + parameters: vec![], + rpo_index, + } + } + + pub fn add_parameter(&mut self, param: Opnd) { + self.parameters.push(param); + } + + pub fn push_insn(&mut self, insn: Insn) { + self.insns.push(insn); + } + + pub fn edges(&self) -> EdgePair { + assert!(self.insns.last().unwrap().is_terminator()); + let extract_edge = |insn: &Insn| -> Option<BranchEdge> { + if let Some(Target::Block(edge)) = insn.target() { + Some(edge.clone()) + } else { + None + } + }; + + match self.insns.as_slice() { + [] => panic!("empty block"), + [.., second_last, last] => { + EdgePair(extract_edge(second_last), extract_edge(last)) + }, + [.., last] => { + EdgePair(extract_edge(last), None) + } + } + } + + /// Sort key for scheduling blocks in code layout order + pub fn sort_key(&self) -> (usize, usize) { + (self.rpo_index, self.id.0) + } +} + pub use crate::backend::current::{ mem_base_reg, Reg, @@ -309,13 +408,15 @@ pub struct SideExit { /// Branch target (something that we can jump to) /// for branch instructions -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum Target { /// Pointer to a piece of ZJIT-generated code CodePtr(CodePtr), /// A label within the generated code Label(Label), + /// An LIR branch edge + Block(BranchEdge), /// Side exit to the interpreter SideExit { /// Context used for compiling the side exit @@ -325,6 +426,32 @@ pub enum Target }, } +impl fmt::Debug for Target { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Target::CodePtr(ptr) => write!(f, "CodePtr({:?})", ptr), + Target::Label(label) => write!(f, "Label({:?})", label), + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, "Block({:?})", edge.target) + } else { + write!(f, "Block({:?}(", edge.target)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}", arg)?; + } + write!(f, "))") + } + } + Target::SideExit { exit, reason } => { + write!(f, "SideExit {{ exit: {:?}, reason: {:?} }}", exit, reason) + } + } + } +} + impl Target { pub fn unwrap_label(&self) -> Label { @@ -377,17 +504,19 @@ pub enum Insn { /// Pop a register from the C stack CPop { out: Opnd }, - /// Pop all of the caller-save registers and the flags from the C stack - CPopAll, - /// Pop a register from the C stack and store it into another register CPopInto(Opnd), + /// Pop a pair of registers from the C stack and store it into a pair of registers. + /// The registers are popped from left to right. + CPopPairInto(Opnd, Opnd), + /// Push a register onto the C stack CPush(Opnd), - /// Push all of the caller-save registers and the flags to the C stack - CPushAll, + /// Push a pair of registers onto the C stack. + /// The registers are pushed from left to right. + CPushPair(Opnd, Opnd), // C function call with N arguments (variadic) CCall { @@ -614,10 +743,10 @@ impl Insn { Insn::Comment(_) => "Comment", Insn::Cmp { .. } => "Cmp", Insn::CPop { .. } => "CPop", - Insn::CPopAll => "CPopAll", Insn::CPopInto(_) => "CPopInto", + Insn::CPopPairInto(_, _) => "CPopPairInto", Insn::CPush(_) => "CPush", - Insn::CPushAll => "CPushAll", + Insn::CPushPair(_, _) => "CPushPair", Insn::CCall { .. } => "CCall", Insn::CRet(_) => "CRet", Insn::CSelE { .. } => "CSelE", @@ -769,6 +898,29 @@ impl Insn { _ => None } } + + /// Returns true if this instruction is a terminator (ends a basic block). + pub fn is_terminator(&self) -> bool { + match self { + Insn::Jbe(_) | + Insn::Jb(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jg(_) | + Insn::Jge(_) | + Insn::Jmp(_) | + Insn::JmpOpnd(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::JoMul(_) | + Insn::Jz(_) | + Insn::Joz(..) | + Insn::Jonz(..) | + Insn::CRet(_) => true, + _ => false + } + } } /// An iterator that will yield a non-mutable reference to each operand in turn @@ -804,22 +956,33 @@ impl<'a> Iterator for InsnOpndIterator<'a> { Insn::Label(target) | Insn::LeaJumpTarget { target, .. } | Insn::PatchPoint { target, .. } => { - if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { - let stack_idx = self.idx; - if stack_idx < stack.len() { - let opnd = &stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } - let local_idx = self.idx - stack.len(); - if local_idx < locals.len() { - let opnd = &locals[local_idx]; - self.idx += 1; - return Some(opnd); + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + if self.idx < edge.args.len() { + let opnd = &edge.args[self.idx]; + self.idx += 1; + return Some(opnd); + } + None } + _ => None } - None } Insn::Joz(opnd, target) | @@ -829,30 +992,40 @@ impl<'a> Iterator for InsnOpndIterator<'a> { return Some(opnd); } - if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { - let stack_idx = self.idx - 1; - if stack_idx < stack.len() { - let opnd = &stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } - let local_idx = stack_idx - stack.len(); - if local_idx < locals.len() { - let opnd = &locals[local_idx]; - self.idx += 1; - return Some(opnd); + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + let arg_idx = self.idx - 1; + if arg_idx < edge.args.len() { + let opnd = &edge.args[arg_idx]; + self.idx += 1; + return Some(opnd); + } + None } + _ => None } - None } Insn::BakeString(_) | Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | Insn::PadPatchPoint | Insn::PosMarker(_) => None, @@ -875,6 +1048,8 @@ impl<'a> Iterator for InsnOpndIterator<'a> { }, Insn::Add { left: opnd0, right: opnd1, .. } | Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | Insn::Cmp { left: opnd0, right: opnd1 } | Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | @@ -973,22 +1148,33 @@ impl<'a> InsnOpndMutIterator<'a> { Insn::Label(target) | Insn::LeaJumpTarget { target, .. } | Insn::PatchPoint { target, .. } => { - if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { - let stack_idx = self.idx; - if stack_idx < stack.len() { - let opnd = &mut stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } - let local_idx = self.idx - stack.len(); - if local_idx < locals.len() { - let opnd = &mut locals[local_idx]; - self.idx += 1; - return Some(opnd); + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + if self.idx < edge.args.len() { + let opnd = &mut edge.args[self.idx]; + self.idx += 1; + return Some(opnd); + } + None } + _ => None } - None } Insn::Joz(opnd, target) | @@ -998,30 +1184,40 @@ impl<'a> InsnOpndMutIterator<'a> { return Some(opnd); } - if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { - let stack_idx = self.idx - 1; - if stack_idx < stack.len() { - let opnd = &mut stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } + match target { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } - let local_idx = stack_idx - stack.len(); - if local_idx < locals.len() { - let opnd = &mut locals[local_idx]; - self.idx += 1; - return Some(opnd); + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + None + } + Target::Block(edge) => { + let arg_idx = self.idx - 1; + if arg_idx < edge.args.len() { + let opnd = &mut edge.args[arg_idx]; + self.idx += 1; + return Some(opnd); + } + None } + _ => None } - None } Insn::BakeString(_) | Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | Insn::FrameSetup { .. } | Insn::FrameTeardown { .. } | Insn::PadPatchPoint | @@ -1046,6 +1242,8 @@ impl<'a> InsnOpndMutIterator<'a> { }, Insn::Add { left: opnd0, right: opnd1, .. } | Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | Insn::Cmp { left: opnd0, right: opnd1 } | Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | @@ -1330,7 +1528,12 @@ const ASSEMBLER_INSNS_CAPACITY: usize = 256; /// optimized and lowered #[derive(Clone)] pub struct Assembler { - pub(super) insns: Vec<Insn>, + pub basic_blocks: Vec<BasicBlock>, + + /// The block to which new instructions are added. Used during HIR to LIR lowering to + /// determine which LIR block we should add instructions to. Set by `set_current_block()` + /// and automatically set to new entry blocks created by `new_block()`. + current_block_id: BlockId, /// Live range for each VReg indexed by its `idx`` pub(super) live_ranges: Vec<LiveRange>, @@ -1348,7 +1551,10 @@ pub struct Assembler { pub(super) stack_base_idx: usize, /// If Some, the next ccall should verify its leafness - leaf_ccall_stack_size: Option<usize> + leaf_ccall_stack_size: Option<usize>, + + /// Current instruction index, incremented for each instruction pushed + idx: usize, } impl Assembler @@ -1356,12 +1562,14 @@ impl Assembler /// Create an Assembler with defaults pub fn new() -> Self { Self { - insns: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), - live_ranges: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), label_names: Vec::default(), accept_scratch_reg: false, stack_base_idx: 0, leaf_ccall_stack_size: None, + basic_blocks: Vec::default(), + current_block_id: BlockId(0), + live_ranges: Vec::default(), + idx: 0, } } @@ -1385,11 +1593,62 @@ impl Assembler stack_base_idx: old_asm.stack_base_idx, ..Self::new() }; - // Bump the initial VReg index to allow the use of the VRegs for the old Assembler + + // Initialize basic blocks from the old assembler, preserving hir_block_id and entry flag + // but with empty instruction lists + for old_block in &old_asm.basic_blocks { + asm.new_block_from_old_block(&old_block); + } + + // Initialize live_ranges to match the old assembler's size + // This allows reusing VRegs from the old assembler asm.live_ranges.resize(old_asm.live_ranges.len(), LiveRange { start: None, end: None }); + asm } + // Create a new LIR basic block. Returns the newly created block ID + pub fn new_block(&mut self, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, hir_block_id, is_entry, rpo_index); + self.basic_blocks.push(lir_bb); + if is_entry { + self.set_current_block(bb_id); + } + bb_id + } + + // Create a new LIR basic block from an old one. This should only be used + // when creating new assemblers during passes when we want to translate + // one assembler to a new one. + pub fn new_block_from_old_block(&mut self, old_block: &BasicBlock) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, old_block.hir_block_id, old_block.is_entry, old_block.rpo_index); + self.basic_blocks.push(lir_bb); + bb_id + } + + // Create a LIR basic block without a valid HIR block ID (for testing or internal use). + pub fn new_block_without_id(&mut self) -> BlockId { + self.new_block(hir::BlockId(DUMMY_HIR_BLOCK_ID), true, DUMMY_RPO_INDEX) + } + + pub fn set_current_block(&mut self, block_id: BlockId) { + self.current_block_id = block_id; + } + + pub fn current_block(&mut self) -> &mut BasicBlock { + &mut self.basic_blocks[self.current_block_id.0] + } + + /// Return basic blocks sorted by RPO index, then by block ID. + /// TODO: Use a more advanced scheduling algorithm + pub fn sorted_blocks(&self) -> Vec<&BasicBlock> { + let mut sorted: Vec<&BasicBlock> = self.basic_blocks.iter().collect(); + sorted.sort_by_key(|block| block.sort_key()); + sorted + } + /// Return true if `opnd` is or depends on `reg` pub fn has_reg(opnd: Opnd, reg: Reg) -> bool { match opnd { @@ -1400,11 +1659,100 @@ impl Assembler } pub fn instruction_iterator(&mut self) -> InsnIter { - let insns = take(&mut self.insns); - InsnIter { - old_insns_iter: insns.into_iter(), + let mut blocks = take(&mut self.basic_blocks); + blocks.sort_by_key(|block| block.sort_key()); + + let mut iter = InsnIter { + blocks, + current_block_idx: 0, + current_insn_iter: vec![].into_iter(), // Will be replaced immediately peeked: None, index: 0, + }; + + // Set up first block's iterator + if !iter.blocks.is_empty() { + iter.current_insn_iter = take(&mut iter.blocks[0].insns).into_iter(); + } + + iter + } + + /// Return an operand for a basic block argument at a given index. + /// To simplify the implementation, we allocate a fixed register or a stack slot + /// for each basic block argument. + pub fn param_opnd(idx: usize) -> Opnd { + use crate::backend::current::ALLOC_REGS; + use crate::cruby::SIZEOF_VALUE_I32; + + if idx < ALLOC_REGS.len() { + Opnd::Reg(ALLOC_REGS[idx]) + } else { + // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register. + // To avoid clobbering it, we need to start from the next slot, hence `+ 1` for the index. + Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) + } + } + + pub fn linearize_instructions(&self) -> Vec<Insn> { + // Emit instructions with labels, expanding branch parameters + let mut insns = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); + + for block in self.sorted_blocks() { + // Process each instruction, expanding branch params if needed + for insn in &block.insns { + self.expand_branch_insn(insn, &mut insns); + } + } + insns + } + + /// Expand and linearize a branch instruction: + /// 1. If the branch has Target::Block with arguments, insert a ParallelMov first + /// 2. Convert Target::Block to Target::Label + /// 3. Push the converted instruction + fn expand_branch_insn(&self, insn: &Insn, insns: &mut Vec<Insn>) { + // Helper to process branch arguments and return the label target + let mut process_edge = |edge: &BranchEdge| -> Label { + if !edge.args.is_empty() { + insns.push(Insn::ParallelMov { + moves: edge.args.iter().enumerate() + .map(|(idx, &arg)| (Assembler::param_opnd(idx), arg)) + .collect() + }); + } + self.block_label(edge.target) + }; + + // Convert Target::Block to Target::Label, processing args if needed + let stripped_insn = match insn { + Insn::Jmp(Target::Block(edge)) => Insn::Jmp(Target::Label(process_edge(edge))), + Insn::Jz(Target::Block(edge)) => Insn::Jz(Target::Label(process_edge(edge))), + Insn::Jnz(Target::Block(edge)) => Insn::Jnz(Target::Label(process_edge(edge))), + Insn::Je(Target::Block(edge)) => Insn::Je(Target::Label(process_edge(edge))), + Insn::Jne(Target::Block(edge)) => Insn::Jne(Target::Label(process_edge(edge))), + Insn::Jl(Target::Block(edge)) => Insn::Jl(Target::Label(process_edge(edge))), + Insn::Jg(Target::Block(edge)) => Insn::Jg(Target::Label(process_edge(edge))), + Insn::Jge(Target::Block(edge)) => Insn::Jge(Target::Label(process_edge(edge))), + Insn::Jbe(Target::Block(edge)) => Insn::Jbe(Target::Label(process_edge(edge))), + Insn::Jb(Target::Block(edge)) => Insn::Jb(Target::Label(process_edge(edge))), + Insn::Jo(Target::Block(edge)) => Insn::Jo(Target::Label(process_edge(edge))), + Insn::JoMul(Target::Block(edge)) => Insn::JoMul(Target::Label(process_edge(edge))), + Insn::Joz(opnd, Target::Block(edge)) => Insn::Joz(*opnd, Target::Label(process_edge(edge))), + Insn::Jonz(opnd, Target::Block(edge)) => Insn::Jonz(*opnd, Target::Label(process_edge(edge))), + _ => insn.clone() + }; + + // Push the stripped instruction + insns.push(stripped_insn); + } + + // Get the label for a given block by extracting it from the first instruction. + pub(super) fn block_label(&self, block_id: BlockId) -> Label { + let block = &self.basic_blocks[block_id.0]; + match block.insns.first() { + Some(Insn::Label(Target::Label(label))) => *label, + other => panic!("Expected first instruction of block {:?} to be a Label, but found: {:?}", block_id, other), } } @@ -1442,7 +1790,7 @@ impl Assembler /// operands to this instruction. pub fn push_insn(&mut self, insn: Insn) { // Index of this instruction - let insn_idx = self.insns.len(); + let insn_idx = self.idx; // Initialize the live range of the output VReg to insn_idx..=insn_idx if let Some(Opnd::VReg { idx, .. }) = insn.out_opnd() { @@ -1473,7 +1821,9 @@ impl Assembler } } - self.insns.push(insn); + self.idx += 1; + + self.current_block().push_insn(insn); } /// Create a new label instance that we can jump to @@ -1531,6 +1881,7 @@ impl Assembler Some(new_moves) } + /// Sets the out field on the various instructions that require allocated /// registers because their output is used as the operand on a subsequent /// instruction. This is our implementation of the linear scan algorithm. @@ -1546,17 +1897,22 @@ impl Assembler let mut saved_regs: Vec<(Reg, usize)> = vec![]; // Remember the indexes of Insn::FrameSetup to update the stack size later - let mut frame_setup_idxs: Vec<usize> = vec![]; + let mut frame_setup_idxs: Vec<(BlockId, usize)> = vec![]; // live_ranges is indexed by original `index` given by the iterator. - let mut asm = Assembler::new_with_asm(&self); + let mut asm_local = Assembler::new_with_asm(&self); + + let iterator = &mut self.instruction_iterator(); + + let asm = &mut asm_local; + let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - while let Some((index, mut insn)) = iterator.next() { + while let Some((index, mut insn)) = iterator.next(asm) { // Remember the index of FrameSetup to bump slot_count when we know the max number of spilled VRegs. if let Insn::FrameSetup { .. } = insn { - frame_setup_idxs.push(asm.insns.len()); + assert!(asm.current_block().is_entry); + frame_setup_idxs.push((asm.current_block().id, asm.current_block().insns.len())); } let before_ccall = match (&insn, iterator.peek().map(|(_, insn)| insn)) { @@ -1604,9 +1960,19 @@ impl Assembler saved_regs = pool.live_regs(); // Save live registers - for &(reg, _) in saved_regs.iter() { - asm.cpush(Opnd::Reg(reg)); - pool.dealloc_opnd(&Opnd::Reg(reg)); + for pair in saved_regs.chunks(2) { + match *pair { + [(reg0, _), (reg1, _)] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + pool.dealloc_opnd(&Opnd::Reg(reg0)); + pool.dealloc_opnd(&Opnd::Reg(reg1)); + } + [(reg, _)] => { + asm.cpush(Opnd::Reg(reg)); + pool.dealloc_opnd(&Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } } // On x86_64, maintain 16-byte stack alignment if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { @@ -1703,17 +2069,6 @@ impl Assembler // Push instruction(s) let is_ccall = matches!(insn, Insn::CCall { .. }); match insn { - Insn::ParallelMov { moves } => { - // For trampolines that use scratch registers, attempt to lower ParallelMov without scratch_reg. - if let Some(moves) = Self::resolve_parallel_moves(&moves, None) { - for (dst, src) in moves { - asm.mov(dst, src); - } - } else { - // If it needs a scratch_reg, leave it to *_split_with_scratch_regs to handle it. - asm.push_insn(Insn::ParallelMov { moves }); - } - } Insn::CCall { opnds, fptr, start_marker, end_marker, out } => { // Split start_marker and end_marker here to avoid inserting push/pop between them. if let Some(start_marker) = start_marker { @@ -1737,17 +2092,27 @@ impl Assembler asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0)); } // Restore saved registers - for &(reg, vreg_idx) in saved_regs.iter().rev() { - asm.cpop_into(Opnd::Reg(reg)); - pool.take_reg(®, vreg_idx); + for pair in saved_regs.chunks(2).rev() { + match *pair { + [(reg, vreg_idx)] => { + asm.cpop_into(Opnd::Reg(reg)); + pool.take_reg(®, vreg_idx); + } + [(reg0, vreg_idx0), (reg1, vreg_idx1)] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + pool.take_reg(®1, vreg_idx1); + pool.take_reg(®0, vreg_idx0); + } + _ => unreachable!("chunks(2)") + } } saved_regs.clear(); } } // Extend the stack space for spilled operands - for frame_setup_idx in frame_setup_idxs { - match &mut asm.insns[frame_setup_idx] { + for (block_id, frame_setup_idx) in frame_setup_idxs { + match &mut asm.basic_blocks[block_id.0].insns[frame_setup_idx] { Insn::FrameSetup { slot_count, .. } => { *slot_count += pool.stack_state.stack_size; } @@ -1756,7 +2121,7 @@ impl Assembler } assert!(pool.is_empty(), "Expected all registers to be returned to the pool"); - Ok(asm) + Ok(asm_local) } /// Compile the instructions down to machine code. @@ -1830,16 +2195,19 @@ impl Assembler // Extract targets first so that we can update instructions while referencing part of them. let mut targets = HashMap::new(); - for (idx, insn) in self.insns.iter().enumerate() { - if let Some(target @ Target::SideExit { .. }) = insn.target() { - targets.insert(idx, target.clone()); + + for block in self.sorted_blocks().iter() { + for (idx, insn) in block.insns.iter().enumerate() { + if let Some(target @ Target::SideExit { .. }) = insn.target() { + targets.insert((block.id.0, idx), target.clone()); + } } } // Map from SideExit to compiled Label. This table is used to deduplicate side exit code. let mut compiled_exits: HashMap<SideExit, Label> = HashMap::new(); - for (idx, target) in targets { + for ((block_id, idx), target) in targets { // Compile a side exit. Note that this is past the split pass and alloc_regs(), // so you can't use an instruction that returns a VReg. if let Target::SideExit { exit: exit @ SideExit { pc, .. }, reason } = target { @@ -1867,10 +2235,7 @@ impl Assembler } if should_record_exit { - // Preserve caller-saved registers that may be used in the shared exit. - self.cpush_all(); asm_ccall!(self, rb_zjit_record_exit_stack, pc); - self.cpop_all(); } // If the side exit has already been compiled, jump to it. @@ -1895,7 +2260,7 @@ impl Assembler new_exit }; - *self.insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); + *self.basic_blocks[block_id].insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); } } } @@ -1930,7 +2295,7 @@ impl fmt::Display for Assembler { } } - for insn in self.insns.iter() { + for insn in self.linearize_instructions().iter() { match insn { Insn::Comment(comment) => { writeln!(f, " {bold_begin}# {comment}{bold_end}")?; @@ -1966,6 +2331,20 @@ impl fmt::Display for Assembler { Target::CodePtr(code_ptr) => write!(f, " {code_ptr:?}")?, Target::Label(Label(label_idx)) => write!(f, " {}", label_name(self, *label_idx, &label_counts))?, Target::SideExit { reason, .. } => write!(f, " Exit({reason})")?, + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, " bb{}", edge.target.0)?; + } else { + write!(f, " bb{}(", edge.target.0)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + write!(f, ")")?; + } + } } } @@ -1981,6 +2360,17 @@ impl fmt::Display for Assembler { } _ => {} } + } else if let Some(Target::Block(_)) = insn.target() { + // If the instruction has a Block target, avoid using opnd_iter() for branch args + // since they're already printed inline with the target. Only print non-target operands. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } } else if let Insn::ParallelMov { moves } = insn { // Print operands with a special syntax for ParallelMov moves.iter().try_fold(" ", |prefix, (dst, src)| write!(f, "{prefix}{dst} <- {src}").and(Ok(", ")))?; @@ -2000,7 +2390,7 @@ impl fmt::Debug for Assembler { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { writeln!(fmt, "Assembler")?; - for (idx, insn) in self.insns.iter().enumerate() { + for (idx, insn) in self.linearize_instructions().iter().enumerate() { writeln!(fmt, " {idx:03} {insn:?}")?; } @@ -2009,7 +2399,9 @@ impl fmt::Debug for Assembler { } pub struct InsnIter { - old_insns_iter: std::vec::IntoIter<Insn>, + blocks: Vec<BasicBlock>, + current_block_idx: usize, + current_insn_iter: std::vec::IntoIter<Insn>, peeked: Option<(usize, Insn)>, index: usize, } @@ -2020,7 +2412,7 @@ impl InsnIter { pub fn peek(&mut self) -> Option<&(usize, Insn)> { // If we don't have a peeked value, get one if self.peeked.is_none() { - let insn = self.old_insns_iter.next()?; + let insn = self.current_insn_iter.next()?; let idx = self.index; self.index += 1; self.peeked = Some((idx, insn)); @@ -2029,17 +2421,34 @@ impl InsnIter { self.peeked.as_ref() } - // Get the next instruction. Right now we're passing the "new" assembler - // (the assembler we're copying in to) as a parameter. Once we've - // introduced basic blocks to LIR, we'll use the to set the correct BB - // on the new assembler, but for now it is unused. - pub fn next(&mut self, _new_asm: &mut Assembler) -> Option<(usize, Insn)> { + // Get the next instruction, advancing to the next block when current block is exhausted. + // Sets the current block on new_asm when moving to a new block. + pub fn next(&mut self, new_asm: &mut Assembler) -> Option<(usize, Insn)> { // If we have a peeked value, return it if let Some(item) = self.peeked.take() { return Some(item); } - // Otherwise get the next from underlying iterator - let insn = self.old_insns_iter.next()?; + + // Try to get the next instruction from current block + if let Some(insn) = self.current_insn_iter.next() { + let idx = self.index; + self.index += 1; + return Some((idx, insn)); + } + + // Current block is exhausted, move to next block + self.current_block_idx += 1; + if self.current_block_idx >= self.blocks.len() { + return None; + } + + // Set up the next block + let next_block = &mut self.blocks[self.current_block_idx]; + new_asm.set_current_block(next_block.id); + self.current_insn_iter = take(&mut next_block.insns).into_iter(); + + // Get first instruction from the new block + let insn = self.current_insn_iter.next()?; let idx = self.index; self.index += 1; Some((idx, insn)) @@ -2135,21 +2544,27 @@ impl Assembler { out } - pub fn cpop_all(&mut self) { - self.push_insn(Insn::CPopAll); - } - pub fn cpop_into(&mut self, opnd: Opnd) { assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); self.push_insn(Insn::CPopInto(opnd)); } + #[track_caller] + pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); + } + pub fn cpush(&mut self, opnd: Opnd) { self.push_insn(Insn::CPush(opnd)); } - pub fn cpush_all(&mut self) { - self.push_insn(Insn::CPushAll); + #[track_caller] + pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPushPair(opnd0, opnd1)); } pub fn cret(&mut self, opnd: Opnd) { @@ -2426,6 +2841,43 @@ impl Assembler { self.push_insn(Insn::Xor { left, right, out }); out } + + /// This is used for trampolines that don't allow scratch registers. + /// Linearizes all blocks into a single giant block. + pub fn resolve_parallel_mov_pass(self) -> Assembler { + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = self.accept_scratch_reg; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for insn in linearized_insns { + match insn { + Insn::ParallelMov { moves } => { + // Resolve parallel moves without scratch register + if let Some(resolved_moves) = Assembler::resolve_parallel_moves(&moves, None) { + for (dst, src) in resolved_moves { + asm_local.mov(dst, src); + } + } else { + unreachable!("ParallelMov requires scratch register but scratch_reg is not allowed"); + } + } + _ => { + asm_local.push_insn(insn); + } + } + } + + asm_local + } } /// Macro to use format! for Insn::Comment, which skips a format! call diff --git a/zjit/src/backend/tests.rs b/zjit/src/backend/tests.rs index ece6f8605f..701029b8ec 100644 --- a/zjit/src/backend/tests.rs +++ b/zjit/src/backend/tests.rs @@ -3,10 +3,12 @@ use crate::backend::lir::*; use crate::cruby::*; use crate::codegen::c_callable; use crate::options::rb_zjit_prepare_options; +use crate::hir; #[test] fn test_add() { let mut asm = Assembler::new(); + asm.new_block_without_id(); let out = asm.add(SP, Opnd::UImm(1)); let _ = asm.add(out, Opnd::UImm(2)); } @@ -15,6 +17,7 @@ fn test_add() { fn test_alloc_regs() { rb_zjit_prepare_options(); // for asm.alloc_regs let mut asm = Assembler::new(); + asm.new_block_without_id(); // Get the first output that we're going to reuse later. let out1 = asm.add(EC, Opnd::UImm(1)); @@ -37,7 +40,7 @@ fn test_alloc_regs() { let _ = asm.add(out3, Opnd::UImm(6)); // Here we're going to allocate the registers. - let result = asm.alloc_regs(Assembler::get_alloc_regs()).unwrap(); + let result = &asm.alloc_regs(Assembler::get_alloc_regs()).unwrap().basic_blocks[0]; // Now we're going to verify that the out field has been appropriately // updated for each of the instructions that needs it. @@ -63,7 +66,9 @@ fn test_alloc_regs() { fn setup_asm() -> (Assembler, CodeBlock) { rb_zjit_prepare_options(); // for get_option! on asm.compile - (Assembler::new(), CodeBlock::new_dummy()) + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) } // Test full codegen pipeline @@ -293,6 +298,7 @@ fn test_no_pos_marker_callback_when_compile_fails() { // We don't want to invoke the pos_marker callbacks with positions of malformed code. let mut asm = Assembler::new(); rb_zjit_prepare_options(); // for asm.compile + asm.new_block_without_id(); // Markers around code to exhaust memory limit let fail_if_called = |_code_ptr, _cb: &_| panic!("pos_marker callback should not be called"); diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 9f780617cc..a4cf8dfcc5 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -392,7 +392,7 @@ impl Assembler { /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so /// this splits them and uses scratch registers for it. - pub fn x86_scratch_split(mut self) -> Assembler { + pub fn x86_scratch_split(self) -> Assembler { /// For some instructions, we want to be able to lower a 64-bit operand /// without requiring more registers to be available in the register /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold @@ -468,12 +468,22 @@ impl Assembler { // Prepare StackState to lower MemBase::Stack let stack_state = StackState::new(self.stack_base_idx); - let mut asm_local = Assembler::new_with_asm(&self); + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.live_ranges.resize(self.live_ranges.len(), LiveRange { start: None, end: None }); + + // Create one giant block to linearize everything into + asm_local.new_block_without_id(); + let asm = &mut asm_local; - asm.accept_scratch_reg = true; - let mut iterator = self.instruction_iterator(); - while let Some((_, mut insn)) = iterator.next(asm) { + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + for insn in linearized_insns.iter() { + let mut insn = insn.clone(); match &mut insn { Insn::Add { left, right, out } | Insn::Sub { left, right, out } | @@ -703,7 +713,10 @@ impl Assembler { // For each instruction let mut insn_idx: usize = 0; - while let Some(insn) = self.insns.get(insn_idx) { + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { // Update insn_idx that is shown on panic hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); @@ -836,6 +849,7 @@ impl Assembler { cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| { let disp = dst_addr - src_addr; lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap())); + Ok(()) }); } else { // Set output to the jump target's raw address @@ -850,30 +864,19 @@ impl Assembler { Insn::CPush(opnd) => { push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + push(cb, opnd0.into()); + push(cb, opnd1.into()); + }, Insn::CPop { out } => { pop(cb, out.into()); }, Insn::CPopInto(opnd) => { pop(cb, opnd.into()); }, - - // Push and pop to the C stack all caller-save registers and the - // flags - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - push(cb, X86Opnd::Reg(reg)); - } - pushfq(cb); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - popfq(cb); - for reg in regs.into_iter().rev() { - pop(cb, X86Opnd::Reg(reg)); - } + Insn::CPopPairInto(opnd0, opnd1) => { + pop(cb, opnd0.into()); + pop(cb, opnd1.into()); }, // C function call @@ -917,6 +920,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), Target::Label(label) => jmp_label(cb, label), + Target::Block(ref edge) => jmp_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -925,6 +929,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), Target::Label(label) => je_label(cb, label), + Target::Block(ref edge) => je_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -933,6 +938,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), Target::Label(label) => jne_label(cb, label), + Target::Block(ref edge) => jne_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -941,6 +947,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), Target::Label(label) => jl_label(cb, label), + Target::Block(ref edge) => jl_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -949,6 +956,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr), Target::Label(label) => jg_label(cb, label), + Target::Block(ref edge) => jg_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -957,6 +965,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr), Target::Label(label) => jge_label(cb, label), + Target::Block(ref edge) => jge_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -965,6 +974,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), Target::Label(label) => jbe_label(cb, label), + Target::Block(ref edge) => jbe_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -973,6 +983,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr), Target::Label(label) => jb_label(cb, label), + Target::Block(ref edge) => jb_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -981,6 +992,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), Target::Label(label) => jz_label(cb, label), + Target::Block(ref edge) => jz_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -989,6 +1001,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), Target::Label(label) => jnz_label(cb, label), + Target::Block(ref edge) => jnz_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -998,6 +1011,7 @@ impl Assembler { match *target { Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), Target::Label(label) => jo_label(cb, label), + Target::Block(ref edge) => jo_label(cb, self.block_label(edge.target)), Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -1062,7 +1076,7 @@ impl Assembler { } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { - if let Insn::PosMarker(callback) = self.insns.get(insn_idx).unwrap() { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { callback(pos, cb); } else { panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); @@ -1092,6 +1106,10 @@ impl Assembler { if use_scratch_regs { asm = asm.x86_scratch_split(); asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = asm.resolve_parallel_mov_pass(); + asm_dump!(asm, resolve_parallel_mov); } // Create label instances in the code block @@ -1104,7 +1122,7 @@ impl Assembler { let gc_offsets = asm.x86_emit(cb); if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { - cb.link_labels(); + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; Ok((start_ptr, gc_offsets)) } else { cb.clear_labels(); @@ -1125,7 +1143,9 @@ mod tests { fn setup_asm() -> (Assembler, CodeBlock) { rb_zjit_prepare_options(); // for get_option! on asm.compile - (Assembler::new(), CodeBlock::new_dummy()) + let mut asm = Assembler::new(); + asm.new_block_without_id(); + (asm, CodeBlock::new_dummy()) } #[test] @@ -1133,6 +1153,7 @@ mod tests { use crate::hir::SideExitReason; let mut asm = Assembler::new(); + asm.new_block_without_id(); asm.stack_base_idx = 1; let label = asm.new_label("bb0"); @@ -1667,6 +1688,119 @@ mod tests { } #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: push rdi + 0x15: push rsi + 0x16: push rdx + 0x17: push rcx + 0x18: mov eax, 0 + 0x1d: call rax + 0x1f: pop rcx + 0x20: pop rdx + 0x21: pop rsi + 0x22: pop rdi + 0x23: add rdi, rsi + 0x26: add rdx, rcx + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74801ca"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: mov r8d, 5 + 0x1a: push rdi + 0x1b: push rsi + 0x1c: push rdx + 0x1d: push rcx + 0x1e: push r8 + 0x20: push r8 + 0x22: mov eax, 0 + 0x27: call rax + 0x29: pop r8 + 0x2b: pop r8 + 0x2d: pop rcx + 0x2e: pop rdx + 0x2f: pop rsi + 0x30: pop rdi + 0x31: add rdi, rsi + 0x34: mov rdi, rdx + 0x37: add rdi, rcx + 0x3a: add rdx, r8 + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141504150b800000000ffd041584158595a5e5f4801f74889d74801cf4c01c2"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: push rdi + 0xb: push rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: pop rdi + 0xb: pop rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e"); + } + + #[test] fn test_cmov_mem() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 7afcff5863..8714518866 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -18,8 +18,8 @@ use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, NATIVE_STACK_PTR, Opnd, SP, SideExit, Target, asm_ccall, asm_comment}; -use crate::hir::{iseq_to_hir, BlockId, BranchEdge, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_STACK_PTR, Opnd, SP, SideExit, Target, asm_ccall, asm_comment}; +use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; use crate::hir::{Const, FrameState, Function, Insn, InsnId, SendFallbackReason}; use crate::hir_type::{types, Type}; use crate::options::get_option; @@ -75,12 +75,17 @@ impl JITState { } /// Find or create a label for a given BlockId - fn get_label(&mut self, asm: &mut Assembler, block_id: BlockId) -> Target { - match &self.labels[block_id.0] { + fn get_label(&mut self, asm: &mut Assembler, lir_block_id: lir::BlockId, hir_block_id: BlockId) -> Target { + // Extend labels vector if the requested index is out of bounds + if lir_block_id.0 >= self.labels.len() { + self.labels.resize(lir_block_id.0 + 1, None); + } + + match &self.labels[lir_block_id.0] { Some(label) => label.clone(), None => { - let label = asm.new_label(&format!("{block_id}")); - self.labels[block_id.0] = Some(label.clone()); + let label = asm.new_label(&format!("{hir_block_id}_{lir_block_id}")); + self.labels[lir_block_id.0] = Some(label.clone()); label } } @@ -176,6 +181,7 @@ fn register_with_perf(iseq_name: String, start_ptr: usize, code_size: usize) { pub fn gen_entry_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { // Set up registers for CFP, EC, SP, and basic block arguments let mut asm = Assembler::new(); + asm.new_block_without_id(); gen_entry_prologue(&mut asm); // Jump to the first block using a call instruction. This trampoline is used @@ -264,11 +270,28 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func let mut jit = JITState::new(iseq, version, function.num_insns(), function.num_blocks()); let mut asm = Assembler::new_with_stack_slots(num_spilled_params); - // Compile each basic block + // Mapping from HIR block IDs to LIR block IDs. + // This is is a one-to-one mapping from HIR to LIR blocks used for finding + // jump targets in LIR (LIR should always jump to the head of an HIR block) + let mut hir_to_lir: Vec<Option<lir::BlockId>> = vec![None; function.num_blocks()]; + let reverse_post_order = function.rpo(); - for &block_id in reverse_post_order.iter() { + + // Create all LIR basic blocks corresponding to HIR basic blocks + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + let lir_block_id = asm.new_block(block_id, function.is_entry_block(block_id), rpo_idx); + hir_to_lir[block_id.0] = Some(lir_block_id); + } + + // Compile each basic block + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + // Set the current block to the LIR block that corresponds to this + // HIR block. + let lir_block_id = hir_to_lir[block_id.0].unwrap(); + asm.set_current_block(lir_block_id); + // Write a label to jump to the basic block - let label = jit.get_label(&mut asm, block_id); + let label = jit.get_label(&mut asm, lir_block_id, block_id); asm.write_label(label); let block = function.block(block_id); @@ -291,15 +314,73 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func // Compile all instructions for &insn_id in block.insns() { let insn = function.find(insn_id); - if let Err(last_snapshot) = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn) { - debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); - gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); - gen_side_exit(&mut jit, &mut asm, &SideExitReason::UnhandledHIRInsn(insn_id), &function.frame_state(last_snapshot)); - // Don't bother generating code after a side-exit. We won't run it. - // TODO(max): Generate ud2 or equivalent. - break; - }; - // It's fine; we generated the instruction + match insn { + Insn::IfFalse { val, target } => { + let val_opnd = jit.get_opnd(val); + + let lir_target = hir_to_lir[target.target.0].unwrap(); + + let fall_through_target = asm.new_block(block_id, false, rpo_idx); + + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![] + }; + + gen_if_false(&mut asm, val_opnd, branch_edge, fall_through_edge); + asm.set_current_block(fall_through_target); + + let label = jit.get_label(&mut asm, fall_through_target, block_id); + asm.write_label(label); + }, + Insn::IfTrue { val, target } => { + let val_opnd = jit.get_opnd(val); + + let lir_target = hir_to_lir[target.target.0].unwrap(); + + let fall_through_target = asm.new_block(block_id, false, rpo_idx); + + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![] + }; + + gen_if_true(&mut asm, val_opnd, branch_edge, fall_through_edge); + asm.set_current_block(fall_through_target); + + let label = jit.get_label(&mut asm, fall_through_target, block_id); + asm.write_label(label); + } + Insn::Jump(target) => { + let lir_target = hir_to_lir[target.target.0].unwrap(); + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + gen_jump(&mut asm, branch_edge); + }, + _ => { + if let Err(last_snapshot) = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn) { + debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); + gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); + gen_side_exit(&mut jit, &mut asm, &SideExitReason::UnhandledHIRInsn(insn_id), &function.frame_state(last_snapshot)); + // Don't bother generating code after a side-exit. We won't run it. + // TODO(max): Generate ud2 or equivalent. + break; + }; + // It's fine; we generated the instruction + } + } } // Make sure the last patch point has enough space to insert a jump asm.pad_patch_point(); @@ -395,13 +476,10 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), Insn::Param => unreachable!("block.insns should not have Insn::Param"), Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment - Insn::Jump(branch) => no_output!(gen_jump(jit, asm, branch)), - Insn::IfTrue { val, target } => no_output!(gen_if_true(jit, asm, opnd!(val), target)), - Insn::IfFalse { val, target } => no_output!(gen_if_false(jit, asm, opnd!(val), target)), &Insn::Send { cd, blockiseq, state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), - Insn::SendWithoutBlockDirect { cme, iseq, recv, args, state, .. } => gen_send_iseq_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), None), + Insn::SendWithoutBlockDirect { cme, iseq, recv, args, kw_bits, state, .. } => gen_send_iseq_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), *kw_bits, &function.frame_state(*state), None), &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), @@ -446,6 +524,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), Insn::Test { val } => gen_test(asm, opnd!(val)), + Insn::RefineType { val, .. } => opnd!(val), Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), Insn::GuardTypeNot { val, guard_type, state } => gen_guard_type_not(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), &Insn::GuardBitEquals { val, expected, reason, state } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, &function.frame_state(state)), @@ -454,8 +533,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::GuardNotShared { recv, state } => gen_guard_not_shared(jit, asm, opnd!(recv), &function.frame_state(*state)), &Insn::GuardLess { left, right, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), &Insn::GuardGreaterEq { left, right, state } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), - &Insn::GuardSuperMethodEntry { cme, state } => no_output!(gen_guard_super_method_entry(jit, asm, cme, &function.frame_state(state))), - Insn::GetBlockHandler => gen_get_block_handler(jit, asm), + &Insn::GuardSuperMethodEntry { lep, cme, state } => no_output!(gen_guard_super_method_entry(jit, asm, opnd!(lep), cme, &function.frame_state(state))), + Insn::GetBlockHandler { lep } => gen_get_block_handler(asm, opnd!(lep)), Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), // Give up CCallWithFrame for 7+ args since asm.ccall() supports at most 6 args (recv + args). @@ -471,6 +550,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), &Insn::GetLocal { ep_offset, level, use_sp, .. } => gen_getlocal(asm, ep_offset, level, use_sp), + &Insn::IsBlockParamModified { level } => gen_is_block_param_modified(asm, level), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), @@ -498,11 +579,12 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::GuardShape { val, shape, state } => gen_guard_shape(jit, asm, opnd!(val), shape, &function.frame_state(state)), Insn::LoadPC => gen_load_pc(asm), Insn::LoadEC => gen_load_ec(), + Insn::GetLEP => gen_get_lep(jit, asm), Insn::LoadSelf => gen_load_self(), &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(asm, opnd!(recv), opnd!(val), function.type_of(val))), - &Insn::IsBlockGiven => gen_is_block_given(jit, asm), + &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), opnd!(buffer), &function.frame_state(*state)), &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), @@ -511,6 +593,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::ArrayMax { state, .. } | &Insn::Throw { state, .. } => return Err(state), + &Insn::IfFalse { .. } | Insn::IfTrue { .. } + | &Insn::Jump { .. } => unreachable!(), }; assert!(insn.has_output(), "Cannot write LIR output of HIR instruction with no output: {insn}"); @@ -608,16 +692,10 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, } /// Similar to gen_defined for DEFINED_YIELD -fn gen_is_block_given(jit: &JITState, asm: &mut Assembler) -> Opnd { - let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; - if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { - let lep = gen_get_lep(jit, asm); - let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); - asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); - asm.csel_e(Qfalse.into(), Qtrue.into()) - } else { - Qfalse.into() - } +fn gen_is_block_given(asm: &mut Assembler, lep: Opnd) -> Opnd { + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qfalse.into(), Qtrue.into()) } fn gen_unbox_fixnum(asm: &mut Assembler, val: Opnd) -> Opnd { @@ -667,6 +745,46 @@ fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: } } +/// Returns 1 (as CBool) when VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set; returns 0 otherwise. +fn gen_is_block_param_modified(asm: &mut Assembler, level: u32) -> Opnd { + let ep = gen_get_ep(asm, level); + let flags = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32))); + asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) +} + +/// Get the block parameter as a Proc, write it to the environment, +/// and mark the flag as modified. +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + // Bail out if write barrier is required. + let ep = gen_get_ep(asm, level); + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); + asm.jnz(side_exit(jit, state, SideExitReason::BlockParamWbRequired)); + + // Convert block handler to Proc. + let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let proc = asm_ccall!(asm, rb_vm_bh_to_procval, EC, block_handler); + + // Write Proc to EP and mark modified. + let ep = gen_get_ep(asm, level); + let local_ep_offset = c_int::try_from(ep_offset).unwrap_or_else(|_| { + panic!("Could not convert local_ep_offset {ep_offset} to i32") + }); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(VALUE_BITS, ep, offset), proc); + + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + let flags_val = asm.load(flags); + let modified = asm.or(flags_val, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.store(flags, modified); + + // Read the Proc from EP. + let ep = gen_get_ep(asm, level); + asm.load(Opnd::mem(VALUE_BITS, ep, offset)) +} + fn gen_guard_block_param_proxy(jit: &JITState, asm: &mut Assembler, level: u32, state: &FrameState) { // Bail out if the `&block` local variable has been modified let ep = gen_get_ep(asm, level); @@ -723,11 +841,11 @@ fn gen_guard_greater_eq(jit: &JITState, asm: &mut Assembler, left: Opnd, right: fn gen_guard_super_method_entry( jit: &JITState, asm: &mut Assembler, + lep: Opnd, cme: *const rb_callable_method_entry_t, state: &FrameState, ) { asm_comment!(asm, "guard super method entry"); - let lep = gen_get_lep(jit, asm); let ep_me_opnd = Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF); let ep_me = asm.load(ep_me_opnd); asm.cmp(ep_me, Opnd::UImm(cme as u64)); @@ -735,9 +853,8 @@ fn gen_guard_super_method_entry( } /// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). -fn gen_get_block_handler(jit: &JITState, asm: &mut Assembler) -> Opnd { +fn gen_get_block_handler(asm: &mut Assembler, lep: Opnd) -> Opnd { asm_comment!(asm, "get block handler from LEP"); - let lep = gen_get_lep(jit, asm); asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)) } @@ -1190,18 +1307,6 @@ fn gen_entry_prologue(asm: &mut Assembler) { asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); } -/// Set branch params to basic block arguments -fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { - if branch.args.is_empty() { - return; - } - - asm_comment!(asm, "set branch params: {}", branch.args.len()); - asm.parallel_mov(branch.args.iter().enumerate().map(|(idx, &arg)| - (param_opnd(idx), jit.get_opnd(arg)) - ).collect()); -} - /// Compile a constant fn gen_const_value(val: VALUE) -> lir::Opnd { // Just propagate the constant value and generate nothing @@ -1228,7 +1333,7 @@ fn gen_const_uint32(val: u32) -> lir::Opnd { /// Compile a basic block argument fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { // Allocate a register or a stack slot - match param_opnd(idx) { + match Assembler::param_opnd(idx) { // If it's a register, insert LiveReg instruction to reserve the register // in the register pool for register allocation. param @ Opnd::Reg(_) => asm.live_reg_opnd(param), @@ -1237,45 +1342,25 @@ fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { } /// Compile a jump to a basic block -fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { - // Set basic block arguments - gen_branch_params(jit, asm, branch); - +fn gen_jump(asm: &mut Assembler, branch: lir::BranchEdge) { // Jump to the basic block - let target = jit.get_label(asm, branch.target); - asm.jmp(target); + asm.jmp(Target::Block(branch)); } /// Compile a conditional branch to a basic block -fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { +fn gen_if_true(asm: &mut Assembler, val: lir::Opnd, branch: lir::BranchEdge, fall_through: lir::BranchEdge) { // If val is zero, move on to the next instruction. - let if_false = asm.new_label("if_false"); asm.test(val, val); - asm.jz(if_false.clone()); - - // If val is not zero, set basic block arguments and jump to the branch target. - // TODO: Consider generating the loads out-of-line - let if_true = jit.get_label(asm, branch.target); - gen_branch_params(jit, asm, branch); - asm.jmp(if_true); - - asm.write_label(if_false); + asm.jz(Target::Block(fall_through)); + asm.jmp(Target::Block(branch)); } /// Compile a conditional branch to a basic block -fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { +fn gen_if_false(asm: &mut Assembler, val: lir::Opnd, branch: lir::BranchEdge, fall_through: lir::BranchEdge) { // If val is not zero, move on to the next instruction. - let if_true = asm.new_label("if_true"); asm.test(val, val); - asm.jnz(if_true.clone()); - - // If val is zero, set basic block arguments and jump to the branch target. - // TODO: Consider generating the loads out-of-line - let if_false = jit.get_label(asm, branch.target); - gen_branch_params(jit, asm, branch); - asm.jmp(if_false); - - asm.write_label(if_true); + asm.jnz(Target::Block(fall_through)); + asm.jmp(Target::Block(branch)); } /// Compile a dynamic dispatch with block @@ -1358,6 +1443,7 @@ fn gen_send_iseq_direct( iseq: IseqPtr, recv: Opnd, args: Vec<Opnd>, + kw_bits: u32, state: &FrameState, block_handler: Option<Opnd>, ) -> lir::Opnd { @@ -1404,12 +1490,13 @@ fn gen_send_iseq_direct( // Write "keyword_bits" to the callee's frame if the callee accepts keywords. // This is a synthetic local/parameter that the callee reads via checkkeyword to determine // which optional keyword arguments need their defaults evaluated. + // We write this to the local table slot at bits_start so that: + // 1. The interpreter can read it via checkkeyword if we side-exit + // 2. The JIT entry can read it via GetLocal if unsafe { rb_get_iseq_flags_has_kw(iseq) } { let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; let bits_start = unsafe { (*keyword).bits_start } as usize; - // Currently we only support required keywords, so all bits are 0 (all keywords specified). - // TODO: When supporting optional keywords, calculate actual unspecified_bits here. - let unspecified_bits = VALUE::fixnum_from_usize(0); + let unspecified_bits = VALUE::fixnum_from_usize(kw_bits as usize); let bits_offset = (state.stack().len() - args.len() + bits_start) * SIZEOF_VALUE; asm_comment!(asm, "write keyword bits to callee frame"); asm.store(Opnd::mem(64, SP, bits_offset as i32), unspecified_bits.into()); @@ -1435,10 +1522,11 @@ fn gen_send_iseq_direct( let lead_num = params.lead_num as u32; let opt_num = params.opt_num as u32; let keyword = params.keyword; - let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } } as u32; - let req_num = lead_num + kw_req_num; - assert!(args.len() as u32 <= req_num + opt_num); - let num_optionals_passed = args.len() as u32 - req_num; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } } as u32; + assert!(args.len() as u32 <= lead_num + opt_num + kw_total_num); + // For computing optional positional entry point, only count positional args + let positional_argc = args.len() as u32 - kw_total_num; + let num_optionals_passed = positional_argc.saturating_sub(lead_num); num_optionals_passed } else { 0 @@ -1743,7 +1831,46 @@ fn gen_dup_array_include( } fn gen_is_a(asm: &mut Assembler, obj: Opnd, class: Opnd) -> lir::Opnd { - asm_ccall!(asm, rb_obj_is_kind_of, obj, class) + let builtin_type = match class { + Opnd::Value(value) if value == unsafe { rb_cString } => Some(RUBY_T_STRING), + Opnd::Value(value) if value == unsafe { rb_cArray } => Some(RUBY_T_ARRAY), + Opnd::Value(value) if value == unsafe { rb_cHash } => Some(RUBY_T_HASH), + _ => None + }; + + if let Some(builtin_type) = builtin_type { + asm_comment!(asm, "IsA by matching builtin type"); + let ret_label = asm.new_label("is_a_ret"); + let false_label = asm.new_label("is_a_false"); + + let val = match obj { + Opnd::Reg(_) | Opnd::VReg { .. } => obj, + _ => asm.load(obj), + }; + + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(ret_label.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(false_label.clone()); + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let obj_builtin_type = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(obj_builtin_type, Opnd::UImm(builtin_type as u64)); + asm.jmp(ret_label.clone()); + + // If we get here then the value was false, unset the Z flag + // so that csel_e will select false instead of true + asm.write_label(false_label); + asm.test(Opnd::UImm(1), Opnd::UImm(1)); + + asm.write_label(ret_label); + asm.csel_e(Qtrue.into(), Qfalse.into()) + } else { + asm_ccall!(asm, rb_obj_is_kind_of, obj, class) + } } /// Compile a new hash instruction @@ -2149,6 +2276,9 @@ fn gen_guard_type_not(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, g /// Compile an identity check with a side exit fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + if matches!(reason, SideExitReason::GuardShape(_) ) { + gen_incr_counter(asm, Counter::guard_shape_count); + } let expected_opnd: Opnd = match expected { crate::hir::Const::Value(v) => { Opnd::Value(v) } crate::hir::Const::CInt64(v) => { v.into() } @@ -2369,19 +2499,6 @@ fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &Fra asm.jbe(side_exit(jit, state, StackOverflow)); } -/// Return an operand we use for the basic block argument at a given index -fn param_opnd(idx: usize) -> Opnd { - // To simplify the implementation, allocate a fixed register or a stack slot for each basic block argument for now. - // Note that this is implemented here as opposed to automatically inside LIR machineries. - // TODO: Allow allocating arbitrary registers for basic block arguments - if idx < ALLOC_REGS.len() { - Opnd::Reg(ALLOC_REGS[idx]) - } else { - // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register. - // To avoid clobbering it, we need to start from the next slot, hence `+ 1` for the index. - Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) - } -} /// Inverse of ep_offset_to_local_idx(). See ep_offset_to_local_idx() for details. pub fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { @@ -2576,6 +2693,7 @@ fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result /// Compile a stub for an ISEQ called by SendWithoutBlockDirect fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodePtr, CompileError> { let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq.get(), 0)); // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. @@ -2593,14 +2711,24 @@ fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodeP /// See [gen_function_stub] for how it's used. pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id(); asm_comment!(asm, "function_stub_hit trampoline"); // Maintain alignment for x86_64, and set up a frame for arm64 properly asm.frame_setup(&[]); asm_comment!(asm, "preserve argument registers"); - for ® in ALLOC_REGS.iter() { - asm.cpush(Opnd::Reg(reg)); + + for pair in ALLOC_REGS.chunks(2) { + match *pair { + [reg0, reg1] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + } + [reg] => { + asm.cpush(Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } } if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 @@ -2614,8 +2742,17 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, C if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { asm.cpop_into(Opnd::Reg(ALLOC_REGS[0])); } - for ® in ALLOC_REGS.iter().rev() { - asm.cpop_into(Opnd::Reg(reg)); + + for pair in ALLOC_REGS.chunks(2).rev() { + match *pair { + [reg] => { + asm.cpop_into(Opnd::Reg(reg)); + } + [reg0, reg1] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + } + _ => unreachable!("chunks(2)") + } } // Discard the current frame since the JIT function will set it up again @@ -2633,6 +2770,7 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, C /// Generate a trampoline that is used when a function exits without restoring PC and the stack pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { let mut asm = Assembler::new(); + asm.new_block_without_id(); asm_comment!(asm, "side-exit trampoline"); asm.frame_teardown(&[]); // matching the setup in gen_entry_point() @@ -2647,6 +2785,7 @@ pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> /// Generate a trampoline that increments exit_compilation_failure and jumps to exit_trampoline. pub fn gen_exit_trampoline_with_counter(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { let mut asm = Assembler::new(); + asm.new_block_without_id(); asm_comment!(asm, "function stub exit trampoline"); gen_incr_counter(&mut asm, exit_compile_error); @@ -2763,6 +2902,7 @@ fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: /// Generate a JIT entry that just increments exit_compilation_failure and exits fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> { let mut asm = Assembler::new(); + asm.new_block_without_id(); gen_incr_counter(&mut asm, exit_compile_error); gen_incr_counter(&mut asm, exit_counter_for_compile_error(compile_error)); asm.cret(Qundef.into()); @@ -2855,6 +2995,7 @@ impl IseqCall { fn regenerate(&self, cb: &mut CodeBlock, callback: impl Fn(&mut Assembler)) { cb.with_write_ptr(self.start_addr.get().unwrap(), |cb| { let mut asm = Assembler::new(); + asm.new_block_without_id(); callback(&mut asm); asm.compile(cb).unwrap(); assert_eq!(self.end_addr.get().unwrap(), cb.get_write_ptr()); diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 357c8b0c12..8121b0065f 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -318,8 +318,14 @@ fn inline_kernel_itself(_fun: &mut hir::Function, _block: hir::BlockId, recv: hi fn inline_kernel_block_given_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { let &[] = args else { return None; }; - // TODO(max): In local iseq types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. - Some(fun.push_insn(block, hir::Insn::IsBlockGiven)) + + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(fun.iseq()) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + let lep = fun.push_insn(block, hir::Insn::GetLEP); + Some(fun.push_insn(block, hir::Insn::IsBlockGiven { lep })) + } else { + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) + } } fn inline_array_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 6c2bd09ad3..2aa74dce8b 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -506,6 +506,7 @@ pub enum SideExitReason { Interrupt, BlockParamProxyModified, BlockParamProxyNotIseqOrIfunc, + BlockParamWbRequired, StackOverflow, FixnumModByZero, FixnumDivByZero, @@ -625,9 +626,9 @@ pub enum SendFallbackReason { SendWithoutBlockBopRedefined, SendWithoutBlockOperandsNotFixnum, SendWithoutBlockDirectKeywordMismatch, - SendWithoutBlockDirectOptionalKeywords, SendWithoutBlockDirectKeywordCountMismatch, SendWithoutBlockDirectMissingKeyword, + SendWithoutBlockDirectTooManyKeywords, SendPolymorphic, SendMegamorphic, SendNoProfiles, @@ -686,9 +687,9 @@ impl Display for SendFallbackReason { SendWithoutBlockBopRedefined => write!(f, "SendWithoutBlock: basic operation was redefined"), SendWithoutBlockOperandsNotFixnum => write!(f, "SendWithoutBlock: operands are not fixnums"), SendWithoutBlockDirectKeywordMismatch => write!(f, "SendWithoutBlockDirect: keyword mismatch"), - SendWithoutBlockDirectOptionalKeywords => write!(f, "SendWithoutBlockDirect: optional keywords"), SendWithoutBlockDirectKeywordCountMismatch => write!(f, "SendWithoutBlockDirect: keyword count mismatch"), SendWithoutBlockDirectMissingKeyword => write!(f, "SendWithoutBlockDirect: missing keyword"), + SendWithoutBlockDirectTooManyKeywords => write!(f, "SendWithoutBlockDirect: too many keywords for fixnum bitmask"), SendPolymorphic => write!(f, "Send: polymorphic call site"), SendMegamorphic => write!(f, "Send: megamorphic call site"), SendNoProfiles => write!(f, "Send: no profile data available"), @@ -802,7 +803,7 @@ pub enum Insn { GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, /// Kernel#block_given? but without pushing a frame. Similar to [`Insn::Defined`] with /// `DEFINED_YIELD` - IsBlockGiven, + IsBlockGiven { lep: InsnId }, /// Test the bit at index of val, a Fixnum. /// Return Qtrue if the bit is set, else Qfalse. FixnumBitCheck { val: InsnId, index: u8 }, @@ -839,6 +840,11 @@ pub enum Insn { /// If `use_sp` is true, it uses the SP register to optimize the read. /// `rest_param` is used by infer_types to infer the ArrayExact type. GetLocal { level: u32, ep_offset: u32, use_sp: bool, rest_param: bool }, + /// Check whether VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set in the environment flags. + /// Returns CBool (0/1). + IsBlockParamModified { level: u32 }, + /// Get the block parameter as a Proc. + GetBlockParam { level: u32, ep_offset: u32, state: InsnId }, /// Set a local variable in a higher scope or the heap SetLocal { level: u32, ep_offset: u32, val: InsnId }, GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, @@ -849,6 +855,10 @@ pub enum Insn { /// Set a class variable `id` to `val` SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + /// Equivalent of GET_LEP() macro. + GetLEP, + /// Own a FrameState so that instructions can look up their dominating FrameState when /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate /// any code. @@ -947,6 +957,7 @@ pub enum Insn { cme: *const rb_callable_method_entry_t, iseq: IseqPtr, args: Vec<InsnId>, + kw_bits: u32, state: InsnId, }, @@ -989,6 +1000,10 @@ pub enum Insn { ObjToString { val: InsnId, cd: *const rb_call_data, state: InsnId }, AnyToString { val: InsnId, str: InsnId, state: InsnId }, + /// Refine the known type information of with additional type information. + /// Computes the intersection of the existing type and the new type. + RefineType { val: InsnId, new_type: Type }, + /// Side-exit if val doesn't have the expected type. GuardType { val: InsnId, guard_type: Type, state: InsnId }, GuardTypeNot { val: InsnId, guard_type: Type, state: InsnId }, @@ -1011,9 +1026,9 @@ pub enum Insn { GuardLess { left: InsnId, right: InsnId, state: InsnId }, /// Side-exit if the method entry at ep[VM_ENV_DATA_INDEX_ME_CREF] doesn't match the expected CME. /// Used to ensure super calls are made from the expected method context. - GuardSuperMethodEntry { cme: *const rb_callable_method_entry_t, state: InsnId }, + GuardSuperMethodEntry { lep: InsnId, cme: *const rb_callable_method_entry_t, state: InsnId }, /// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). - GetBlockHandler, + GetBlockHandler { lep: InsnId }, /// Generate no code (or padding if necessary) and insert a patch point /// that can be rewritten to a side exit when the Invariant is broken. @@ -1130,6 +1145,7 @@ impl Insn { Insn::DefinedIvar { .. } => effects::Any, Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), Insn::LoadEC { .. } => effects::Empty, + Insn::GetLEP { .. } => effects::Empty, Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), Insn::StoreField { .. } => effects::Any, @@ -1140,6 +1156,8 @@ impl Insn { Insn::GetSpecialNumber { .. } => effects::Any, Insn::GetClassVar { .. } => effects::Any, Insn::SetClassVar { .. } => effects::Any, + Insn::IsBlockParamModified { .. } => effects::Any, + Insn::GetBlockParam { .. } => effects::Any, Insn::Snapshot { .. } => effects::Empty, Insn::Jump(_) => effects::Any, Insn::IfTrue { .. } => effects::Any, @@ -1206,6 +1224,7 @@ impl Insn { Insn::IncrCounterPtr { .. } => effects::Any, Insn::CheckInterrupts { .. } => effects::Any, Insn::InvokeProc { .. } => effects::Any, + Insn::RefineType { .. } => effects::Empty, } } @@ -1501,6 +1520,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, Insn::GuardType { val, guard_type, .. } => { write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map)) }, + Insn::RefineType { val, new_type, .. } => { write!(f, "RefineType {val}, {}", new_type.print(self.ptr_map)) }, Insn::GuardTypeNot { val, guard_type, .. } => { write!(f, "GuardTypeNot {val}, {}", guard_type.print(self.ptr_map)) }, Insn::GuardBitEquals { val, expected, .. } => { write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map)) }, &Insn::GuardShape { val, shape, .. } => { write!(f, "GuardShape {val}, {:p}", self.ptr_map.map_shape(shape)) }, @@ -1509,11 +1529,16 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::GuardNotShared { recv, .. } => write!(f, "GuardNotShared {recv}"), Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), Insn::GuardGreaterEq { left, right, .. } => write!(f, "GuardGreaterEq {left}, {right}"), - Insn::GuardSuperMethodEntry { cme, .. } => write!(f, "GuardSuperMethodEntry {:p}", self.ptr_map.map_ptr(cme)), - Insn::GetBlockHandler => write!(f, "GetBlockHandler"), + Insn::GuardSuperMethodEntry { lep, cme, .. } => write!(f, "GuardSuperMethodEntry {lep}, {:p}", self.ptr_map.map_ptr(cme)), + Insn::GetBlockHandler { lep } => write!(f, "GetBlockHandler {lep}"), + &Insn::GetBlockParam { level, ep_offset, .. } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset) + .map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetBlockParam {name}l{level}, EP@{ep_offset}") + }, Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, Insn::GetConstantPath { ic, .. } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, - Insn::IsBlockGiven => { write!(f, "IsBlockGiven") }, + Insn::IsBlockGiven { lep } => { write!(f, "IsBlockGiven {lep}") }, Insn::FixnumBitCheck {val, index} => { write!(f, "FixnumBitCheck {val}, {index}") }, Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => { write!(f, "CCall {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; @@ -1561,6 +1586,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), Insn::LoadPC => write!(f, "LoadPC"), Insn::LoadEC => write!(f, "LoadEC"), + Insn::GetLEP => write!(f, "GetLEP"), Insn::LoadSelf => write!(f, "LoadSelf"), &Insn::LoadField { recv, id, offset, return_type: _ } => write!(f, "LoadField {recv}, :{}@{:p}", id.contents_lossy(), self.ptr_map.map_offset(offset)), &Insn::StoreField { recv, id, offset, val } => write!(f, "StoreField {recv}, :{}@{:p}, {val}", id.contents_lossy(), self.ptr_map.map_offset(offset)), @@ -1576,6 +1602,9 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); write!(f, "GetLocal {name}l{level}, EP@{ep_offset}{}", if rest_param { ", *" } else { "" }) }, + &Insn::IsBlockParamModified { level } => { + write!(f, "IsBlockParamModified l{level}") + }, &Insn::SetLocal { val, level, ep_offset } => { let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); write!(f, "SetLocal {name}l{level}, EP@{ep_offset}, {val}") @@ -1792,7 +1821,7 @@ pub enum ValidationError { MiscValidationError(InsnId, String), } -fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, send_insn: InsnId, args: &[InsnId]) -> bool { +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, ci: *const rb_callinfo, send_insn: InsnId, args: &[InsnId]) -> bool { let mut can_send = true; let mut count_failure = |counter| { can_send = false; @@ -1807,44 +1836,43 @@ fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } if 0 != params.flags.has_kwrest() { count_failure(complex_arg_pass_param_kwrest) } - if 0 != params.flags.has_kw() { - let keyword = params.keyword; - if !keyword.is_null() { - let num = unsafe { (*keyword).num }; - let required_num = unsafe { (*keyword).required_num }; - // Only support required keywords for now (no optional keywords) - if num != required_num { - count_failure(complex_arg_pass_param_kw_opt) - } - } - } if !can_send { function.set_dynamic_send_reason(send_insn, ComplexArgPass); return false; } - // asm.ccall() doesn't support 6+ args - if args.len() + 1 > C_ARG_OPNDS.len() { // +1 for self - function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); - return false; - } - // Because we exclude e.g. post parameters above, they are also excluded from the sum below. let lead_num = params.lead_num; let opt_num = params.opt_num; let keyword = params.keyword; let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; - let req_num = lead_num + kw_req_num; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; + // Minimum args: all required positional + all required keywords + let min_argc = lead_num + kw_req_num; + // Maximum args: all positional (required + optional) + all keywords (required + optional) + let max_argc = lead_num + opt_num + kw_total_num; + can_send = c_int::try_from(args.len()) .as_ref() - .map(|argc| (req_num..=req_num + opt_num).contains(argc)) + .map(|argc| (min_argc..=max_argc).contains(argc)) .unwrap_or(false); if !can_send { function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); return false } + // asm.ccall() doesn't support 6+ args. Compute the final argc after keyword setup: + // final_argc = caller's positional args + callee's total keywords (all kw slots are filled). + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_positional = args.len() - caller_kw_count; + let final_argc = caller_positional + kw_total_num as usize; + if final_argc + 1 > C_ARG_OPNDS.len() { // +1 for self + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + can_send } @@ -1969,6 +1997,10 @@ impl Function { } } + pub fn iseq(&self) -> *const rb_iseq_t { + self.iseq + } + // Add an instruction to the function without adding it to any block fn new_insn(&mut self, insn: Insn) -> InsnId { let id = InsnId(self.insns.len()); @@ -2119,15 +2151,16 @@ impl Function { result@(Const {..} | Param | GetConstantPath {..} - | IsBlockGiven | PatchPoint {..} | PutSpecialObject {..} | GetGlobal {..} | GetLocal {..} + | IsBlockParamModified {..} | SideExit {..} | EntryPoint {..} | LoadPC | LoadEC + | GetLEP | LoadSelf | IncrCounterPtr {..} | IncrCounter(_)) => result.clone(), @@ -2164,6 +2197,7 @@ impl Function { Jump(target) => Jump(find_branch_edge!(target)), &IfTrue { val, ref target } => IfTrue { val: find!(val), target: find_branch_edge!(target) }, &IfFalse { val, ref target } => IfFalse { val: find!(val), target: find_branch_edge!(target) }, + &RefineType { val, new_type } => RefineType { val: find!(val), new_type }, &GuardType { val, guard_type, state } => GuardType { val: find!(val), guard_type, state }, &GuardTypeNot { val, guard_type, state } => GuardTypeNot { val: find!(val), guard_type, state }, &GuardBitEquals { val, expected, reason, state } => GuardBitEquals { val: find!(val), expected, reason, state }, @@ -2173,8 +2207,10 @@ impl Function { &GuardNotShared { recv, state } => GuardNotShared { recv: find!(recv), state }, &GuardGreaterEq { left, right, state } => GuardGreaterEq { left: find!(left), right: find!(right), state }, &GuardLess { left, right, state } => GuardLess { left: find!(left), right: find!(right), state }, - &GuardSuperMethodEntry { cme, state } => GuardSuperMethodEntry { cme, state }, - &GetBlockHandler => GetBlockHandler, + &GuardSuperMethodEntry { lep, cme, state } => GuardSuperMethodEntry { lep: find!(lep), cme, state }, + &GetBlockHandler { lep } => GetBlockHandler { lep: find!(lep) }, + &IsBlockGiven { lep } => IsBlockGiven { lep: find!(lep) }, + &GetBlockParam { level, ep_offset, state } => GetBlockParam { level, ep_offset, state: find!(state) }, &FixnumAdd { left, right, state } => FixnumAdd { left: find!(left), right: find!(right), state }, &FixnumSub { left, right, state } => FixnumSub { left: find!(left), right: find!(right), state }, &FixnumMult { left, right, state } => FixnumMult { left: find!(left), right: find!(right), state }, @@ -2208,12 +2244,13 @@ impl Function { state, reason, }, - &SendWithoutBlockDirect { recv, cd, cme, iseq, ref args, state } => SendWithoutBlockDirect { + &SendWithoutBlockDirect { recv, cd, cme, iseq, ref args, kw_bits, state } => SendWithoutBlockDirect { recv: find!(recv), cd, cme, iseq, args: find_vec!(args), + kw_bits, state, }, &Send { recv, cd, blockiseq, ref args, state, reason } => Send { @@ -2411,6 +2448,7 @@ impl Function { Insn::CCall { return_type, .. } => *return_type, &Insn::CCallVariadic { return_type, .. } => return_type, Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), + Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), Insn::GuardTypeNot { .. } => types::BasicObject, Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), Insn::GuardShape { val, .. } => self.type_of(*val), @@ -2445,7 +2483,7 @@ impl Function { Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), Insn::DefinedIvar { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), Insn::GetConstantPath { .. } => types::BasicObject, - Insn::IsBlockGiven => types::BoolExact, + Insn::IsBlockGiven { .. } => types::BoolExact, Insn::FixnumBitCheck { .. } => types::BoolExact, Insn::ArrayMax { .. } => types::BasicObject, Insn::ArrayInclude { .. } => types::BoolExact, @@ -2456,6 +2494,7 @@ impl Function { Insn::GetIvar { .. } => types::BasicObject, Insn::LoadPC => types::CPtr, Insn::LoadEC => types::CPtr, + Insn::GetLEP => types::CPtr, Insn::LoadSelf => types::BasicObject, &Insn::LoadField { return_type, .. } => return_type, Insn::GetSpecialSymbol { .. } => types::BasicObject, @@ -2467,7 +2506,9 @@ impl Function { Insn::AnyToString { .. } => types::String, Insn::GetLocal { rest_param: true, .. } => types::ArrayExact, Insn::GetLocal { .. } => types::BasicObject, - Insn::GetBlockHandler => types::RubyValue, + Insn::IsBlockParamModified { .. } => types::CBool, + Insn::GetBlockParam { .. } => types::BasicObject, + Insn::GetBlockHandler { .. } => types::RubyValue, // The type of Snapshot doesn't really matter; it's never materialized. It's used only // as a reference for FrameState, which we use to generate side-exit code. Insn::Snapshot { .. } => types::Any, @@ -2581,6 +2622,7 @@ impl Function { | Insn::GuardTypeNot { val, .. } | Insn::GuardShape { val, .. } | Insn::GuardBitEquals { val, .. } => self.chase_insn(val), + | Insn::RefineType { val, .. } => self.chase_insn(val), _ => id, } } @@ -2596,31 +2638,74 @@ impl Function { } } - /// Reorder keyword arguments to match the callee's expectation. + /// Prepare arguments for a direct send, handling keyword argument reordering and default synthesis. + /// Returns the (state, processed_args, kw_bits) to use for the SendWithoutBlockDirect instruction, + /// or Err with the fallback reason if direct send isn't possible. + fn prepare_direct_send_args( + &mut self, + block: BlockId, + args: &[InsnId], + ci: *const rb_callinfo, + iseq: IseqPtr, + state: InsnId, + ) -> Result<(InsnId, Vec<InsnId>, u32), SendFallbackReason> { + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (processed_args, caller_argc, kw_bits) = self.setup_keyword_arguments(block, args, kwarg, iseq)?; + + // If args were reordered or synthesized, create a new snapshot with the updated stack + let send_state = if processed_args != args { + let new_state = self.frame_state(state).with_replaced_args(&processed_args, caller_argc); + self.push_insn(block, Insn::Snapshot { state: new_state }) + } else { + state + }; + + Ok((send_state, processed_args, kw_bits)) + } + + /// Reorder keyword arguments to match the callee's expected order, and synthesize + /// default values for any optional keywords not provided by the caller. /// - /// Returns Ok with reordered arguments if successful, or Err with the fallback reason if not. - fn reorder_keyword_arguments( - &self, + /// The output always contains all of the callee's keyword arguments (required + optional), + /// so the returned vec may be larger than the input args. + /// + /// Returns Ok with (processed_args, caller_argc, kw_bits) if successful, or Err with the fallback reason if not. + /// - caller_argc: number of arguments the caller actually pushed (for stack calculations) + /// - kw_bits: bitmask indicating which optional keywords were NOT provided by the caller + /// (used by checkkeyword to determine if non-constant defaults need evaluation) + fn setup_keyword_arguments( + &mut self, + block: BlockId, args: &[InsnId], kwarg: *const rb_callinfo_kwarg, iseq: IseqPtr, - ) -> Result<Vec<InsnId>, SendFallbackReason> { + ) -> Result<(Vec<InsnId>, usize, u32), SendFallbackReason> { let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; if callee_keyword.is_null() { - // Caller is passing kwargs but callee doesn't expect them. - return Err(SendWithoutBlockDirectKeywordMismatch); + if !kwarg.is_null() { + // Caller is passing kwargs but callee doesn't expect them. + return Err(SendWithoutBlockDirectKeywordMismatch); + } + // Neither caller nor callee have keywords - nothing to do + return Ok((args.to_vec(), args.len(), 0)); } - let caller_kw_count = unsafe { get_cikw_keyword_len(kwarg) } as usize; + // kwarg may be null if caller passes no keywords but callee has optional keywords + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; let callee_kw_count = unsafe { (*callee_keyword).num } as usize; + + // When there are 31+ keywords, CRuby uses a hash instead of a fixnum bitmask + // for kw_bits. Fall back to VM dispatch for this rare case. + if callee_kw_count >= VM_KW_SPECIFIED_BITS_MAX as usize { + return Err(SendWithoutBlockDirectTooManyKeywords); + } + let callee_kw_required = unsafe { (*callee_keyword).required_num } as usize; let callee_kw_table = unsafe { (*callee_keyword).table }; + let default_values = unsafe { (*callee_keyword).default_values }; - // For now, only handle the case where all keywords are required. - if callee_kw_count != callee_kw_required { - return Err(SendWithoutBlockDirectOptionalKeywords); - } - if caller_kw_count != callee_kw_count { + // Caller can't provide more keywords than callee expects (no **kwrest support yet). + if caller_kw_count > callee_kw_count { return Err(SendWithoutBlockDirectKeywordCountMismatch); } @@ -2629,13 +2714,35 @@ impl Function { // Build a mapping from caller keywords to their positions. let mut caller_kw_order: Vec<ID> = Vec::with_capacity(caller_kw_count); - for i in 0..caller_kw_count { - let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; - let id = unsafe { rb_sym2id(sym) }; - caller_kw_order.push(id); + if !kwarg.is_null() { + for i in 0..caller_kw_count { + let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; + let id = unsafe { rb_sym2id(sym) }; + caller_kw_order.push(id); + } + } + + // Verify all caller keywords are expected by callee (no unknown keywords). + // Without **kwrest, unexpected keywords should raise ArgumentError at runtime. + for &caller_id in &caller_kw_order { + let mut found = false; + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + if caller_id == expected_id { + found = true; + break; + } + } + if !found { + // Caller is passing an unknown keyword - this will raise ArgumentError. + // Fall back to VM dispatch to handle the error. + return Err(SendWithoutBlockDirectKeywordMismatch); + } } // Reorder keyword arguments to match callee expectation. + // Track which optional keywords were not provided via kw_bits. + let mut kw_bits: u32 = 0; let mut reordered_kw_args: Vec<InsnId> = Vec::with_capacity(callee_kw_count); for i in 0..callee_kw_count { let expected_id = unsafe { *callee_kw_table.add(i) }; @@ -2652,14 +2759,36 @@ impl Function { if !found { // Required keyword not provided by caller which will raise an ArgumentError. - return Err(SendWithoutBlockDirectMissingKeyword); + if i < callee_kw_required { + return Err(SendWithoutBlockDirectMissingKeyword); + } + + // Optional keyword not provided - use default value + let default_idx = i - callee_kw_required; + let default_value = unsafe { *default_values.add(default_idx) }; + + if default_value == Qundef { + // Non-constant default (e.g., `def foo(a: compute())`). + // Set the bit so checkkeyword knows to evaluate the default at runtime. + // Push Qnil as a placeholder; the callee's checkkeyword will detect this + // and branch to evaluate the default expression. + kw_bits |= 1 << default_idx; + let nil_insn = self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + reordered_kw_args.push(nil_insn); + } else { + // Constant default value - use it directly + let const_insn = self.push_insn(block, Insn::Const { val: Const::Value(default_value) }); + reordered_kw_args.push(const_insn); + } } } // Replace the keyword arguments with the reordered ones. + // Keep track of the original caller argc for stack calculations. + let caller_argc = args.len(); let mut processed_args = args[..kw_args_start].to_vec(); processed_args.extend(reordered_kw_args); - Ok(processed_args) + Ok((processed_args, caller_argc, kw_bits)) } /// Resolve the receiver type for method dispatch optimization. @@ -2894,7 +3023,7 @@ impl Function { // Only specialize positional-positional calls // TODO(max): Handle other kinds of parameter passing let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice()) { self.push_insn_id(block, insn_id); continue; } // Check singleton class assumption first, before emitting other patchpoints @@ -2907,24 +3036,12 @@ impl Function { recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); } - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; - let (send_state, processed_args) = if !kwarg.is_null() { - match self.reorder_keyword_arguments(&args, kwarg, iseq) { - Ok(reordered) => { - let new_state = self.frame_state(state).with_reordered_args(&reordered); - let snapshot = self.push_insn(block, Insn::Snapshot { state: new_state }); - (snapshot, reordered) - } - Err(reason) => { - self.set_dynamic_send_reason(insn_id, reason); - self.push_insn_id(block, insn_id); continue; - } - } - } else { - (state, args.clone()) + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; }; - let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, state: send_state }); + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state }); self.make_equal_to(insn_id, send_direct); } else if def_type == VM_METHOD_TYPE_BMETHOD { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; @@ -2939,7 +3056,7 @@ impl Function { let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice()) { self.push_insn_id(block, insn_id); continue; } // Can't pass a block to a block for now @@ -2962,24 +3079,12 @@ impl Function { recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); } - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; - let (send_state, processed_args) = if !kwarg.is_null() { - match self.reorder_keyword_arguments(&args, kwarg, iseq) { - Ok(reordered) => { - let new_state = self.frame_state(state).with_reordered_args(&reordered); - let snapshot = self.push_insn(block, Insn::Snapshot { state: new_state }); - (snapshot, reordered) - } - Err(reason) => { - self.set_dynamic_send_reason(insn_id, reason); - self.push_insn_id(block, insn_id); continue; - } - } - } else { - (state, args.clone()) + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; }; - let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, state: send_state }); + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state }); self.make_equal_to(insn_id, send_direct); } else if def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. @@ -3332,7 +3437,7 @@ impl Function { // Check if the super method's parameters support direct send. // If not, we can't do direct dispatch. let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; - if !can_direct_send(self, block, super_iseq, insn_id, args.as_slice()) { + if !can_direct_send(self, block, super_iseq, ci, insn_id, args.as_slice()) { self.push_insn_id(block, insn_id); self.set_dynamic_send_reason(insn_id, SuperTargetComplexArgsPass); continue; @@ -3349,10 +3454,15 @@ impl Function { }); // Guard that we're calling `super` from the expected method context. - self.push_insn(block, Insn::GuardSuperMethodEntry { cme: current_cme, state }); + let lep = self.push_insn(block, Insn::GetLEP); + self.push_insn(block, Insn::GuardSuperMethodEntry { + lep, + cme: current_cme, + state + }); // Guard that no block is being passed (implicit or explicit). - let block_handler = self.push_insn(block, Insn::GetBlockHandler); + let block_handler = self.push_insn(block, Insn::GetBlockHandler { lep }); self.push_insn(block, Insn::GuardBitEquals { val: block_handler, expected: Const::Value(VALUE(VM_BLOCK_HANDLER_NONE as usize)), @@ -3360,14 +3470,20 @@ impl Function { state }); + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, super_iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + // Use SendWithoutBlockDirect with the super method's CME and ISEQ. let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme: super_cme, iseq: super_iseq, - args, - state + args: processed_args, + kw_bits, + state: send_state }); self.make_equal_to(insn_id, send_direct); } @@ -3666,15 +3782,18 @@ impl Function { }; // Do method lookup - let cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; if cme.is_null() { fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); return Err(()); } // Filter for C methods - // TODO(max): Handle VM_METHOD_TYPE_ALIAS - let def_type = unsafe { get_cme_def_type(cme) }; + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } if def_type != VM_METHOD_TYPE_CFUNC { return Err(()); } @@ -4287,16 +4406,21 @@ impl Function { | &Insn::EntryPoint { .. } | &Insn::LoadPC | &Insn::LoadEC + | &Insn::GetLEP | &Insn::LoadSelf | &Insn::GetLocal { .. } - | &Insn::GetBlockHandler + | &Insn::IsBlockParamModified { .. } | &Insn::PutSpecialObject { .. } - | &Insn::IsBlockGiven | &Insn::IncrCounter(_) | &Insn::IncrCounterPtr { .. } => {} + &Insn::GetBlockHandler { lep } + | &Insn::IsBlockGiven { lep } => { + worklist.push_back(lep); + } &Insn::PatchPoint { state, .. } | &Insn::CheckInterrupts { state } + | &Insn::GetBlockParam { state, .. } | &Insn::GetConstantPath { ic: _, state } => { worklist.push_back(state); } @@ -4355,6 +4479,7 @@ impl Function { worklist.extend(values); worklist.push_back(state); } + | &Insn::RefineType { val, .. } | &Insn::Return { val } | &Insn::Test { val } | &Insn::SetLocal { val, .. } @@ -4519,12 +4644,15 @@ impl Function { worklist.push_back(val); } &Insn::GuardBlockParamProxy { state, .. } | - &Insn::GuardSuperMethodEntry { state, .. } | &Insn::GetGlobal { state, .. } | &Insn::GetSpecialSymbol { state, .. } | &Insn::GetSpecialNumber { state, .. } | &Insn::ObjectAllocClass { state, .. } | &Insn::SideExit { state, .. } => worklist.push_back(state), + &Insn::GuardSuperMethodEntry { lep, state, .. } => { + worklist.push_back(lep); + worklist.push_back(state); + } &Insn::UnboxFixnum { val } => worklist.push_back(val), &Insn::FixnumAref { recv, index } => { worklist.push_back(recv); @@ -4632,6 +4760,10 @@ impl Function { entry_blocks } + pub fn is_entry_block(&self, block_id: BlockId) -> bool { + self.entry_block == block_id || self.jit_entry_blocks.contains(&block_id) + } + /// Return a traversal of the `Function`'s `BlockId`s in reverse post-order. pub fn rpo(&self) -> Vec<BlockId> { let mut result = self.po_from(self.entry_blocks()); @@ -5025,17 +5157,18 @@ impl Function { | Insn::PutSpecialObject { .. } | Insn::LoadField { .. } | Insn::GetConstantPath { .. } - | Insn::IsBlockGiven + | Insn::IsBlockGiven { .. } | Insn::GetGlobal { .. } | Insn::LoadPC | Insn::LoadEC + | Insn::GetLEP | Insn::LoadSelf | Insn::Snapshot { .. } | Insn::Jump { .. } | Insn::EntryPoint { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } - | Insn::GetBlockHandler + | Insn::GetBlockHandler { .. } | Insn::PatchPoint { .. } | Insn::SideExit { .. } | Insn::IncrCounter { .. } @@ -5045,6 +5178,8 @@ impl Function { | Insn::GetSpecialNumber { .. } | Insn::GetSpecialSymbol { .. } | Insn::GetLocal { .. } + | Insn::GetBlockParam { .. } + | Insn::IsBlockParamModified { .. } | Insn::StoreField { .. } => { Ok(()) } @@ -5272,6 +5407,7 @@ impl Function { self.assert_subtype(insn_id, val, types::BasicObject)?; self.assert_subtype(insn_id, class, types::Class) } + Insn::RefineType { .. } => Ok(()), } } @@ -5455,14 +5591,28 @@ impl FrameState { state } - /// Return itself with send args reordered. Used when kwargs are reordered for callee. - fn with_reordered_args(&self, reordered_args: &[InsnId]) -> Self { + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. + /// `original_argc` is the number of args originally on the stack (before processing). + fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { let mut state = self.clone(); - let args_start = state.stack.len() - reordered_args.len(); + let args_start = state.stack.len() - original_argc; state.stack.truncate(args_start); - state.stack.extend_from_slice(reordered_args); + state.stack.extend_from_slice(new_args); state } + + fn replace(&mut self, old: InsnId, new: InsnId) { + for slot in &mut self.stack { + if *slot == old { + *slot = new; + } + } + for slot in &mut self.locals { + if *slot == old { + *slot = new; + } + } + } } /// Print adaptor for [`FrameState`]. See [`PtrPrintMap`]. @@ -6121,7 +6271,17 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let ep_offset = get_arg(pc, 0).as_u32(); let index = get_arg(pc, 1).as_u64(); let index: u8 = index.try_into().map_err(|_| ParseError::MalformedIseq(insn_idx))?; - let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + // Use FrameState to get kw_bits when possible, just like getlocal_WC_0. + let val = if !local_inval { + state.getlocal(ep_offset) + } else if ep_escaped || has_blockiseq { + fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }) + } else { + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + state.getlocal(ep_offset) + }; state.stack_push(fun.push_insn(block, Insn::FixnumBitCheck { val, index })); } YARVINSN_opt_getconstant_path => { @@ -6136,10 +6296,17 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let test_id = fun.push_insn(block, Insn::Test { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + let mut iffalse_state = state.clone(); + iffalse_state.replace(val, nil_false); let _branch_id = fun.push_insn(block, Insn::IfFalse { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + target: BranchEdge { target, args: iffalse_state.as_args(self_param) } }); + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + state.replace(val, not_nil_false); queue.push_back((state.clone(), target, target_idx, local_inval)); } YARVINSN_branchif => { @@ -6149,10 +6316,17 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let test_id = fun.push_insn(block, Insn::Test { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, not_nil_false); let _branch_id = fun.push_insn(block, Insn::IfTrue { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + target: BranchEdge { target, args: iftrue_state.as_args(self_param) } }); + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + state.replace(val, nil_false); queue.push_back((state.clone(), target, target_idx, local_inval)); } YARVINSN_branchnil => { @@ -6162,10 +6336,16 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let test_id = fun.push_insn(block, Insn::IsNil { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; + let nil = fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, nil); let _branch_id = fun.push_insn(block, Insn::IfTrue { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + target: BranchEdge { target, args: iftrue_state.as_args(self_param) } }); + let new_type = types::NotNil; + let not_nil = fun.push_insn(block, Insn::RefineType { val, new_type }); + state.replace(val, not_nil); queue.push_back((state.clone(), target, target_idx, local_inval)); } YARVINSN_opt_case_dispatch => { @@ -6275,6 +6455,112 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) })); } + YARVINSN_getblockparam => { + fn new_branch_block( + fun: &mut Function, + insn_idx: u32, + exit_state: &FrameState, + locals_count: usize, + stack_count: usize, + ) -> (BlockId, InsnId, FrameState, InsnId) { + let block = fun.new_block(insn_idx); + let self_param = fun.push_insn(block, Insn::Param); + let mut state = exit_state.clone(); + state.locals.clear(); + state.stack.clear(); + state.locals.extend((0..locals_count).map(|_| fun.push_insn(block, Insn::Param))); + state.stack.extend((0..stack_count).map(|_| fun.push_insn(block, Insn::Param))); + let snapshot = fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + (block, self_param, state, snapshot) + } + + fn finish_getblockparam_branch( + fun: &mut Function, + block: BlockId, + self_param: InsnId, + state: &mut FrameState, + join_block: BlockId, + ep_offset: u32, + level: u32, + val: InsnId, + ) { + if level == 0 { + state.setlocal(ep_offset, val); + } + state.stack_push(val); + fun.push_insn(block, Insn::Jump(BranchEdge { + target: join_block, + args: state.as_args(self_param), + })); + } + + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let branch_insn_idx = exit_state.insn_idx as u32; + + // If the block param is already a Proc (modified), read it from EP. + // Otherwise, convert it to a Proc and store it to EP. + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { level }); + + let locals_count = state.locals.len(); + let stack_count = state.stack.len(); + let entry_args = state.as_args(self_param); + + // Set up branch and join blocks. + let (modified_block, modified_self_param, mut modified_state, ..) = + new_branch_block(&mut fun, branch_insn_idx, &exit_state, locals_count, stack_count); + let (unmodified_block, unmodified_self_param, mut unmodified_state, unmodified_exit_id) = + new_branch_block(&mut fun, branch_insn_idx, &exit_state, locals_count, stack_count); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + + fun.push_insn(block, Insn::IfTrue { + val: is_modified, + target: BranchEdge { target: modified_block, args: entry_args.clone() }, + }); + fun.push_insn(block, Insn::Jump(BranchEdge { + target: unmodified_block, + args: entry_args, + })); + + // Push modified block: read Proc from EP. + let modified_val = fun.push_insn(modified_block, Insn::GetLocal { + ep_offset, + level, + use_sp: false, + rest_param: false, + }); + finish_getblockparam_branch( + &mut fun, + modified_block, + modified_self_param, + &mut modified_state, + join_block, + ep_offset, + level, + modified_val, + ); + + // Push unmodified block: convert block handler to Proc. + let unmodified_val = fun.push_insn(unmodified_block, Insn::GetBlockParam { + ep_offset, + level, + state: unmodified_exit_id, + }); + finish_getblockparam_branch( + &mut fun, + unmodified_block, + unmodified_self_param, + &mut unmodified_state, + join_block, + ep_offset, + level, + unmodified_val, + ); + + // Continue compilation from the join block at the next instruction. + queue.push_back((unmodified_state, join_block, insn_idx, local_inval)); + break; + } YARVINSN_pop => { state.stack_pop()?; } YARVINSN_dup => { state.stack_push(state.stack_top()?); } YARVINSN_dupn => { @@ -6845,10 +7131,12 @@ fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_ent // Omitted optionals are locals, so they start as nils before their code run entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); } else if Some(local_idx) == kw_bits_idx { - // We currently only support required keywords so the unspecified bits will always be zero. - // TODO: Make this a parameter when we start writing anything other than zero. - let unspecified_bits = VALUE::fixnum_from_usize(0); - entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(unspecified_bits) })); + // Read the kw_bits value written by the caller to the callee frame. + // This tells us which optional keywords were NOT provided and need their defaults evaluated. + // Note: The caller writes kw_bits to memory via gen_send_iseq_direct but does NOT pass it + // as a C argument, so we must read it from memory using GetLocal rather than Param. + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::GetLocal { level: 0, ep_offset, use_sp: false, rest_param: false })); } else if local_idx < param_size { entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Param)); } else { @@ -7582,21 +7870,23 @@ mod graphviz_tests { <TR><TD ALIGN="left" PORT="v12">PatchPoint NoTracePoint </TD></TR> <TR><TD ALIGN="left" PORT="v14">CheckInterrupts </TD></TR> <TR><TD ALIGN="left" PORT="v15">v15:CBool = Test v9 </TD></TR> - <TR><TD ALIGN="left" PORT="v16">IfFalse v15, bb3(v8, v9) </TD></TR> - <TR><TD ALIGN="left" PORT="v18">PatchPoint NoTracePoint </TD></TR> - <TR><TD ALIGN="left" PORT="v19">v19:Fixnum[3] = Const Value(3) </TD></TR> - <TR><TD ALIGN="left" PORT="v21">PatchPoint NoTracePoint </TD></TR> - <TR><TD ALIGN="left" PORT="v22">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v23">Return v19 </TD></TR> + <TR><TD ALIGN="left" PORT="v16">v16:Falsy = RefineType v9, Falsy </TD></TR> + <TR><TD ALIGN="left" PORT="v17">IfFalse v15, bb3(v8, v16) </TD></TR> + <TR><TD ALIGN="left" PORT="v18">v18:Truthy = RefineType v9, Truthy </TD></TR> + <TR><TD ALIGN="left" PORT="v20">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v21">v21:Fixnum[3] = Const Value(3) </TD></TR> + <TR><TD ALIGN="left" PORT="v23">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v24">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v25">Return v21 </TD></TR> </TABLE>>]; - bb2:v16 -> bb3:params:n; + bb2:v17 -> bb3:params:n; bb3 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> - <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb3(v24:BasicObject, v25:BasicObject) </TD></TR> - <TR><TD ALIGN="left" PORT="v28">PatchPoint NoTracePoint </TD></TR> - <TR><TD ALIGN="left" PORT="v29">v29:Fixnum[4] = Const Value(4) </TD></TR> - <TR><TD ALIGN="left" PORT="v31">PatchPoint NoTracePoint </TD></TR> - <TR><TD ALIGN="left" PORT="v32">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v33">Return v29 </TD></TR> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb3(v26:BasicObject, v27:Falsy) </TD></TR> + <TR><TD ALIGN="left" PORT="v30">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v31">v31:Fixnum[4] = Const Value(4) </TD></TR> + <TR><TD ALIGN="left" PORT="v33">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v34">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v35">Return v31 </TD></TR> </TABLE>>]; } "#); diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 60b4c25986..0110af3f2c 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -52,9 +52,10 @@ mod hir_opt_tests { bb2(v8:BasicObject, v9:NilClass): v13:TrueClass = Const Value(true) CheckInterrupts - v23:Fixnum[3] = Const Value(3) + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) CheckInterrupts - Return v23 + Return v25 "); } @@ -84,9 +85,10 @@ mod hir_opt_tests { bb2(v8:BasicObject, v9:NilClass): v13:FalseClass = Const Value(false) CheckInterrupts - v33:Fixnum[4] = Const Value(4) + v20:FalseClass = RefineType v13, Falsy + v35:Fixnum[4] = Const Value(4) CheckInterrupts - Return v33 + Return v35 "); } @@ -267,12 +269,12 @@ mod hir_opt_tests { v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) - v40:TrueClass = Const Value(true) + v42:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v22:Fixnum[3] = Const Value(3) + v24:Fixnum[3] = Const Value(3) CheckInterrupts - Return v22 + Return v24 "); } @@ -300,18 +302,18 @@ mod hir_opt_tests { v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) - v55:TrueClass = Const Value(true) + v59:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v21:Fixnum[2] = Const Value(2) v23:Fixnum[2] = Const Value(2) + v25:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) - v57:TrueClass = Const Value(true) + v61:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v33:Fixnum[3] = Const Value(3) + v37:Fixnum[3] = Const Value(3) CheckInterrupts - Return v33 + Return v37 "); } @@ -339,12 +341,12 @@ mod hir_opt_tests { v10:Fixnum[2] = Const Value(2) v12:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) - v40:TrueClass = Const Value(true) + v42:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v22:Fixnum[3] = Const Value(3) + v24:Fixnum[3] = Const Value(3) CheckInterrupts - Return v22 + Return v24 "); } @@ -372,18 +374,18 @@ mod hir_opt_tests { v10:Fixnum[2] = Const Value(2) v12:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) - v55:TrueClass = Const Value(true) + v59:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v21:Fixnum[2] = Const Value(2) v23:Fixnum[2] = Const Value(2) + v25:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) - v57:TrueClass = Const Value(true) + v61:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v33:Fixnum[3] = Const Value(3) + v37:Fixnum[3] = Const Value(3) CheckInterrupts - Return v33 + Return v37 "); } @@ -411,12 +413,12 @@ mod hir_opt_tests { v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) - v40:FalseClass = Const Value(false) + v42:FalseClass = Const Value(false) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v31:Fixnum[4] = Const Value(4) + v33:Fixnum[4] = Const Value(4) CheckInterrupts - Return v31 + Return v33 "); } @@ -444,12 +446,12 @@ mod hir_opt_tests { v10:Fixnum[2] = Const Value(2) v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) - v40:TrueClass = Const Value(true) + v42:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v22:Fixnum[3] = Const Value(3) + v24:Fixnum[3] = Const Value(3) CheckInterrupts - Return v22 + Return v24 "); } @@ -478,12 +480,12 @@ mod hir_opt_tests { v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v41:TrueClass = Const Value(true) + v43:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v22:Fixnum[3] = Const Value(3) + v24:Fixnum[3] = Const Value(3) CheckInterrupts - Return v22 + Return v24 "); } @@ -512,12 +514,12 @@ mod hir_opt_tests { v12:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v41:FalseClass = Const Value(false) + v43:FalseClass = Const Value(false) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v31:Fixnum[4] = Const Value(4) + v33:Fixnum[4] = Const Value(4) CheckInterrupts - Return v31 + Return v33 "); } @@ -628,7 +630,7 @@ mod hir_opt_tests { Jump bb2(v1, v2, v3) bb1(v6:BasicObject, v7:BasicObject): EntryPoint JIT(0) - v8:Fixnum[0] = Const Value(0) + v8:BasicObject = GetLocal <empty>, l0, EP@3 Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): CheckInterrupts @@ -830,6 +832,38 @@ mod hir_opt_tests { } #[test] + fn test_optimize_send_to_aliased_cfunc_from_module() { + eval(" + class C + include Enumerable + def each; yield 1; end + alias bar map + end + def test(o) = o.bar { |x| x } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, bar@0x1008, cme:0x1010) + v23:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:BasicObject = CCallWithFrame v23, :Enumerable#bar@0x1038, block=0x1040 + v15:BasicObject = GetLocal :o, l0, EP@3 + CheckInterrupts + Return v24 + "); + } + + #[test] fn test_optimize_nonexistent_top_level_call() { eval(" def foo @@ -1059,6 +1093,47 @@ mod hir_opt_tests { } #[test] + fn test_call_with_correct_and_too_many_args_for_method() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4) = [a, b, c, d] + def test = [target(), target(10, 20, 30), begin; target(10, 20, 30, 40, 50) rescue ArgumentError; end] + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v44:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v45:BasicObject = SendWithoutBlockDirect v44, :target (0x1038) + v14:Fixnum[10] = Const Value(10) + v16:Fixnum[20] = Const Value(20) + v18:Fixnum[30] = Const Value(30) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v49:BasicObject = SendWithoutBlockDirect v48, :target (0x1038), v14, v16, v18 + v24:Fixnum[10] = Const Value(10) + v26:Fixnum[20] = Const Value(20) + v28:Fixnum[30] = Const Value(30) + v30:Fixnum[40] = Const Value(40) + v32:Fixnum[50] = Const Value(50) + v34:BasicObject = SendWithoutBlock v6, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count + v37:ArrayExact = NewArray v45, v49, v34 + CheckInterrupts + Return v37 + "); + } + + #[test] fn test_optimize_variadic_ccall() { eval(" def test @@ -2612,10 +2687,11 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Object@0x1000) PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] - v20:BoolExact = IsBlockGiven + v20:CPtr = GetLEP + v21:BoolExact = IsBlockGiven v20 IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v20 + Return v21 "); } @@ -2638,7 +2714,7 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Object@0x1000) PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] - v20:BoolExact = IsBlockGiven + v20:FalseClass = Const Value(false) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts Return v20 @@ -2927,9 +3003,9 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Object@0x1000) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] - v24:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 CheckInterrupts - Return v24 + Return v23 "); } @@ -2994,7 +3070,7 @@ mod hir_opt_tests { } #[test] - fn dont_specialize_call_with_positional_and_optional_kw() { + fn specialize_call_with_positional_and_optional_kw() { eval(" def foo(x, a: 1) = [x, a] def test = foo(0, a: 2) @@ -3013,10 +3089,12 @@ mod hir_opt_tests { bb2(v6:BasicObject): v11:Fixnum[0] = Const Value(0) v13:Fixnum[2] = Const Value(2) - IncrCounter complex_arg_pass_param_kw_opt - v15:BasicObject = SendWithoutBlock v6, :foo, v11, v13 # SendFallbackReason: Complex argument passing + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 CheckInterrupts - Return v15 + Return v23 "); } @@ -3044,22 +3122,105 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Object@0x1000) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v37:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] - v39:BasicObject = SendWithoutBlockDirect v37, :foo (0x1038), v11, v13, v15 + v38:BasicObject = SendWithoutBlockDirect v37, :foo (0x1038), v11, v13, v15 v20:Fixnum[1] = Const Value(1) v22:Fixnum[2] = Const Value(2) v24:Fixnum[4] = Const Value(4) v26:Fixnum[3] = Const Value(3) PatchPoint NoSingletonClass(Object@0x1000) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v42:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] - v44:BasicObject = SendWithoutBlockDirect v42, :foo (0x1038), v20, v22, v26, v24 - v30:ArrayExact = NewArray v39, v44 + v41:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v43:BasicObject = SendWithoutBlockDirect v41, :foo (0x1038), v20, v22, v26, v24 + v30:ArrayExact = NewArray v38, v43 CheckInterrupts Return v30 "); } #[test] + fn specialize_call_with_pos_optional_and_kw_optional() { + eval(" + def foo(r, x = 2, a:, b: 4) = [r, x, a, b] + def test = [foo(1, a: 3), foo(1, 2, b: 40, a: 30)] # with and without the optionals + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v35:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v36:Fixnum[4] = Const Value(4) + v38:BasicObject = SendWithoutBlockDirect v35, :foo (0x1038), v11, v13, v36 + v18:Fixnum[1] = Const Value(1) + v20:Fixnum[2] = Const Value(2) + v22:Fixnum[40] = Const Value(40) + v24:Fixnum[30] = Const Value(30) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v41:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v43:BasicObject = SendWithoutBlockDirect v41, :foo (0x1038), v18, v20, v24, v22 + v28:ArrayExact = NewArray v38, v43 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_call_with_pos_optional_and_maybe_too_many_args() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[6] = Const Value(6) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v49:BasicObject = SendWithoutBlockDirect v48, :target (0x1038), v11 + v16:Fixnum[10] = Const Value(10) + v18:Fixnum[20] = Const Value(20) + v20:Fixnum[30] = Const Value(30) + v22:Fixnum[6] = Const Value(6) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v52:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v53:BasicObject = SendWithoutBlockDirect v52, :target (0x1038), v16, v18, v20, v22 + v27:Fixnum[10] = Const Value(10) + v29:Fixnum[20] = Const Value(20) + v31:Fixnum[30] = Const Value(30) + v33:Fixnum[40] = Const Value(40) + v35:Fixnum[50] = Const Value(50) + v37:Fixnum[60] = Const Value(60) + v39:BasicObject = SendWithoutBlock v6, :target, v27, v29, v31, v33, v35, v37 # SendFallbackReason: Too many arguments for LIR + v41:ArrayExact = NewArray v49, v53, v39 + CheckInterrupts + Return v41 + "); + } + + #[test] fn test_send_call_to_iseq_with_optional_kw() { eval(" def foo(a: 1) = a @@ -3078,10 +3239,12 @@ mod hir_opt_tests { Jump bb2(v4) bb2(v6:BasicObject): v11:Fixnum[2] = Const Value(2) - IncrCounter complex_arg_pass_param_kw_opt - v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 CheckInterrupts - Return v13 + Return v21 "); } @@ -3112,7 +3275,7 @@ mod hir_opt_tests { } #[test] - fn dont_specialize_call_to_iseq_with_optional_param_kw() { + fn specialize_call_to_iseq_with_optional_param_kw_using_default() { eval(" def foo(int: 1) = int + 1 def test = foo @@ -3129,10 +3292,13 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v4) bb2(v6:BasicObject): - IncrCounter complex_arg_pass_param_kw_opt - v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Complex argument passing + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:Fixnum[1] = Const Value(1) + v21:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038), v19 CheckInterrupts - Return v11 + Return v21 "); } @@ -3506,7 +3672,6 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Hash@0x1008, new@0x1009, cme:0x1010) v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) IncrCounter complex_arg_pass_param_block - IncrCounter complex_arg_pass_param_kw_opt v20:BasicObject = SendWithoutBlock v46, :initialize # SendFallbackReason: Complex argument passing CheckInterrupts CheckInterrupts @@ -3720,6 +3885,67 @@ mod hir_opt_tests { } #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:CBool = IsBlockParamModified l0 + IfTrue v13, bb3(v8, v9) + v24:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb5(v8, v24, v24) + bb3(v14:BasicObject, v15:BasicObject): + v22:BasicObject = GetLocal :block, l0, EP@3 + Jump bb5(v14, v22, v22) + bb5(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @r" + fn block in test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:CBool = IsBlockParamModified l1 + IfTrue v10, bb3(v6) + v19:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb5(v6, v19) + bb3(v11:BasicObject): + v17:BasicObject = GetLocal :block, l1, EP@3 + Jump bb5(v11, v17) + bb5(v21:BasicObject, v22:BasicObject): + CheckInterrupts + Return v22 + "); + } + + #[test] fn test_getinstancevariable() { eval(" def test = @foo @@ -4861,8 +5087,9 @@ mod hir_opt_tests { bb2(v8:BasicObject, v9:NilClass): v13:NilClass = Const Value(nil) CheckInterrupts + v21:NilClass = Const Value(nil) CheckInterrupts - Return v13 + Return v21 "); } @@ -4889,10 +5116,11 @@ mod hir_opt_tests { bb2(v8:BasicObject, v9:NilClass): v13:Fixnum[1] = Const Value(1) CheckInterrupts + v23:Fixnum[1] = RefineType v13, NotNil PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v13 + Return v23 "); } @@ -5709,20 +5937,22 @@ mod hir_opt_tests { bb2(v8:BasicObject, v9:BasicObject): CheckInterrupts v15:CBool = Test v9 - IfFalse v15, bb3(v8, v9) - v18:FalseClass = Const Value(false) + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + v20:FalseClass = Const Value(false) CheckInterrupts - Jump bb4(v8, v9, v18) - bb3(v22:BasicObject, v23:BasicObject): - v26:NilClass = Const Value(nil) - Jump bb4(v22, v23, v26) - bb4(v28:BasicObject, v29:BasicObject, v30:NilClass|FalseClass): + Jump bb4(v8, v18, v20) + bb3(v24:BasicObject, v25:Falsy): + v28:NilClass = Const Value(nil) + Jump bb4(v24, v25, v28) + bb4(v30:BasicObject, v31:BasicObject, v32:Falsy): PatchPoint MethodRedefined(NilClass@0x1000, !@0x1008, cme:0x1010) - v41:NilClass = GuardType v30, NilClass - v42:TrueClass = Const Value(true) + v43:NilClass = GuardType v32, NilClass + v44:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v42 + Return v44 "); } @@ -9836,7 +10066,6 @@ mod hir_opt_tests { IncrCounter complex_arg_pass_param_rest IncrCounter complex_arg_pass_param_block IncrCounter complex_arg_pass_param_kwrest - IncrCounter complex_arg_pass_param_kw_opt v13:BasicObject = SendWithoutBlock v6, :fancy, v11 # SendFallbackReason: Complex argument passing CheckInterrupts Return v13 @@ -9929,9 +10158,9 @@ mod hir_opt_tests { bb2(v6:BasicObject): PatchPoint NoSingletonClass(C@0x1000) PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) - v40:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + v42:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] IncrCounter inline_iseq_optimized_send_count - v44:Class[C@0x1000] = Const Value(VALUE(0x1000)) + v46:Class[C@0x1000] = Const Value(VALUE(0x1000)) IncrCounter inline_cfunc_optimized_send_count v13:StaticSymbol[:_lex_actions] = Const Value(VALUE(0x1038)) v15:TrueClass = Const Value(true) @@ -9939,12 +10168,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Class@0x1040, respond_to?@0x1048, cme:0x1050) PatchPoint NoSingletonClass(Class@0x1040) PatchPoint MethodRedefined(Class@0x1040, _lex_actions@0x1078, cme:0x1080) - v52:TrueClass = Const Value(true) + v54:TrueClass = Const Value(true) IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - v24:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) + v26:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) CheckInterrupts - Return v24 + Return v26 "); } @@ -10100,23 +10329,23 @@ mod hir_opt_tests { CheckInterrupts SetLocal :formatted, l0, EP@3, v15 PatchPoint SingleRactorMode - v54:HeapBasicObject = GuardType v14, HeapBasicObject - v55:CShape = LoadField v54, :_shape_id@0x1000 - v56:CShape[0x1001] = GuardBitEquals v55, CShape(0x1001) - StoreField v54, :@formatted@0x1002, v15 - WriteBarrier v54, v15 - v59:CShape[0x1003] = Const CShape(0x1003) - StoreField v54, :_shape_id@0x1000, v59 - v43:Class[VMFrozenCore] = Const Value(VALUE(0x1008)) + v56:HeapBasicObject = GuardType v14, HeapBasicObject + v57:CShape = LoadField v56, :_shape_id@0x1000 + v58:CShape[0x1001] = GuardBitEquals v57, CShape(0x1001) + StoreField v56, :@formatted@0x1002, v15 + WriteBarrier v56, v15 + v61:CShape[0x1003] = Const CShape(0x1003) + StoreField v56, :_shape_id@0x1000, v61 + v45:Class[VMFrozenCore] = Const Value(VALUE(0x1008)) PatchPoint NoSingletonClass(Class@0x1010) PatchPoint MethodRedefined(Class@0x1010, lambda@0x1018, cme:0x1020) - v64:BasicObject = CCallWithFrame v43, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 - v46:BasicObject = GetLocal :a, l0, EP@6 - v47:BasicObject = GetLocal :_b, l0, EP@5 - v48:BasicObject = GetLocal :_c, l0, EP@4 - v49:BasicObject = GetLocal :formatted, l0, EP@3 + v66:BasicObject = CCallWithFrame v45, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 + v48:BasicObject = GetLocal :a, l0, EP@6 + v49:BasicObject = GetLocal :_b, l0, EP@5 + v50:BasicObject = GetLocal :_c, l0, EP@4 + v51:BasicObject = GetLocal :formatted, l0, EP@3 CheckInterrupts - Return v64 + Return v66 "); } @@ -10814,12 +11043,13 @@ mod hir_opt_tests { Jump bb2(v4) bb2(v6:BasicObject): PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) - GuardSuperMethodEntry 0x1038 - v18:RubyValue = GetBlockHandler - v19:FalseClass = GuardBitEquals v18, Value(false) - v20:BasicObject = SendWithoutBlockDirect v6, :foo (0x1040) + v17:CPtr = GetLEP + GuardSuperMethodEntry v17, 0x1038 + v19:RubyValue = GetBlockHandler v17 + v20:FalseClass = GuardBitEquals v19, Value(false) + v21:BasicObject = SendWithoutBlockDirect v6, :foo (0x1040) CheckInterrupts - Return v20 + Return v21 "); } @@ -10857,17 +11087,18 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) - GuardSuperMethodEntry 0x1038 - v27:RubyValue = GetBlockHandler - v28:FalseClass = GuardBitEquals v27, Value(false) - v29:BasicObject = SendWithoutBlockDirect v8, :foo (0x1040), v9 + v26:CPtr = GetLEP + GuardSuperMethodEntry v26, 0x1038 + v28:RubyValue = GetBlockHandler v26 + v29:FalseClass = GuardBitEquals v28, Value(false) + v30:BasicObject = SendWithoutBlockDirect v8, :foo (0x1040), v9 v17:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v32:Fixnum = GuardType v29, Fixnum - v33:Fixnum = FixnumAdd v32, v17 + v33:Fixnum = GuardType v30, Fixnum + v34:Fixnum = FixnumAdd v33, v17 IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v33 + Return v34 "); } @@ -11120,4 +11351,53 @@ mod hir_opt_tests { Return v31 "); } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb5(v8, v16) + v18:Truthy = RefineType v9, Truthy + CheckInterrupts + v26:Truthy = RefineType v18, Truthy + CheckInterrupts + v34:Truthy = RefineType v26, Truthy + v37:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v37 + bb5(v42:BasicObject, v43:Falsy): + v47:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v47 + "); + } } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 1ce5488f47..56f1928f1f 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -114,12 +114,11 @@ mod snapshot_tests { PatchPoint NoSingletonClass(Object@0x1010) PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) v22:HeapObject[class_exact*:Object@VALUE(0x1010)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1010)] - v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } - v24:BasicObject = SendWithoutBlockDirect v22, :foo (0x1048), v11, v13 - v16:Any = Snapshot FrameState { pc: 0x1050, stack: [v24], locals: [] } + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1048), v11, v13 + v16:Any = Snapshot FrameState { pc: 0x1050, stack: [v23], locals: [] } PatchPoint NoTracePoint CheckInterrupts - Return v24 + Return v23 "); } @@ -1084,14 +1083,16 @@ pub mod hir_build_tests { v10:TrueClass|NilClass = DefinedIvar v6, :@foo CheckInterrupts v13:CBool = Test v10 + v14:NilClass = RefineType v10, Falsy IfFalse v13, bb3(v6) - v17:Fixnum[3] = Const Value(3) + v16:TrueClass = RefineType v10, Truthy + v19:Fixnum[3] = Const Value(3) CheckInterrupts - Return v17 - bb3(v22:BasicObject): - v26:Fixnum[4] = Const Value(4) + Return v19 + bb3(v24:BasicObject): + v28:Fixnum[4] = Const Value(4) CheckInterrupts - Return v26 + Return v28 "); } @@ -1147,14 +1148,16 @@ pub mod hir_build_tests { bb2(v8:BasicObject, v9:BasicObject): CheckInterrupts v15:CBool = Test v9 - IfFalse v15, bb3(v8, v9) - v19:Fixnum[3] = Const Value(3) + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + v21:Fixnum[3] = Const Value(3) CheckInterrupts - Return v19 - bb3(v24:BasicObject, v25:BasicObject): - v29:Fixnum[4] = Const Value(4) + Return v21 + bb3(v26:BasicObject, v27:Falsy): + v31:Fixnum[4] = Const Value(4) CheckInterrupts - Return v29 + Return v31 "); } @@ -1185,16 +1188,18 @@ pub mod hir_build_tests { bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): CheckInterrupts v18:CBool = Test v11 - IfFalse v18, bb3(v10, v11, v12) - v22:Fixnum[3] = Const Value(3) + v19:Falsy = RefineType v11, Falsy + IfFalse v18, bb3(v10, v19, v12) + v21:Truthy = RefineType v11, Truthy + v24:Fixnum[3] = Const Value(3) CheckInterrupts - Jump bb4(v10, v11, v22) - bb3(v27:BasicObject, v28:BasicObject, v29:NilClass): - v33:Fixnum[4] = Const Value(4) - Jump bb4(v27, v28, v33) - bb4(v36:BasicObject, v37:BasicObject, v38:Fixnum): + Jump bb4(v10, v21, v24) + bb3(v29:BasicObject, v30:Falsy, v31:NilClass): + v35:Fixnum[4] = Const Value(4) + Jump bb4(v29, v30, v35) + bb4(v38:BasicObject, v39:BasicObject, v40:Fixnum): CheckInterrupts - Return v38 + Return v40 "); } @@ -1485,16 +1490,18 @@ pub mod hir_build_tests { v35:BasicObject = SendWithoutBlock v28, :>, v32 # SendFallbackReason: Uncategorized(opt_gt) CheckInterrupts v38:CBool = Test v35 + v39:Truthy = RefineType v35, Truthy IfTrue v38, bb3(v26, v27, v28) - v41:NilClass = Const Value(nil) + v41:Falsy = RefineType v35, Falsy + v43:NilClass = Const Value(nil) CheckInterrupts Return v27 - bb3(v49:BasicObject, v50:BasicObject, v51:BasicObject): - v56:Fixnum[1] = Const Value(1) - v59:BasicObject = SendWithoutBlock v50, :+, v56 # SendFallbackReason: Uncategorized(opt_plus) - v64:Fixnum[1] = Const Value(1) - v67:BasicObject = SendWithoutBlock v51, :-, v64 # SendFallbackReason: Uncategorized(opt_minus) - Jump bb4(v49, v59, v67) + bb3(v51:BasicObject, v52:BasicObject, v53:BasicObject): + v58:Fixnum[1] = Const Value(1) + v61:BasicObject = SendWithoutBlock v52, :+, v58 # SendFallbackReason: Uncategorized(opt_plus) + v66:Fixnum[1] = Const Value(1) + v69:BasicObject = SendWithoutBlock v53, :-, v66 # SendFallbackReason: Uncategorized(opt_minus) + Jump bb4(v51, v61, v69) "); } @@ -1550,14 +1557,16 @@ pub mod hir_build_tests { v13:TrueClass = Const Value(true) CheckInterrupts v19:CBool[true] = Test v13 - IfFalse v19, bb3(v8, v13) - v23:Fixnum[3] = Const Value(3) + v20 = RefineType v13, Falsy + IfFalse v19, bb3(v8, v20) + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) CheckInterrupts - Return v23 - bb3(v28, v29): - v33 = Const Value(4) + Return v25 + bb3(v30, v31): + v35 = Const Value(4) CheckInterrupts - Return v33 + Return v35 "); } @@ -2675,6 +2684,71 @@ pub mod hir_build_tests { } #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:CBool = IsBlockParamModified l0 + IfTrue v13, bb3(v8, v9) + Jump bb4(v8, v9) + bb3(v14:BasicObject, v15:BasicObject): + v22:BasicObject = GetLocal :block, l0, EP@3 + Jump bb5(v14, v22, v22) + bb4(v17:BasicObject, v18:BasicObject): + v24:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb5(v17, v24, v24) + bb5(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @r" + fn block in test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:CBool = IsBlockParamModified l1 + IfTrue v10, bb3(v6) + Jump bb4(v6) + bb3(v11:BasicObject): + v17:BasicObject = GetLocal :block, l1, EP@3 + Jump bb5(v11, v17) + bb4(v13:BasicObject): + v19:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb5(v13, v19) + bb5(v21:BasicObject, v22:BasicObject): + CheckInterrupts + Return v22 + "); + } + + #[test] fn test_splatarray_mut() { eval(" def test(a) = [*a] @@ -3091,12 +3165,123 @@ pub mod hir_build_tests { bb2(v8:BasicObject, v9:BasicObject): CheckInterrupts v16:CBool = IsNil v9 - IfTrue v16, bb3(v8, v9, v9) - v19:BasicObject = SendWithoutBlock v9, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) - Jump bb3(v8, v9, v19) - bb3(v21:BasicObject, v22:BasicObject, v23:BasicObject): + v17:NilClass = Const Value(nil) + IfTrue v16, bb3(v8, v17, v17) + v19:NotNil = RefineType v9, NotNil + v21:BasicObject = SendWithoutBlock v19, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb3(v8, v19, v21) + bb3(v23:BasicObject, v24:BasicObject, v25:BasicObject): CheckInterrupts - Return v23 + Return v25 + "); + } + + #[test] + fn test_infer_nilability_from_branchif() { + eval(" + def test(x) + if x + x&.itself + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_branchnil); + // Note that IsNil has as its operand a value that we know statically *cannot* be nil + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb3(v8, v16) + v18:Truthy = RefineType v9, Truthy + CheckInterrupts + v24:CBool[false] = IsNil v18 + v25:NilClass = Const Value(nil) + IfTrue v24, bb4(v8, v25, v25) + v27:Truthy = RefineType v18, NotNil + v29:BasicObject = SendWithoutBlock v27, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v29 + bb3(v34:BasicObject, v35:Falsy): + v39:Fixnum[4] = Const Value(4) + Jump bb4(v34, v35, v39) + bb4(v41:BasicObject, v42:Falsy, v43:Fixnum[4]): + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + v16:Falsy = RefineType v9, Falsy + IfFalse v15, bb5(v8, v16) + v18:Truthy = RefineType v9, Truthy + CheckInterrupts + v23:CBool[true] = Test v18 + v24 = RefineType v18, Falsy + IfFalse v23, bb4(v8, v24) + v26:Truthy = RefineType v18, Truthy + CheckInterrupts + v31:CBool[true] = Test v26 + v32 = RefineType v26, Falsy + IfFalse v31, bb3(v8, v32) + v34:Truthy = RefineType v26, Truthy + v37:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v37 + bb5(v42:BasicObject, v43:Falsy): + v47:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v47 + bb4(v52, v53): + v57 = Const Value(5) + CheckInterrupts + Return v57 + bb3(v62, v63): + v67 = Const Value(4) + CheckInterrupts + Return v67 "); } @@ -3114,7 +3299,7 @@ pub mod hir_build_tests { Jump bb2(v1, v2, v3, v4) bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject): EntryPoint JIT(0) - v10:Fixnum[0] = Const Value(0) + v10:BasicObject = GetLocal <empty>, l0, EP@3 Jump bb2(v7, v8, v9, v10) bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): v19:Float = InvokeBuiltin rb_f_float, v12, v13, v14 @@ -3165,7 +3350,7 @@ pub mod hir_build_tests { Jump bb2(v1, v2, v3, v4, v5, v6) bb1(v9:BasicObject, v10:BasicObject, v11:BasicObject, v13:BasicObject): EntryPoint JIT(0) - v12:Fixnum[0] = Const Value(0) + v12:BasicObject = GetLocal <empty>, l0, EP@5 v14:NilClass = Const Value(nil) Jump bb2(v9, v10, v11, v12, v13, v14) bb2(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:NilClass): @@ -3175,14 +3360,16 @@ pub mod hir_build_tests { v32:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) CheckInterrupts v35:CBool[true] = Test v32 + v36 = RefineType v32, Falsy IfFalse v35, bb3(v16, v17, v18, v19, v20, v25) - v40:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) - v43:BasicObject = InvokeBuiltin dir_s_close, v16, v25 + v38:HeapObject[BlockParamProxy] = RefineType v32, Truthy + v42:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) + v45:BasicObject = InvokeBuiltin dir_s_close, v16, v25 CheckInterrupts - Return v40 - bb3(v49, v50, v51, v52, v53, v54): + Return v42 + bb3(v51, v52, v53, v54, v55, v56): CheckInterrupts - Return v54 + Return v56 "); } @@ -3226,7 +3413,7 @@ pub mod hir_build_tests { Jump bb2(v1, v2, v3, v4, v5) bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject): EntryPoint JIT(0) - v12:Fixnum[0] = Const Value(0) + v12:BasicObject = GetLocal <empty>, l0, EP@3 Jump bb2(v8, v9, v10, v11, v12) bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:BasicObject): v25:FalseClass = Const Value(false) @@ -3303,14 +3490,16 @@ pub mod hir_build_tests { v21:BasicObject = SendWithoutBlock v9, :[], v16, v18 # SendFallbackReason: Uncategorized(opt_send_without_block) CheckInterrupts v25:CBool = Test v21 - IfTrue v25, bb3(v8, v9, v13, v9, v16, v18, v21) - v29:Fixnum[2] = Const Value(2) - v32:BasicObject = SendWithoutBlock v9, :[]=, v16, v18, v29 # SendFallbackReason: Uncategorized(opt_send_without_block) + v26:Truthy = RefineType v21, Truthy + IfTrue v25, bb3(v8, v9, v13, v9, v16, v18, v26) + v28:Falsy = RefineType v21, Falsy + v31:Fixnum[2] = Const Value(2) + v34:BasicObject = SendWithoutBlock v9, :[]=, v16, v18, v31 # SendFallbackReason: Uncategorized(opt_send_without_block) CheckInterrupts - Return v29 - bb3(v38:BasicObject, v39:BasicObject, v40:NilClass, v41:BasicObject, v42:Fixnum[0], v43:Fixnum[1], v44:BasicObject): + Return v31 + bb3(v40:BasicObject, v41:BasicObject, v42:NilClass, v43:BasicObject, v44:Fixnum[0], v45:Fixnum[1], v46:Truthy): CheckInterrupts - Return v44 + Return v46 "); } @@ -3647,21 +3836,22 @@ pub mod hir_build_tests { Jump bb2(v1, v2, v3) bb1(v6:BasicObject, v7:BasicObject): EntryPoint JIT(0) - v8:Fixnum[0] = Const Value(0) + v8:BasicObject = GetLocal <empty>, l0, EP@3 Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - v15:BasicObject = GetLocal <empty>, l0, EP@3 - v16:BoolExact = FixnumBitCheck v15, 0 + v15:BoolExact = FixnumBitCheck v12, 0 CheckInterrupts - v19:CBool = Test v16 - IfTrue v19, bb3(v10, v11, v12) - v22:Fixnum[1] = Const Value(1) - v24:Fixnum[1] = Const Value(1) - v27:BasicObject = SendWithoutBlock v22, :+, v24 # SendFallbackReason: Uncategorized(opt_plus) - Jump bb3(v10, v27, v12) - bb3(v30:BasicObject, v31:BasicObject, v32:BasicObject): + v18:CBool = Test v15 + v19:TrueClass = RefineType v15, Truthy + IfTrue v18, bb3(v10, v11, v12) + v21:FalseClass = RefineType v15, Falsy + v23:Fixnum[1] = Const Value(1) + v25:Fixnum[1] = Const Value(1) + v28:BasicObject = SendWithoutBlock v23, :+, v25 # SendFallbackReason: Uncategorized(opt_plus) + Jump bb3(v10, v28, v12) + bb3(v31:BasicObject, v32:BasicObject, v33:BasicObject): CheckInterrupts - Return v31 + Return v32 "); } @@ -3719,7 +3909,7 @@ pub mod hir_build_tests { Jump bb2(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) bb1(v38:BasicObject, v39:BasicObject, v40:BasicObject, v41:BasicObject, v42:BasicObject, v43:BasicObject, v44:BasicObject, v45:BasicObject, v46:BasicObject, v47:BasicObject, v48:BasicObject, v49:BasicObject, v50:BasicObject, v51:BasicObject, v52:BasicObject, v53:BasicObject, v54:BasicObject, v55:BasicObject, v56:BasicObject, v57:BasicObject, v58:BasicObject, v59:BasicObject, v60:BasicObject, v61:BasicObject, v62:BasicObject, v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject, v69:BasicObject, v70:BasicObject, v71:BasicObject): EntryPoint JIT(0) - v72:Fixnum[0] = Const Value(0) + v72:BasicObject = GetLocal <empty>, l0, EP@3 Jump bb2(v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63, v64, v65, v66, v67, v68, v69, v70, v71, v72) bb2(v74:BasicObject, v75:BasicObject, v76:BasicObject, v77:BasicObject, v78:BasicObject, v79:BasicObject, v80:BasicObject, v81:BasicObject, v82:BasicObject, v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject, v91:BasicObject, v92:BasicObject, v93:BasicObject, v94:BasicObject, v95:BasicObject, v96:BasicObject, v97:BasicObject, v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject, v102:BasicObject, v103:BasicObject, v104:BasicObject, v105:BasicObject, v106:BasicObject, v107:BasicObject, v108:BasicObject): SideExit TooManyKeywordParameters diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb index 9576d2b1c0..f952a8b715 100644 --- a/zjit/src/hir_type/gen_hir_type.rb +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -178,10 +178,15 @@ add_union "BuiltinExact", $builtin_exact add_union "Subclass", $subclass add_union "BoolExact", [true_exact.name, false_exact.name] add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] +add_union "Falsy", [nil_exact.name, false_exact.name] $bits["HeapBasicObject"] = ["BasicObject & !Immediate"] $numeric_bits["HeapBasicObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] $bits["HeapObject"] = ["Object & !Immediate"] $numeric_bits["HeapObject"] = $numeric_bits["Object"] & ~$numeric_bits["Immediate"] +$bits["Truthy"] = ["BasicObject & !Falsy"] +$numeric_bits["Truthy"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Falsy"] +$bits["NotNil"] = ["BasicObject & !NilClass"] +$numeric_bits["NotNil"] = $numeric_bits["BasicObject"] & ~$numeric_bits["NilClass"] # ===== Finished generating the DAG; write Rust code ===== diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs index b388b3a0d1..886b4b54dd 100644 --- a/zjit/src/hir_type/hir_type.inc.rs +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -32,6 +32,7 @@ mod bits { pub const DynamicSymbol: u64 = 1u64 << 20; pub const Empty: u64 = 0u64; pub const FalseClass: u64 = 1u64 << 21; + pub const Falsy: u64 = FalseClass | NilClass; pub const Fixnum: u64 = 1u64 << 22; pub const Float: u64 = Flonum | HeapFloat; pub const Flonum: u64 = 1u64 << 23; @@ -47,6 +48,7 @@ mod bits { pub const ModuleExact: u64 = 1u64 << 27; pub const ModuleSubclass: u64 = 1u64 << 28; pub const NilClass: u64 = 1u64 << 29; + pub const NotNil: u64 = BasicObject & !NilClass; pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; pub const NumericExact: u64 = 1u64 << 30; pub const NumericSubclass: u64 = 1u64 << 31; @@ -70,14 +72,17 @@ mod bits { pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; pub const Symbol: u64 = DynamicSymbol | StaticSymbol; pub const TrueClass: u64 = 1u64 << 43; + pub const Truthy: u64 = BasicObject & !Falsy; pub const Undef: u64 = 1u64 << 44; - pub const AllBitPatterns: [(&str, u64); 71] = [ + pub const AllBitPatterns: [(&str, u64); 74] = [ ("Any", Any), ("RubyValue", RubyValue), ("Immediate", Immediate), ("Undef", Undef), ("BasicObject", BasicObject), ("Object", Object), + ("NotNil", NotNil), + ("Truthy", Truthy), ("BuiltinExact", BuiltinExact), ("BoolExact", BoolExact), ("TrueClass", TrueClass), @@ -103,6 +108,7 @@ mod bits { ("Numeric", Numeric), ("NumericSubclass", NumericSubclass), ("NumericExact", NumericExact), + ("Falsy", Falsy), ("NilClass", NilClass), ("Module", Module), ("ModuleSubclass", ModuleSubclass), @@ -180,6 +186,7 @@ pub mod types { pub const DynamicSymbol: Type = Type::from_bits(bits::DynamicSymbol); pub const Empty: Type = Type::from_bits(bits::Empty); pub const FalseClass: Type = Type::from_bits(bits::FalseClass); + pub const Falsy: Type = Type::from_bits(bits::Falsy); pub const Fixnum: Type = Type::from_bits(bits::Fixnum); pub const Float: Type = Type::from_bits(bits::Float); pub const Flonum: Type = Type::from_bits(bits::Flonum); @@ -195,6 +202,7 @@ pub mod types { pub const ModuleExact: Type = Type::from_bits(bits::ModuleExact); pub const ModuleSubclass: Type = Type::from_bits(bits::ModuleSubclass); pub const NilClass: Type = Type::from_bits(bits::NilClass); + pub const NotNil: Type = Type::from_bits(bits::NotNil); pub const Numeric: Type = Type::from_bits(bits::Numeric); pub const NumericExact: Type = Type::from_bits(bits::NumericExact); pub const NumericSubclass: Type = Type::from_bits(bits::NumericSubclass); @@ -218,6 +226,7 @@ pub mod types { pub const Subclass: Type = Type::from_bits(bits::Subclass); pub const Symbol: Type = Type::from_bits(bits::Symbol); pub const TrueClass: Type = Type::from_bits(bits::TrueClass); + pub const Truthy: Type = Type::from_bits(bits::Truthy); pub const Undef: Type = Type::from_bits(bits::Undef); pub const ExactBitsAndClass: [(u64, *const VALUE); 17] = [ (bits::ObjectExact, &raw const crate::cruby::rb_cObject), diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index d183eb18ab..f1180acf2a 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -16,6 +16,7 @@ macro_rules! compile_patch_points { for patch_point in $patch_points { let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { let mut asm = Assembler::new(); + asm.new_block_without_id(); asm_comment!(asm, $($comment_args)*); asm.jmp(patch_point.side_exit_ptr.into()); asm.compile(cb).expect("can write existing code"); diff --git a/zjit/src/options.rs b/zjit/src/options.rs index 40b49146b7..9121e49bff 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -180,6 +180,8 @@ pub enum DumpLIR { alloc_regs, /// Dump LIR after compile_exits compile_exits, + /// Dump LIR after resolve_parallel_mov + resolve_parallel_mov, /// Dump LIR after {arch}_scratch_split scratch_split, } @@ -190,6 +192,7 @@ const DUMP_LIR_ALL: &[DumpLIR] = &[ DumpLIR::split, DumpLIR::alloc_regs, DumpLIR::compile_exits, + DumpLIR::resolve_parallel_mov, DumpLIR::scratch_split, ]; diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 7a584afd6f..c1feb75952 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -159,23 +159,8 @@ fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE - match profile.super_cme.get(&profiler.insn_idx) { - None => { - // If `None`, then this is our first time looking at `super` for this instruction. - profile.super_cme.insert(profiler.insn_idx, Some(cme_value)); - }, - Some(Some(existing_cme)) => { - // Check if the stored method entry is the same as the current one. If it isn't, then - // mark the call site as polymorphic. - if *existing_cme != cme_value { - profile.super_cme.insert(profiler.insn_idx, None); - } - } - Some(None) => { - // We've visited this instruction and explicitly stored `None` to mark the call site - // as polymorphic. - } - } + profile.super_cme.entry(profiler.insn_idx) + .or_insert_with(|| TypeDistribution::new()).observe(ProfiledType::object(cme_value)); unsafe { rb_gc_writebarrier(profiler.iseq.into(), cme_value) }; @@ -359,7 +344,7 @@ pub struct IseqProfile { num_profiles: Vec<NumProfiles>, /// Method entries for `super` calls (stored as VALUE to be GC-safe) - super_cme: HashMap<usize, Option<VALUE>> + super_cme: HashMap<usize, TypeDistribution> } impl IseqProfile { @@ -377,8 +362,14 @@ impl IseqProfile { } pub fn get_super_method_entry(&self, insn_idx: usize) -> Option<*const rb_callable_method_entry_t> { - self.super_cme.get(&insn_idx) - .and_then(|opt| opt.map(|v| v.0 as *const rb_callable_method_entry_t)) + let Some(entry) = self.super_cme.get(&insn_idx) else { return None }; + let summary = TypeDistributionSummary::new(entry); + + if summary.is_monomorphic() { + Some(summary.bucket(0).class.0 as *const rb_callable_method_entry_t) + } else { + None + } } /// Run a given callback with every object in IseqProfile @@ -392,9 +383,9 @@ impl IseqProfile { } } - for cme_value in self.super_cme.values() { - if let Some(cme) = cme_value { - callback(*cme); + for super_cme_values in self.super_cme.values() { + for profiled_type in super_cme_values.each_item() { + callback(profiled_type.class) } } } @@ -411,9 +402,9 @@ impl IseqProfile { } // Update CME references if they move during compaction. - for cme_value in self.super_cme.values_mut() { - if let Some(cme) = cme_value { - callback(cme); + for super_cme_values in self.super_cme.values_mut() { + for ref mut profiled_type in super_cme_values.each_item_mut() { + callback(&mut profiled_type.class) } } } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 506bd82686..556a1417a4 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -210,6 +210,7 @@ make_counters! { exit_stackoverflow, exit_block_param_proxy_modified, exit_block_param_proxy_not_iseq_or_ifunc, + exit_block_param_wb_required, exit_too_many_keyword_parameters, } @@ -228,9 +229,9 @@ make_counters! { send_fallback_send_without_block_bop_redefined, send_fallback_send_without_block_operands_not_fixnum, send_fallback_send_without_block_direct_keyword_mismatch, - send_fallback_send_without_block_direct_optional_keywords, send_fallback_send_without_block_direct_keyword_count_mismatch, send_fallback_send_without_block_direct_missing_keyword, + send_fallback_send_without_block_direct_too_many_keywords, send_fallback_send_polymorphic, send_fallback_send_megamorphic, send_fallback_send_no_profiles, @@ -306,6 +307,7 @@ make_counters! { compile_error_iseq_stack_too_large, compile_error_exception_handler, compile_error_out_of_memory, + compile_error_label_linking_failure, compile_error_jit_to_jit_optional, compile_error_register_spill_on_ccall, compile_error_register_spill_on_alloc, @@ -386,7 +388,6 @@ make_counters! { // Unsupported parameter features complex_arg_pass_param_rest, complex_arg_pass_param_post, - complex_arg_pass_param_kw_opt, complex_arg_pass_param_kwrest, complex_arg_pass_param_block, complex_arg_pass_param_forwardable, @@ -466,6 +467,10 @@ pub enum CompileError { ExceptionHandler, OutOfMemory, ParseError(ParseError), + /// When a ZJIT function is too large, the branches may have + /// offsets that don't fit in one instruction. We error in + /// error that case. + LabelLinkingFailure, } /// Return a raw pointer to the exit counter for a given CompileError @@ -479,6 +484,7 @@ pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { IseqStackTooLarge => compile_error_iseq_stack_too_large, ExceptionHandler => compile_error_exception_handler, OutOfMemory => compile_error_out_of_memory, + LabelLinkingFailure => compile_error_label_linking_failure, ParseError(parse_error) => match parse_error { StackUnderflow(_) => compile_error_parse_stack_underflow, MalformedIseq(_) => compile_error_parse_malformed_iseq, @@ -552,6 +558,7 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { StackOverflow => exit_stackoverflow, BlockParamProxyModified => exit_block_param_proxy_modified, BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + BlockParamWbRequired => exit_block_param_wb_required, TooManyKeywordParameters => exit_too_many_keyword_parameters, PatchPoint(Invariant::BOPRedefined { .. }) => exit_patchpoint_bop_redefined, @@ -593,9 +600,9 @@ pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, SendWithoutBlockDirectKeywordMismatch => send_fallback_send_without_block_direct_keyword_mismatch, - SendWithoutBlockDirectOptionalKeywords => send_fallback_send_without_block_direct_optional_keywords, SendWithoutBlockDirectKeywordCountMismatch=> send_fallback_send_without_block_direct_keyword_count_mismatch, SendWithoutBlockDirectMissingKeyword => send_fallback_send_without_block_direct_missing_keyword, + SendWithoutBlockDirectTooManyKeywords => send_fallback_send_without_block_direct_too_many_keywords, SendPolymorphic => send_fallback_send_polymorphic, SendMegamorphic => send_fallback_send_megamorphic, SendNoProfiles => send_fallback_send_no_profiles, |