diff options
685 files changed, 19795 insertions, 12601 deletions
diff --git a/.github/actions/setup/directories/action.yml b/.github/actions/setup/directories/action.yml index a59f575c99..5264e0e969 100644 --- a/.github/actions/setup/directories/action.yml +++ b/.github/actions/setup/directories/action.yml @@ -88,7 +88,7 @@ runs: git config --global init.defaultBranch garbage - if: inputs.checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: path: ${{ inputs.srcdir }} fetch-depth: ${{ inputs.fetch-depth }} diff --git a/.github/actions/setup/macos/action.yml b/.github/actions/setup/macos/action.yml index 896930de84..b96e959aa6 100644 --- a/.github/actions/setup/macos/action.yml +++ b/.github/actions/setup/macos/action.yml @@ -18,7 +18,11 @@ runs: - name: Set ENV shell: bash run: | - for lib in openssl@1.1 gmp; do + for lib in gmp; do + ruby_configure_args="${ruby_configure_args:+$ruby_configure_args }--with-${lib%@*}-dir=$(brew --prefix $lib)" + done + for lib in openssl@1.1; do CONFIGURE_ARGS="${CONFIGURE_ARGS:+$CONFIGURE_ARGS }--with-${lib%@*}-dir=$(brew --prefix $lib)" done + echo ruby_configure_args="${ruby_configure_args}" >> $GITHUB_ENV echo CONFIGURE_ARGS="${CONFIGURE_ARGS}" >> $GITHUB_ENV diff --git a/.github/actions/slack/action.yml b/.github/actions/slack/action.yml index c98be085a8..f0481f5bc2 100644 --- a/.github/actions/slack/action.yml +++ b/.github/actions/slack/action.yml @@ -24,7 +24,7 @@ runs: using: composite steps: - - uses: ruby/action-slack@0bd85c72233cdbb6a0fe01d37aaeff1d21b5fce1 # v3.2.1 + - uses: ruby/action-slack@54175162371f1f7c8eb94d7c8644ee2479fcd375 # v3.2.2 with: payload: | { diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml index 35e74d64d8..9b0cefdbdd 100644 --- a/.github/workflows/annocheck.yml +++ b/.github/workflows/annocheck.yml @@ -4,7 +4,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +12,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -63,7 +63,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -74,7 +74,7 @@ jobs: builddir: build makeup: true - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml index 12f98d4a1e..e7a245e1dc 100644 --- a/.github/workflows/baseruby.yml +++ b/.github/workflows/baseruby.yml @@ -4,7 +4,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +12,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -51,12 +51,12 @@ jobs: - ruby-3.3 steps: - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: ${{ matrix.ruby }} bundler: none - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: ./.github/actions/setup/ubuntu diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml index fcc005239b..6f06451a99 100644 --- a/.github/workflows/bundled_gems.yml +++ b/.github/workflows/bundled_gems.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml index fd7e2f58e5..9fb52444de 100644 --- a/.github/workflows/check_dependencies.yml +++ b/.github/workflows/check_dependencies.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -45,7 +45,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: ./.github/actions/setup/ubuntu if: ${{ contains(matrix.os, 'ubuntu') }} @@ -55,7 +55,7 @@ jobs: - uses: ./.github/actions/setup/directories - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/check_misc.yml b/.github/workflows/check_misc.yml index 48434a47a9..f26319f448 100644 --- a/.github/workflows/check_misc.yml +++ b/.github/workflows/check_misc.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c7d0def7ae..f733234906 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -5,7 +5,7 @@ on: branches: ['master'] paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -13,7 +13,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Install libraries if: ${{ contains(matrix.os, 'macos') }} @@ -80,15 +80,15 @@ jobs: run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - name: Initialize CodeQL - uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/autobuild@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 with: category: '/language:${{ matrix.language }}' upload: False @@ -118,7 +118,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 5e5eb69b6f..85769bd879 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -1,10 +1,11 @@ +# Some tests depending on this name 'Compilations' via $GITHUB_WORKFLOW. Make sure to update such tests when renaming this workflow. name: Compilations on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +13,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -136,7 +137,7 @@ jobs: - { name: '-O0', env: { optflags: '-O0 -march=x86-64 -mtune=generic' } } # - { name: '-O3', env: { optflags: '-O3 -march=x86-64 -mtune=generic' }, check: true } - - { name: gmp, env: { append_configure: '--with-gmp' } } + - { name: gmp, env: { append_configure: '--with-gmp' }, check: true } - { name: jemalloc, env: { append_configure: '--with-jemalloc' } } - { name: valgrind, env: { append_configure: '--with-valgrind' } } - { name: 'coroutine=ucontext', env: { append_configure: '--with-coroutine=ucontext' } } @@ -233,7 +234,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 54e30129e4..b565833e74 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -52,7 +52,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -70,8 +70,14 @@ jobs: # Set fetch-depth: 0 so that Launchable can receive commits information. fetch-depth: 10 + - name: make sure that kern.coredump=1 + run: | + sysctl -n kern.coredump + sudo sysctl -w kern.coredump=1 + sudo chmod -R +rwx /cores/ + - name: Run configure - run: ../src/configure -C --disable-install-doc + run: ../src/configure -C --disable-install-doc ${ruby_configure_args} - run: make prepare-gems if: ${{ matrix.test_task == 'test-bundled-gems' }} @@ -101,6 +107,7 @@ jobs: - name: make ${{ matrix.test_task }} run: | + ulimit -c unlimited make -s ${{ matrix.test_task }} ${TESTS:+TESTS="$TESTS"} timeout-minutes: 60 env: @@ -124,6 +131,22 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot if: ${{ failure() }} + - name: Resolve job ID + id: job_id + uses: actions/github-script@main + env: + matrix: ${{ toJson(matrix) }} + with: + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `${context.job}${matrix ? ` (${Object.values(matrix).join(", ")})` : ""}`; + return workflow_run.jobs.find((job) => job.name === job_name).id; + result: if: ${{ always() }} name: ${{ github.workflow }} result diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index ad39341059..8f38fd1e7b 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -66,7 +66,7 @@ jobs: steps: - name: Set up Ruby & MSYS2 - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: ${{ matrix.baseruby }} @@ -97,7 +97,7 @@ jobs: $result working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 600bc8fc55..56d7298193 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -55,7 +55,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -92,12 +92,12 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="error_highlight/test_error_highlight.rb"' RUN_OPTS: ${{ matrix.run_opts }} - - name: make test-prism-spec + - name: make test-spec run: | - $SETARCH make -s test-prism-spec SPECOPTS="$SPECOPTS" + $SETARCH make -s test-spec SPECOPTS="$SPECOPTS" timeout-minutes: 10 env: GNUMAKEFLAGS: '' diff --git a/.github/workflows/rjit-bindgen.yml b/.github/workflows/rjit-bindgen.yml index 1657721a65..97c99ea829 100644 --- a/.github/workflows/rjit-bindgen.yml +++ b/.github/workflows/rjit-bindgen.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -47,11 +47,11 @@ jobs: steps: - name: Set up Ruby - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.1' - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index 7a5e6cf7c0..21e697315b 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -55,7 +55,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -83,7 +83,7 @@ jobs: timeout-minutes: 30 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' + RUBY_TESTOPTS: '--tty=no' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-all diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 00a0638e96..51423a84f9 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -32,12 +32,12 @@ jobs: steps: - name: 'Checkout code' - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: persist-credentials: false - name: 'Run analysis' - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3 with: results_file: results.sarif results_format: sarif @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: 'Upload to code-scanning' - uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v2.1.27 + uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v2.1.27 with: sarif_file: results.sarif diff --git a/.github/workflows/spec_guards.yml b/.github/workflows/spec_guards.yml index bb48435c7f..4fde74ec46 100644 --- a/.github/workflows/spec_guards.yml +++ b/.github/workflows/spec_guards.yml @@ -45,9 +45,9 @@ jobs: - ruby-3.3 steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: ${{ matrix.ruby }} bundler: none diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index bd1a7056bb..cf2b1f73ab 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -58,7 +58,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -67,7 +67,7 @@ jobs: with: arch: ${{ matrix.arch }} - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 0bd96bf9df..e9ce787f12 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -61,7 +61,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -100,7 +100,7 @@ jobs: run: | echo "WASI_SDK_PATH=/opt/wasi-sdk" >> $GITHUB_ENV - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d5b486355a..0e778a48db 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -91,7 +91,7 @@ jobs: ${{ steps.find-tools.outputs.needs }} if: ${{ steps.find-tools.outputs.needs != '' }} - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 with: ruby-version: '3.0' bundler: none @@ -115,7 +115,7 @@ jobs: - name: Install libraries with vcpkg run: | - vcpkg --triplet x64-windows install libffi libyaml openssl readline zlib + vcpkg --triplet x64-windows install gmp libffi libyaml openssl zlib - name: Install libraries with scoop run: | @@ -123,7 +123,7 @@ jobs: Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH shell: pwsh - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml index 45a6a0578f..9660572e99 100644 --- a/.github/workflows/yjit-macos.yml +++ b/.github/workflows/yjit-macos.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -37,7 +37,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - run: RUST_BACKTRACE=1 cargo test working-directory: yjit @@ -81,7 +81,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -135,6 +135,7 @@ jobs: env: RUBY_TESTOPTS: '-q --tty=no' TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' + SYNTAX_SUGGEST_TIMEOUT: '5' PRECHECK_BUNDLED_GEMS: 'no' continue-on-error: ${{ matrix.continue-on-test_task || false }} diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 146027bcf1..7f6f3393ef 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -36,7 +36,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 # For now we can't run cargo test --offline because it complains about the # capstone dependency, even though the dependency is optional @@ -68,7 +68,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 # Check that we don't have linting errors in release mode, too - run: cargo clippy --all-targets --all-features @@ -127,13 +127,18 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github - uses: ./.github/actions/setup/ubuntu + - uses: ruby/setup-ruby@78c01b705fd9d5ad960d432d3a0cfa341d50e410 # v1.179.1 + with: + ruby-version: '3.0' + bundler: none + - uses: ./.github/actions/setup/directories with: srcdir: src @@ -147,11 +152,6 @@ jobs: if: ${{ matrix.rust_version }} run: rustup install ${{ matrix.rust_version }} --profile minimal - - uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 - with: - ruby-version: '3.0' - bundler: none - - name: Run configure run: ../src/configure -C --disable-install-doc --prefix=$(pwd)/install ${{ matrix.configure }} diff --git a/.rdoc_options b/.rdoc_options index 57e03c9dd7..4dfbfa140c 100644 --- a/.rdoc_options +++ b/.rdoc_options @@ -5,3 +5,5 @@ encoding: UTF-8 main_page: README.md title: Documentation for Ruby development version visibility: :private +rdoc_include: +- doc @@ -727,24 +727,6 @@ mentioned below. for internal or external distribution as long as this notice remains attached. -[ext/nkf/nkf-utf8/config.h] -[ext/nkf/nkf-utf8/nkf.c] -[ext/nkf/nkf-utf8/utf8tbl.c] - - These files are under the following license. So to speak, it is - copyrighted semi-public-domain software. - - >>> - Copyright (C) 1987:: Fujitsu LTD. (Itaru ICHIKAWA) - - Everyone is permitted to do anything on this program - including copying, modifying, improving, - as long as you don't try to pretend that you wrote it. - i.e., the above copyright notice has to appear in all copies. - Binary distribution requires original version messages. - You don't have to ask before copying, redistribution or publishing. - THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE. - [ext/psych] [test/psych] @@ -7,8 +7,8 @@ Note that each entry is kept to a minimum, see links for details. ## Language changes -* String literals in files without a `frozen_string_literal` comment now behave - as if they were frozen. If they are mutated a deprecation warning is emitted. +* String literals in files without a `frozen_string_literal` comment now emit a deprecation warning + when they are mutated. These warnings can be enabled with `-W:deprecated` or by setting `Warning[:deprecated] = true`. To disable this change, you can run Ruby with the `--disable-frozen-string-literal` command line argument. [[Feature #20205]] @@ -19,9 +19,11 @@ Note that each entry is kept to a minimum, see links for details. `**nil` is treated similarly to `**{}`, passing no keywords, and not calling any conversion methods. [[Bug #20064]] -* Block passing is no longer allowed in index. [[Bug #19918]] +* Block passing is no longer allowed in index assignment + (e.g. `a[0, &b] = 1`). [[Bug #19918]] -* Keyword arguments are no longer allowed in index. [[Bug #20218]] +* Keyword arguments are no longer allowed in index assignment + (e.g. `a[0, kw: 1] = 2`). [[Bug #20218]] ## Core classes updates @@ -38,6 +40,13 @@ Note: We're only listing outstanding class updates. ## Stdlib updates +* Tempfile + + * The keyword argument `anonymous: true` is implemented for `Tempfile.create`. + `Tempfile.create(anonymous: true)` removes the created temporary file immediately. + So applications don't need to remove the file. + [[Feature #20497]] + The following default gems are updated. * RubyGems 3.6.0.dev @@ -49,31 +58,33 @@ The following default gems are updated. * json 2.7.2 * net-http 0.4.1 * optparse 0.5.0 -* prism 0.27.0 -* rdoc 6.6.3.1 -* reline 0.5.5 +* prism 0.30.0 +* rdoc 6.7.0 +* reline 0.5.8 * resolv 0.4.0 * stringio 3.1.1 * strscan 3.1.1 The following bundled gems are updated. -* minitest 5.22.3 +* minitest 5.23.1 * rake 13.2.1 * test-unit 3.6.2 -* net-ftp 0.3.4 -* net-imap 0.4.10 +* rexml 3.2.8 +* net-ftp 0.3.5 +* net-imap 0.4.12 * net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 * debug 1.9.2 +* racc 1.8.0 The following bundled gems are promoted from default gems. * mutex_m 0.2.0 * getoptlong 0.2.1 * base64 0.2.0 -* bigdecimal 3.1.7 +* bigdecimal 3.1.8 * observer 0.1.2 * abbrev 0.1.2 * resolv-replace 0.1.1 @@ -122,7 +133,7 @@ See GitHub releases like [GitHub Releases of Logger](https://github.com/ruby/log a warning on verbose mode (`-w`). [[Feature #15554]] -* Redefining some core methods that are specially optimized by the interpeter +* Redefining some core methods that are specially optimized by the interpreter and JIT like `String.freeze` or `Integer#+` now emits a performance class warning (`-W:performance` or `Warning[:performance] = true`). [[Feature #20429]] diff --git a/README.ja.md b/README.ja.md index 0d2d309fb8..49cf72b5fd 100644 --- a/README.ja.md +++ b/README.ja.md @@ -4,7 +4,6 @@ [](https://github.com/ruby/ruby/actions?query=workflow%3A"Windows") [](https://ci.appveyor.com/project/ruby/ruby/branch/master) [](https://app.travis-ci.com/ruby/ruby) -[](https://cirrus-ci.com/github/ruby/ruby/master) # Rubyとは @@ -880,6 +880,19 @@ rb_ary_free(VALUE ary) } } +VALUE +rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len, bool freeze) +{ + fake_ary->basic.flags = T_ARRAY; + VALUE ary = (VALUE)fake_ary; + RBASIC_CLEAR_CLASS(ary); + ARY_SET_PTR(ary, list); + ARY_SET_HEAP_LEN(ary, len); + ARY_SET_CAPA(ary, len); + if (freeze) OBJ_FREEZE(ary); + return ary; +} + size_t rb_ary_memsize(VALUE ary) { @@ -6536,6 +6549,14 @@ rb_ary_shuffle(rb_execution_context_t *ec, VALUE ary, VALUE randgen) return ary; } +static const rb_data_type_t ary_sample_memo_type = { + .wrap_struct_name = "ary_sample_memo", + .function = { + .dfree = (RUBY_DATA_FUNC)st_free_table, + }, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY +}; + static VALUE ary_sample(rb_execution_context_t *ec, VALUE ary, VALUE randgen, VALUE nv, VALUE to_array) { @@ -6617,11 +6638,9 @@ ary_sample(rb_execution_context_t *ec, VALUE ary, VALUE randgen, VALUE nv, VALUE } else if (n <= memo_threshold / 2) { long max_idx = 0; -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - VALUE vmemo = Data_Wrap_Struct(0, 0, st_free_table, 0); + VALUE vmemo = TypedData_Wrap_Struct(0, &ary_sample_memo_type, 0); st_table *memo = st_init_numtable_with_size(n); - DATA_PTR(vmemo) = memo; + RTYPEDDATA_DATA(vmemo) = memo; result = rb_ary_new_capa(n); RARRAY_PTR_USE(result, ptr_result, { for (i=0; i<n; i++) { @@ -6644,7 +6663,7 @@ ary_sample(rb_execution_context_t *ec, VALUE ary, VALUE randgen, VALUE nv, VALUE } }); }); - DATA_PTR(vmemo) = 0; + RTYPEDDATA_DATA(vmemo) = 0; st_free_table(memo); RB_GC_GUARD(vmemo); } diff --git a/benchmark/hash_aref_str_lit.yml b/benchmark/hash_aref_str_lit.yml new file mode 100644 index 0000000000..ed8142bcf1 --- /dev/null +++ b/benchmark/hash_aref_str_lit.yml @@ -0,0 +1,20 @@ +prelude: | + # frozen_string_literal: true + hash = 10.times.to_h do |i| + [i, i] + end + dyn_sym = "dynamic_symbol".to_sym + binary = RubyVM::InstructionSequence.compile("# frozen_string_literal: true\n'iseq_load'").to_binary + iseq_literal_string = RubyVM::InstructionSequence.load_from_binary(binary).eval + + hash[:some_symbol] = 1 + hash[dyn_sym] = 2 + hash["small"] = 3 + hash["frozen_string_literal"] = 4 + hash[iseq_literal_string] = 5 +benchmark: + symbol: hash[:some_symbol] + dyn_symbol: hash[dyn_sym] + small_lit: hash["small"] + frozen_lit: hash["frozen_string_literal"] + iseq_lit: hash[iseq_literal_string] @@ -30,9 +30,6 @@ # define USE_GMP 0 #endif #endif -#if USE_GMP -# include <gmp.h> -#endif #include "id.h" #include "internal.h" @@ -48,6 +45,15 @@ #include "ruby/util.h" #include "ruby_assert.h" +#if USE_GMP +RBIMPL_WARNING_PUSH() +# ifdef _MSC_VER +RBIMPL_WARNING_IGNORED(4146) /* for mpn_neg() */ +# endif +# include <gmp.h> +RBIMPL_WARNING_POP() +#endif + static const bool debug_integer_pack = ( #ifdef DEBUG_INTEGER_PACK DEBUG_INTEGER_PACK+0 diff --git a/bootstraptest/runner.rb b/bootstraptest/runner.rb index 120b78246c..3e54318ac9 100755 --- a/bootstraptest/runner.rb +++ b/bootstraptest/runner.rb @@ -163,6 +163,10 @@ def main BT.quiet = false BT.timeout = 180 BT.timeout_scale = (defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? ? 3 : 1) # for --jit-wait + if (ts = (ENV["RUBY_TEST_TIMEOUT_SCALE"] || ENV["RUBY_TEST_SUBPROCESS_TIMEOUT_SCALE"]).to_i) > 1 + BT.timeout_scale *= ts + end + # BT.wn = 1 dir = nil quiet = false @@ -234,7 +238,7 @@ End end tests ||= ARGV tests = Dir.glob("#{File.dirname($0)}/test_*.rb").sort if tests.empty? - pathes = tests.map {|path| File.expand_path(path) } + paths = tests.map {|path| File.expand_path(path) } BT.progress = %w[- \\ | /] BT.progress_bs = "\b" * BT.progress[0].size @@ -278,7 +282,7 @@ End end in_temporary_working_directory(dir) do - exec_test pathes + exec_test paths end end @@ -290,8 +294,8 @@ def erase(e = true) end end -def load_test pathes - pathes.each do |path| +def load_test paths + paths.each do |path| load File.expand_path(path) end end @@ -341,13 +345,13 @@ def concurrent_exec_test end end -def exec_test(pathes) +def exec_test(paths) # setup - load_test pathes + load_test paths BT_STATE.count = 0 BT_STATE.error = 0 BT.columns = 0 - BT.width = pathes.map {|path| File.basename(path).size}.max + 2 + BT.width = paths.map {|path| File.basename(path).size}.max + 2 # execute tests if BT.wn > 1 diff --git a/bootstraptest/test_syntax.rb b/bootstraptest/test_syntax.rb index fbc9c6f62e..8301b344c6 100644 --- a/bootstraptest/test_syntax.rb +++ b/bootstraptest/test_syntax.rb @@ -848,7 +848,7 @@ assert_normal_exit %q{ def x(a=1, b, *rest); nil end end end -}, bug2415 +}, bug2415 unless rjit_enabled? # flaky assert_normal_exit %q{ 0.times do @@ -880,7 +880,7 @@ assert_normal_exit %q{ end end end -}, bug2415 +}, bug2415 unless rjit_enabled? # flaky assert_normal_exit %q{ a { diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 8b8f7d1e39..f3e935d99e 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4813,6 +4813,15 @@ assert_equal [0x80000000000, 'a+', :ok].inspect, %q{ tests } +# test integer left shift fusion followed by opt_getconstant_path +assert_equal '33', %q{ + def test(a) + (a << 5) | (Object; a) + end + + test(1) +} + # test String#stebyte with arguments that need conversion assert_equal "abc", %q{ str = +"a00" @@ -4980,3 +4989,70 @@ assert_equal '[[:c_call, :x=], [:c_call, :x]]', %q{ events } + +# regression test for splatting empty array +assert_equal '1', %q{ + def callee(foo) = foo + + def test_body(args) = callee(1, *args) + + test_body([]) + array = Array.new(100) + array.clear + test_body(array) +} + +# regression test for splatting empty array to cfunc +assert_normal_exit %q{ + def test_body(args) = Array(1, *args) + + test_body([]) + 0x100.times do + array = Array.new(100) + array.clear + test_body(array) + end +} + +# compiling code shouldn't emit warnings as it may call into more Ruby code +assert_equal 'ok', <<~'RUBY' + # [Bug #20522] + $VERBOSE = true + Warning[:performance] = true + + module StrictWarnings + def warn(msg, **) + raise msg + end + end + Warning.singleton_class.prepend(StrictWarnings) + + class A + def compiled_method(is_private) + @some_ivar = is_private + end + end + + shape_max_variations = 8 + if defined?(RubyVM::Shape::SHAPE_MAX_VARIATIONS) && RubyVM::Shape::SHAPE_MAX_VARIATIONS != shape_max_variations + raise "Expected SHAPE_MAX_VARIATIONS to be #{shape_max_variations}, got: #{RubyVM::Shape::SHAPE_MAX_VARIATIONS}" + end + + 100.times do |i| + klass = Class.new(A) + (shape_max_variations - 1).times do |j| + obj = klass.new + obj.instance_variable_set("@base_#{i}", 42) + obj.instance_variable_set("@ivar_#{j}", 42) + end + obj = klass.new + obj.instance_variable_set("@base_#{i}", 42) + begin + obj.compiled_method(true) + rescue + # expected + end + end + + :ok +RUBY @@ -106,7 +106,6 @@ PRISM_FILES = prism/api_node.$(OBJEXT) \ prism/util/pm_memchr.$(OBJEXT) \ prism/util/pm_newline_list.$(OBJEXT) \ prism/util/pm_string.$(OBJEXT) \ - prism/util/pm_string_list.$(OBJEXT) \ prism/util/pm_strncasecmp.$(OBJEXT) \ prism/util/pm_strpbrk.$(OBJEXT) \ prism/prism.$(OBJEXT) \ @@ -1018,15 +1017,6 @@ yes-test-spec: yes-test-spec-precheck $(ACTIONS_ENDGROUP) no-test-spec: -test-prism-spec: $(TEST_RUNNABLE)-test-prism-spec -yes-test-prism-spec: yes-test-spec-precheck - $(ACTIONS_GROUP) - $(gnumake_recursive)$(Q) \ - $(RUNRUBY) -r./$(arch)-fake -r$(tooldir)/lib/_tmpdir \ - $(srcdir)/spec/mspec/bin/mspec run -B $(srcdir)/spec/default.mspec -B $(srcdir)/spec/prism.mspec $(MSPECOPT) $(SPECOPTS) - $(ACTIONS_ENDGROUP) -no-test-prism-spec: - check: $(DOT_WAIT) test-spec RUNNABLE = $(LIBRUBY_RELATIVE:no=un)-runnable @@ -1600,7 +1590,7 @@ yes-test-bundled-gems-prepare: yes-test-bundled-gems-precheck $(ACTIONS_ENDGROUP) PREPARE_BUNDLED_GEMS = test-bundled-gems-prepare -test-bundled-gems: $(TEST_RUNNABLE)-test-bundled-gems $(TEST_RUNNABLE)-test-bundled-gems-spec +test-bundled-gems: $(TEST_RUNNABLE)-test-bundled-gems $(DOT_WAIT) $(TEST_RUNNABLE)-test-bundled-gems-spec yes-test-bundled-gems: test-bundled-gems-run no-test-bundled-gems: @@ -1608,8 +1598,10 @@ no-test-bundled-gems: # TEST_BUNDLED_GEMS_ALLOW_FAILURES = BUNDLED_GEMS = -test-bundled-gems-run: $(PREPARE_BUNDLED_GEMS) +test-bundled-gems-run: $(TEST_RUNNABLE)-test-bundled-gems-run +yes-test-bundled-gems-run: $(PREPARE_BUNDLED_GEMS) $(gnumake_recursive)$(Q) $(XRUBY) $(tooldir)/test-bundled-gems.rb $(BUNDLED_GEMS) +no-test-bundled-gems-run: $(PREPARE_BUNDLED_GEMS) test-bundled-gems-spec: $(TEST_RUNNABLE)-test-bundled-gems-spec yes-test-bundled-gems-spec: yes-test-spec-precheck $(PREPARE_BUNDLED_GEMS) @@ -1620,30 +1612,8 @@ yes-test-bundled-gems-spec: yes-test-spec-precheck $(PREPARE_BUNDLED_GEMS) $(ACTIONS_ENDGROUP) no-test-bundled-gems-spec: -test-syntax-suggest-precheck: $(TEST_RUNNABLE)-test-syntax-suggest-precheck -no-test-syntax-suggest-precheck: -yes-test-syntax-suggest-precheck: main -test-syntax-suggest-prepare: $(TEST_RUNNABLE)-test-syntax-suggest-prepare -no-test-syntax-suggest-prepare: no-test-syntax-suggest-precheck -yes-test-syntax-suggest-prepare: yes-test-syntax-suggest-precheck - $(ACTIONS_GROUP) - $(XRUBY) -C "$(srcdir)" bin/gem install --no-document \ - --install-dir .bundle --conservative "rspec:~> 3" - $(ACTIONS_ENDGROUP) - -RSPECOPTS = -SYNTAX_SUGGEST_SPECS = -PREPARE_SYNTAX_SUGGEST = $(TEST_RUNNABLE)-test-syntax-suggest-prepare -test-syntax-suggest: $(TEST_RUNNABLE)-test-syntax-suggest -yes-test-syntax-suggest: $(PREPARE_SYNTAX_SUGGEST) - $(ACTIONS_GROUP) - $(XRUBY) -C $(srcdir) -Ispec/syntax_suggest:spec/lib .bundle/bin/rspec \ - --require rspec/expectations \ - --require spec_helper --require formatter_overrides --require spec_coverage \ - $(RSPECOPTS) spec/syntax_suggest/$(SYNTAX_SUGGEST_SPECS) - $(ACTIONS_ENDGROUP) -no-test-syntax-suggest: +test-syntax-suggest: check: $(DOT_WAIT) $(PREPARE_SYNTAX_SUGGEST) test-syntax-suggest @@ -1916,6 +1886,22 @@ ChangeLog: -e 'VCS.detect(ARGV[0]).export_changelog(path: ARGV[1])' \ "$(srcdir)" $@ +# CAUTION: If using GNU make 3 which does not support `.WAIT`, this +# recipe with multiple jobs makes build and `git reset` run +# simultaneously, and will cause inconsistent results. Run with `-j1` +# or update GNU make. +nightly: yesterday $(DOT_WAIT) install + $(NULLCMD) + +# Rewind to the last commit "yesterday". "Yesterday" means here the +# period where `RUBY_RELEASE_DATE` is the day before the date to be +# generated now. In short, the yesterday in JST-9 time zone. +yesterday: rewindable + +rewindable: + $(GIT) -C $(srcdir) status --porcelain + $(GIT) -C $(srcdir) diff --quiet + HELP_EXTRA_TASKS = "" help: PHONY @@ -2282,7 +2268,6 @@ ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h ast.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h ast.$(OBJEXT): {$(VPATH)}assert.h @@ -2720,7 +2705,6 @@ builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h builtin.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h builtin.$(OBJEXT): {$(VPATH)}assert.h @@ -3358,7 +3342,6 @@ compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h compile.$(OBJEXT): $(top_srcdir)/prism_compile.c @@ -3824,7 +3807,6 @@ cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h cont.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h cont.$(OBJEXT): {$(VPATH)}$(COROUTINE_H) @@ -6809,7 +6791,6 @@ eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h eval.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h eval.$(OBJEXT): {$(VPATH)}assert.h @@ -7291,7 +7272,6 @@ gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h gc.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h gc.$(OBJEXT): {$(VPATH)}assert.h @@ -7549,7 +7529,6 @@ goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h goruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h goruby.$(OBJEXT): {$(VPATH)}assert.h @@ -7793,7 +7772,6 @@ hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h hash.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h hash.$(OBJEXT): {$(VPATH)}assert.h @@ -8836,7 +8814,6 @@ iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h iseq.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h iseq.$(OBJEXT): {$(VPATH)}assert.h @@ -9093,7 +9070,6 @@ load.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h load.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h load.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h load.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -load.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h load.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h load.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h load.$(OBJEXT): {$(VPATH)}assert.h @@ -10431,7 +10407,6 @@ miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h miniinit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h miniinit.$(OBJEXT): {$(VPATH)}array.rb @@ -12026,7 +12001,6 @@ prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/api_node.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/api_node.$(OBJEXT): {$(VPATH)}assert.h @@ -12222,7 +12196,6 @@ prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/api_pack.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/api_pack.$(OBJEXT): {$(VPATH)}assert.h @@ -12432,7 +12405,6 @@ prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/extension.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/extension.$(OBJEXT): {$(VPATH)}assert.h @@ -12621,7 +12593,6 @@ prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_integer.h prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/node.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/node.$(OBJEXT): {$(VPATH)}prism/ast.h @@ -12669,7 +12640,6 @@ prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/prism.$(OBJEXT): $(top_srcdir)/prism/version.h @@ -12691,7 +12661,6 @@ prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/regexp.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/regexp.$(OBJEXT): {$(VPATH)}prism/ast.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/defines.h @@ -12712,7 +12681,6 @@ prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism/serialize.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism/serialize.$(OBJEXT): {$(VPATH)}prism/ast.h @@ -12782,10 +12750,6 @@ prism/util/pm_newline_list.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism/util/pm_string.$(OBJEXT): $(top_srcdir)/prism/defines.h prism/util/pm_string.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.c prism/util/pm_string.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/util/pm_string_list.$(OBJEXT): $(top_srcdir)/prism/defines.h -prism/util/pm_string_list.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism/util/pm_string_list.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.c -prism/util/pm_string_list.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism/util/pm_strncasecmp.$(OBJEXT): $(top_srcdir)/prism/defines.h prism/util/pm_strncasecmp.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.c prism/util/pm_strncasecmp.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h @@ -12827,7 +12791,6 @@ prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h prism_init.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h prism_init.$(OBJEXT): $(top_srcdir)/prism_init.c @@ -13044,7 +13007,6 @@ proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h proc.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h proc.$(OBJEXT): {$(VPATH)}assert.h @@ -15540,7 +15502,6 @@ rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h rjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h rjit.$(OBJEXT): {$(VPATH)}assert.h @@ -15794,7 +15755,6 @@ rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h rjit_c.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h rjit_c.$(OBJEXT): {$(VPATH)}assert.h @@ -16074,7 +16034,6 @@ ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h ruby.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h ruby.$(OBJEXT): {$(VPATH)}assert.h @@ -18528,7 +18487,6 @@ thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h thread.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h thread.$(OBJEXT): {$(VPATH)}$(COROUTINE_H) @@ -19140,8 +19098,11 @@ transcode.$(OBJEXT): {$(VPATH)}subst.h transcode.$(OBJEXT): {$(VPATH)}transcode.c transcode.$(OBJEXT): {$(VPATH)}transcode_data.h util.$(OBJEXT): $(hdrdir)/ruby/ruby.h +util.$(OBJEXT): $(top_srcdir)/internal/array.h util.$(OBJEXT): $(top_srcdir)/internal/compilers.h +util.$(OBJEXT): $(top_srcdir)/internal/imemo.h util.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h +util.$(OBJEXT): $(top_srcdir)/internal/static_assert.h util.$(OBJEXT): $(top_srcdir)/internal/util.h util.$(OBJEXT): $(top_srcdir)/internal/warnings.h util.$(OBJEXT): {$(VPATH)}assert.h @@ -19795,7 +19756,6 @@ vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h vm.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h vm.$(OBJEXT): {$(VPATH)}assert.h @@ -20054,7 +20014,6 @@ vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h vm_backtrace.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h vm_backtrace.$(OBJEXT): {$(VPATH)}assert.h @@ -20285,7 +20244,6 @@ vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h vm_dump.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h vm_dump.$(OBJEXT): {$(VPATH)}addr2line.h @@ -20728,7 +20686,6 @@ vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h vm_trace.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h vm_trace.$(OBJEXT): {$(VPATH)}assert.h @@ -21171,7 +21128,6 @@ yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_list.h yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_memchr.h yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_newline_list.h yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string.h -yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_string_list.h yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h yjit.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h yjit.$(OBJEXT): {$(VPATH)}assert.h @@ -48,9 +48,6 @@ #include "insns.inc" #include "insns_info.inc" -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - #define FIXNUM_INC(n, i) ((n)+(INT2FIX(i)&~FIXNUM_FLAG)) #define FIXNUM_OR(n, i) ((n)|INT2FIX(i)) @@ -3080,7 +3077,7 @@ iseq_pop_newarray(rb_iseq_t *iseq, INSN *iobj) static int is_frozen_putstring(INSN *insn, VALUE *op) { - if (IS_INSN_ID(insn, putstring)) { + if (IS_INSN_ID(insn, putstring) || IS_INSN_ID(insn, putchilledstring)) { *op = OPERAND_AT(insn, 0); return 1; } @@ -3122,6 +3119,7 @@ optimize_checktype(rb_iseq_t *iseq, INSN *iobj) switch (INSN_OF(iobj)) { case BIN(putstring): + case BIN(putchilledstring): type = INT2FIX(T_STRING); break; case BIN(putnil): @@ -3549,6 +3547,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal enum ruby_vminsn_type previ = ((INSN *)prev)->insn_id; if (previ == BIN(putobject) || previ == BIN(putnil) || previ == BIN(putself) || previ == BIN(putstring) || + previ == BIN(putchilledstring) || previ == BIN(dup) || previ == BIN(getlocal) || previ == BIN(getblockparam) || @@ -3690,7 +3689,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } } - if (IS_INSN_ID(iobj, putstring) || + if (IS_INSN_ID(iobj, putstring) || IS_INSN_ID(iobj, putchilledstring) || (IS_INSN_ID(iobj, putobject) && RB_TYPE_P(OPERAND_AT(iobj, 0), T_STRING))) { /* * putstring "" @@ -4048,6 +4047,8 @@ insn_set_specialized_instruction(rb_iseq_t *iseq, INSN *iobj, int insn_id) return COMPILE_OK; } +#define vm_ci_simple(ci) (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) + static int iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) { @@ -4059,24 +4060,43 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) INSN *niobj = (INSN *)iobj->link.next; if (IS_INSN_ID(niobj, send)) { const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(niobj, 0); - if ((vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) && vm_ci_argc(ci) == 0) { + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0) { switch (vm_ci_mid(ci)) { case idMax: case idMin: case idHash: { VALUE num = iobj->operands[0]; + int operand_len = insn_len(BIN(opt_newarray_send)) - 1; iobj->insn_id = BIN(opt_newarray_send); - iobj->operands = compile_data_calloc2(iseq, insn_len(iobj->insn_id) - 1, sizeof(VALUE)); + iobj->operands = compile_data_calloc2(iseq, operand_len, sizeof(VALUE)); iobj->operands[0] = num; iobj->operands[1] = rb_id2sym(vm_ci_mid(ci)); - iobj->operand_size = insn_len(iobj->insn_id) - 1; + iobj->operand_size = operand_len; ELEM_REMOVE(&niobj->link); return COMPILE_OK; } } } } + else if ((IS_INSN_ID(niobj, putstring) || IS_INSN_ID(niobj, putchilledstring) || + (IS_INSN_ID(niobj, putobject) && RB_TYPE_P(OPERAND_AT(niobj, 0), T_STRING))) && + IS_NEXT_INSN_ID(&niobj->link, send)) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT((INSN *)niobj->link.next, 0); + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 1 && vm_ci_mid(ci) == idPack) { + VALUE num = iobj->operands[0]; + int operand_len = insn_len(BIN(opt_newarray_send)) - 1; + iobj->insn_id = BIN(opt_newarray_send); + iobj->operands = compile_data_calloc2(iseq, operand_len, sizeof(VALUE)); + iobj->operands[0] = FIXNUM_INC(num, 1); + iobj->operands[1] = rb_id2sym(vm_ci_mid(ci)); + iobj->operand_size = operand_len; + ELEM_REMOVE(&iobj->link); + ELEM_REMOVE(niobj->link.next); + ELEM_INSERT_NEXT(&niobj->link, &iobj->link); + return COMPILE_OK; + } + } } if (IS_INSN_ID(iobj, send)) { @@ -4084,7 +4104,7 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 1); #define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt)) - if (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) { + if (vm_ci_simple(ci)) { switch (vm_ci_argc(ci)) { case 0: switch (vm_ci_mid(ci)) { @@ -4406,11 +4426,12 @@ static int compile_dregx(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { int cnt; + int cflag = (int)RNODE_DREGX(node)->as.nd_cflag; if (!RNODE_DREGX(node)->nd_next) { if (!popped) { VALUE src = rb_node_dregx_string_val(node); - VALUE match = rb_reg_compile(src, (int)RNODE_DREGX(node)->nd_cflag, NULL, 0); + VALUE match = rb_reg_compile(src, cflag, NULL, 0); ADD_INSN1(ret, node, putobject, match); RB_OBJ_WRITTEN(iseq, Qundef, match); } @@ -4418,7 +4439,7 @@ compile_dregx(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i } CHECK(compile_dstr_fragments(iseq, ret, node, &cnt)); - ADD_INSN2(ret, node, toregexp, INT2FIX(RNODE_DREGX(node)->nd_cflag), INT2FIX(cnt)); + ADD_INSN2(ret, node, toregexp, INT2FIX(cflag), INT2FIX(cnt)); if (popped) { ADD_INSN(ret, node, pop); @@ -5380,12 +5401,17 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const CHECK(COMPILE_POPPED(pre, "masgn lhs (NODE_ATTRASGN)", node)); + bool safenav_call = false; LINK_ELEMENT *insn_element = LAST_ELEMENT(pre); iobj = (INSN *)get_prev_insn((INSN *)insn_element); /* send insn */ ASSUME(iobj); - ELEM_REMOVE(LAST_ELEMENT(pre)); - ELEM_REMOVE((LINK_ELEMENT *)iobj); - pre->last = iobj->link.prev; + ELEM_REMOVE(insn_element); + if (!IS_INSN_ID(iobj, send)) { + safenav_call = true; + iobj = (INSN *)get_prev_insn(iobj); + ELEM_INSERT_NEXT(&iobj->link, insn_element); + } + (pre->last = iobj->link.prev)->next = 0; const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(iobj, 0); int argc = vm_ci_argc(ci) + 1; @@ -5404,7 +5430,9 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const return COMPILE_NG; } - ADD_ELEM(lhs, (LINK_ELEMENT *)iobj); + iobj->link.prev = lhs->last; + lhs->last->next = &iobj->link; + for (lhs->last = &iobj->link; lhs->last->next; lhs->last = lhs->last->next); if (vm_ci_flag(ci) & VM_CALL_ARGS_SPLAT) { int argc = vm_ci_argc(ci); bool dupsplat = false; @@ -5437,9 +5465,11 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const } INSERT_BEFORE_INSN1(iobj, line_no, node_id, pushtoarray, INT2FIX(1)); } - ADD_INSN(lhs, line_node, pop); - if (argc != 1) { + if (!safenav_call) { ADD_INSN(lhs, line_node, pop); + if (argc != 1) { + ADD_INSN(lhs, line_node, pop); + } } for (int i=0; i < argc; i++) { ADD_INSN(post, line_node, pop); @@ -8311,7 +8341,7 @@ compile_resbody(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, static int compile_ensure(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { - const int line = nd_line(node); + const int line = nd_line(RNODE_ENSURE(node)->nd_ensr); const NODE *line_node = node; DECL_ANCHOR(ensr); const rb_iseq_t *ensure = NEW_CHILD_ISEQ(RNODE_ENSURE(node)->nd_ensr, @@ -11388,7 +11418,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, { /* TODO: body should be frozen */ long i, len = RARRAY_LEN(body); - struct st_table *labels_table = DATA_PTR(labels_wrapper); + struct st_table *labels_table = RTYPEDDATA_DATA(labels_wrapper); int j; int line_no = 0, node_id = -1, insn_idx = 0; int ret = COMPILE_OK; @@ -11566,7 +11596,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, rb_raise(rb_eTypeError, "unexpected object for instruction"); } } - DATA_PTR(labels_wrapper) = 0; + RTYPEDDATA_DATA(labels_wrapper) = 0; RB_GC_GUARD(labels_wrapper); validate_labels(iseq, labels_table); if (!ret) return ret; @@ -11699,6 +11729,15 @@ rb_iseq_mark_and_pin_insn_storage(struct iseq_compile_data_storage *storage) } } +static const rb_data_type_t labels_wrapper_type = { + .wrap_struct_name = "compiler/labels_wrapper", + .function = { + .dmark = (RUBY_DATA_FUNC)rb_mark_set, + .dfree = (RUBY_DATA_FUNC)st_free_table, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, +}; + void rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE misc, VALUE locals, VALUE params, VALUE exception, VALUE body) @@ -11708,7 +11747,7 @@ rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE misc, VALUE locals, VALUE params, unsigned int arg_size, local_size, stack_max; ID *tbl; struct st_table *labels_table = st_init_numtable(); - VALUE labels_wrapper = Data_Wrap_Struct(0, rb_mark_set, st_free_table, labels_table); + VALUE labels_wrapper = TypedData_Wrap_Struct(0, &labels_wrapper_type, labels_table); VALUE arg_opt_labels = rb_hash_aref(params, SYM(opt)); VALUE keywords = rb_hash_aref(params, SYM(keyword)); VALUE sym_arg_rest = ID2SYM(rb_intern_const("#arg_rest")); @@ -13514,7 +13553,7 @@ ibf_load_object_string(const struct ibf_load *load, const struct ibf_object_head VALUE str; if (header->frozen && !header->internal) { - str = rb_enc_interned_str(ptr, len, rb_enc_from_index(encindex)); + str = rb_enc_literal_str(ptr, len, rb_enc_from_index(encindex)); } else { str = rb_enc_str_new(ptr, len, rb_enc_from_index(encindex)); @@ -2470,7 +2470,7 @@ float_arg(VALUE self) * * The rectangular coordinates of a complex number * are called the _real_ and _imaginary_ parts; - * see {Complex number definition}[https://en.wikipedia.org/wiki/Complex_number#Definition]. + * see {Complex number definition}[https://en.wikipedia.org/wiki/Complex_number#Definition_and_basic_operations]. * * You can create a \Complex object from rectangular coordinates with: * @@ -2495,7 +2495,7 @@ float_arg(VALUE self) * * The polar coordinates of a complex number * are called the _absolute_ and _argument_ parts; - * see {Complex polar plane}[https://en.wikipedia.org/wiki/Complex_number#Polar_complex_plane]. + * see {Complex polar plane}[https://en.wikipedia.org/wiki/Complex_number#Polar_form]. * * In this class, the argument part * in expressed {radians}[https://en.wikipedia.org/wiki/Radian] diff --git a/configure.ac b/configure.ac index e9a452ebee..ac059bf862 100644 --- a/configure.ac +++ b/configure.ac @@ -9,44 +9,50 @@ tooldir="$srcdir/tool" AC_DISABLE_OPTION_CHECKING -m4_include([tool/m4/_colorize_result_prepare.m4])dnl -m4_include([tool/m4/ac_msg_result.m4])dnl -m4_include([tool/m4/colorize_result.m4])dnl -m4_include([tool/m4/ruby_append_option.m4])dnl -m4_include([tool/m4/ruby_append_options.m4])dnl -m4_include([tool/m4/ruby_check_builtin_func.m4])dnl -m4_include([tool/m4/ruby_check_builtin_setjmp.m4])dnl -m4_include([tool/m4/ruby_check_printf_prefix.m4])dnl -m4_include([tool/m4/ruby_check_setjmp.m4])dnl -m4_include([tool/m4/ruby_check_signedness.m4])dnl -m4_include([tool/m4/ruby_check_sizeof.m4])dnl -m4_include([tool/m4/ruby_check_sysconf.m4])dnl -m4_include([tool/m4/ruby_cppoutfile.m4])dnl -m4_include([tool/m4/ruby_decl_attribute.m4])dnl -m4_include([tool/m4/ruby_default_arch.m4])dnl -m4_include([tool/m4/ruby_define_if.m4])dnl -m4_include([tool/m4/ruby_defint.m4])dnl -m4_include([tool/m4/ruby_dtrace_available.m4])dnl -m4_include([tool/m4/ruby_dtrace_postprocess.m4])dnl -m4_include([tool/m4/ruby_func_attribute.m4])dnl -m4_include([tool/m4/ruby_mingw32.m4])dnl -m4_include([tool/m4/ruby_prepend_option.m4])dnl -m4_include([tool/m4/ruby_prog_gnu_ld.m4])dnl -m4_include([tool/m4/ruby_prog_makedirs.m4])dnl -m4_include([tool/m4/ruby_replace_funcs.m4])dnl -m4_include([tool/m4/ruby_replace_type.m4])dnl -m4_include([tool/m4/ruby_require_funcs.m4])dnl -m4_include([tool/m4/ruby_rm_recursive.m4])dnl -m4_include([tool/m4/ruby_setjmp_type.m4])dnl -m4_include([tool/m4/ruby_shared_gc.m4])dnl -m4_include([tool/m4/ruby_stack_grow_direction.m4])dnl -m4_include([tool/m4/ruby_thread.m4])dnl -m4_include([tool/m4/ruby_try_cflags.m4])dnl -m4_include([tool/m4/ruby_try_cxxflags.m4])dnl -m4_include([tool/m4/ruby_try_ldflags.m4])dnl -m4_include([tool/m4/ruby_universal_arch.m4])dnl -m4_include([tool/m4/ruby_wasm_tools.m4])dnl -m4_include([tool/m4/ruby_werror_flag.m4])dnl +m4_define([RUBY_M4_INCLUDED], [])dnl +AC_DEFUN([RUBY_M4_INCLUDE], [m4_include([tool/m4/$1])dnl + m4_append([RUBY_M4_INCLUDED], [ \ + $(tooldir)/m4/$1])dnl +]) +RUBY_M4_INCLUDE([_colorize_result_prepare.m4])dnl +RUBY_M4_INCLUDE([ac_msg_result.m4])dnl +RUBY_M4_INCLUDE([colorize_result.m4])dnl +RUBY_M4_INCLUDE([ruby_append_option.m4])dnl +RUBY_M4_INCLUDE([ruby_append_options.m4])dnl +RUBY_M4_INCLUDE([ruby_check_builtin_func.m4])dnl +RUBY_M4_INCLUDE([ruby_check_builtin_setjmp.m4])dnl +RUBY_M4_INCLUDE([ruby_check_header.m4])dnl +RUBY_M4_INCLUDE([ruby_check_printf_prefix.m4])dnl +RUBY_M4_INCLUDE([ruby_check_setjmp.m4])dnl +RUBY_M4_INCLUDE([ruby_check_signedness.m4])dnl +RUBY_M4_INCLUDE([ruby_check_sizeof.m4])dnl +RUBY_M4_INCLUDE([ruby_check_sysconf.m4])dnl +RUBY_M4_INCLUDE([ruby_cppoutfile.m4])dnl +RUBY_M4_INCLUDE([ruby_decl_attribute.m4])dnl +RUBY_M4_INCLUDE([ruby_default_arch.m4])dnl +RUBY_M4_INCLUDE([ruby_define_if.m4])dnl +RUBY_M4_INCLUDE([ruby_defint.m4])dnl +RUBY_M4_INCLUDE([ruby_dtrace_available.m4])dnl +RUBY_M4_INCLUDE([ruby_dtrace_postprocess.m4])dnl +RUBY_M4_INCLUDE([ruby_func_attribute.m4])dnl +RUBY_M4_INCLUDE([ruby_mingw32.m4])dnl +RUBY_M4_INCLUDE([ruby_prepend_option.m4])dnl +RUBY_M4_INCLUDE([ruby_prog_gnu_ld.m4])dnl +RUBY_M4_INCLUDE([ruby_prog_makedirs.m4])dnl +RUBY_M4_INCLUDE([ruby_replace_funcs.m4])dnl +RUBY_M4_INCLUDE([ruby_replace_type.m4])dnl +RUBY_M4_INCLUDE([ruby_require_funcs.m4])dnl +RUBY_M4_INCLUDE([ruby_rm_recursive.m4])dnl +RUBY_M4_INCLUDE([ruby_setjmp_type.m4])dnl +RUBY_M4_INCLUDE([ruby_shared_gc.m4])dnl +RUBY_M4_INCLUDE([ruby_stack_grow_direction.m4])dnl +RUBY_M4_INCLUDE([ruby_thread.m4])dnl +RUBY_M4_INCLUDE([ruby_try_cflags.m4])dnl +RUBY_M4_INCLUDE([ruby_try_cxxflags.m4])dnl +RUBY_M4_INCLUDE([ruby_try_ldflags.m4])dnl +RUBY_M4_INCLUDE([ruby_universal_arch.m4])dnl +RUBY_M4_INCLUDE([ruby_wasm_tools.m4])dnl +RUBY_M4_INCLUDE([ruby_werror_flag.m4])dnl AS_IF([test "x${GITHUB_ACTIONS}" = xtrue], [AC_REQUIRE([_COLORIZE_RESULT_PREPARE])dnl @@ -226,15 +232,23 @@ AS_CASE(["/${rb_CC} "], [*clang*], [ # Ditto for LLVM. Note however that llvm-as is a LLVM-IR to LLVM bitcode # assembler that does not target your machine native binary. + + # Xcode has its own version tools that may be incompatible with + # genuine LLVM tools, use the tools in the same directory. + + AS_IF([$rb_CC -E -dM -xc - < /dev/null | grep -F __apple_build_version__ > /dev/null], + [llvm_prefix=], [llvm_prefix=llvm-]) + # AC_PREPROC_IFELSE cannot be used before AC_USE_SYSTEM_EXTENSIONS + RUBY_CHECK_PROG_FOR_CC([LD], [s/clang/ld/]) # ... maybe try lld ? - RUBY_CHECK_PROG_FOR_CC([AR], [s/clang/llvm-ar/]) -# RUBY_CHECK_PROG_FOR_CC([AS], [s/clang/llvm-as/]) + RUBY_CHECK_PROG_FOR_CC([AR], [s/clang/${llvm_prefix}ar/]) +# RUBY_CHECK_PROG_FOR_CC([AS], [s/clang/${llvm_prefix}as/]) RUBY_CHECK_PROG_FOR_CC([CXX], [s/clang/clang++/]) - RUBY_CHECK_PROG_FOR_CC([NM], [s/clang/llvm-nm/]) - RUBY_CHECK_PROG_FOR_CC([OBJCOPY], [s/clang/llvm-objcopy/]) - RUBY_CHECK_PROG_FOR_CC([OBJDUMP], [s/clang/llvm-objdump/]) - RUBY_CHECK_PROG_FOR_CC([RANLIB], [s/clang/llvm-ranlib/]) - RUBY_CHECK_PROG_FOR_CC([STRIP], [s/clang/llvm-strip/]) + RUBY_CHECK_PROG_FOR_CC([NM], [s/clang/${llvm_prefix}nm/]) + RUBY_CHECK_PROG_FOR_CC([OBJCOPY], [s/clang/${llvm_prefix}objcopy/]) + RUBY_CHECK_PROG_FOR_CC([OBJDUMP], [s/clang/${llvm_prefix}objdump/]) + RUBY_CHECK_PROG_FOR_CC([RANLIB], [s/clang/${llvm_prefix}ranlib/]) + RUBY_CHECK_PROG_FOR_CC([STRIP], [s/clang/${llvm_prefix}strip/]) ]) AS_UNSET(rb_CC) AS_UNSET(rb_dummy) @@ -245,12 +259,6 @@ AS_CASE(["${build_os}"], ], [aix*], [ AC_PATH_TOOL([NM], [nm], [/usr/ccs/bin/nm], [/usr/ccs/bin:$PATH]) -], -[darwin*], [ - # For Apple clang version 14.0.3 (clang-1403.0.22.14.1) - ac_cv_prog_ac_ct_AR=`$CC -print-prog-name=ar` - ac_cv_prog_ac_ct_LD=`$CC -print-prog-name=ld` - ac_cv_prog_ac_ct_NM=`$CC -print-prog-name=nm` ]) AS_CASE(["${target_os}"], [cygwin*|msys*|mingw*|darwin*], [ @@ -430,13 +438,11 @@ AS_CASE(["$build_os"], AC_MSG_CHECKING(for $CC linker warning) suppress_ld_waring=no echo 'int main(void) {return 0;}' > conftest.c - AS_IF([$CC -framework Foundation -o conftest -ggdb3 conftest.c 2>&1 | + AS_IF([$CC -framework Foundation -o conftest conftest.c 2>&1 | grep \ -e '^ld: warning: ignoring duplicate libraries:' \ -e '^ld: warning: text-based stub file' \ -e '^ld: warning: -multiply_defined is obsolete' \ - -e "^warning: '\.debug_macinfo'" \ - -e '^note: while processing' \ >/dev/null], [ suppress_ld_waring=yes ]) @@ -898,9 +904,9 @@ AS_IF([test "$GCC" = yes], [ # suppress annoying -Wstrict-overflow warnings RUBY_TRY_CFLAGS(-fno-strict-overflow, [RUBY_APPEND_OPTION(XCFLAGS, -fno-strict-overflow)]) - test "${debugflags+set}" || {RUBY_TRY_CFLAGS(-ggdb3, [debugflags=-ggdb3])} - test "${debugflags+set}" || {RUBY_TRY_CFLAGS(-ggdb, [debugflags=-ggdb])} - test "${debugflags+set}" || {RUBY_TRY_CFLAGS(-g3, [debugflags=-g3])} + test "${debugflags+set}" || {RUBY_TRY_LDFLAGS(-ggdb3, [debugflags=-ggdb3])} + test "${debugflags+set}" || {RUBY_TRY_LDFLAGS(-ggdb, [debugflags=-ggdb])} + test "${debugflags+set}" || {RUBY_TRY_LDFLAGS(-g3, [debugflags=-g3])} ]) test $ac_cv_prog_cc_g = yes && : ${debugflags=-g} @@ -988,7 +994,6 @@ AC_SUBST(incflags, "$INCFLAGS") test -z "${ac_env_CFLAGS_set}" -a -n "${cflags+set}" && eval CFLAGS="\"$cflags $ARCH_FLAG\"" test -z "${ac_env_CXXFLAGS_set}" -a -n "${cxxflags+set}" && eval CXXFLAGS="\"$cxxflags $ARCH_FLAG\"" -} AC_CACHE_CHECK([whether compiler has statement and declarations in expressions], rb_cv_have_stmt_and_decl_in_expr, @@ -998,6 +1003,7 @@ AC_CACHE_CHECK([whether compiler has statement and declarations in expressions], AS_IF([test "$rb_cv_have_stmt_and_decl_in_expr" = yes], [ AC_DEFINE(HAVE_STMT_AND_DECL_IN_EXPR) ]) +} [begin]_group "header and library section" && { AC_ARG_WITH(winnt-ver, @@ -1253,7 +1259,8 @@ main() # __builtin_longjmp in ppc64* Linux does not restore # the TOC register (r2), which is problematic # when a global exit happens from JITted .so code. - AS_CASE(["$target_cpu"], [powerpc64*], [ + # __builtin_setjmp can have issues on arm64 linux (see [Bug #14480]). + AS_CASE(["$target_cpu"], [powerpc64*|arm64|aarch64], [ ac_cv_func___builtin_setjmp=no ]) # With gcc-8's -fcf-protection, RJIT's __builtin_longjmp fails. @@ -1375,7 +1382,7 @@ AS_CASE("$target_cpu", [x64|x86_64|i[3-6]86*], [ RUBY_UNIVERSAL_CHECK_HEADER([x86_64, i386], x86intrin.h) AS_IF([test "x$with_gmp" != xno], - [AC_CHECK_HEADERS(gmp.h) + [RUBY_CHECK_HEADER(gmp.h) AS_IF([test "x$ac_cv_header_gmp_h" != xno], AC_SEARCH_LIBS([__gmpz_init], [gmp], [AC_DEFINE(HAVE_LIBGMP, 1)]))]) @@ -1385,6 +1392,8 @@ AC_ARG_WITH([jemalloc], [with_jemalloc=$withval], [with_jemalloc=no]) AS_IF([test "x$with_jemalloc" != xno],[ # find jemalloc header first + save_CPPFLAGS="${CPPFLAGS}" + CPPFLAGS="${INCFLAGS} ${CPPFLAGS}" malloc_header= AC_CHECK_HEADER(jemalloc/jemalloc.h, [malloc_header=jemalloc/jemalloc.h], [ AC_CHECK_HEADER(jemalloc.h, [malloc_header=jemalloc.h]) @@ -1416,6 +1425,8 @@ AS_IF([test "x$with_jemalloc" != xno],[ done done ]) + CPPFLAGS="${save_CPPFLAGS}" + unset save_CPPFLAGS with_jemalloc=${rb_cv_jemalloc_library} AS_CASE(["$with_jemalloc"], [no], @@ -1750,7 +1761,7 @@ AS_IF([test "$GCC" = yes], [ AC_CACHE_CHECK(for exported function attribute, rb_cv_func_exported, [ rb_cv_func_exported=no RUBY_WERROR_FLAG([ -for mac in '__attribute__ ((__visibility__("default")))' '__declspec(dllexport)'; do +for mac in '__declspec(dllexport)' '__attribute__ ((__visibility__("default")))'; do AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@define RUBY_FUNC_EXPORTED $mac extern RUBY_FUNC_EXPORTED void conftest_attribute_check(void);]], [[]])], [rb_cv_func_exported="$mac"; break]) @@ -4601,6 +4612,9 @@ AC_CONFIG_FILES(Makefile:template/Makefile.in, [ ], [ echo 'distclean-local::; @$(RM) GNUmakefile uncommon.mk' ]) + + echo; echo '$(srcdir)/$(CONFIGURE):RUBY_M4_INCLUDED \ + $(empty)' } > $tmpmk && AS_IF([! grep '^ruby:' $tmpmk > /dev/null], [ AS_IF([test "${gnumake}" = yes], [ tmpgmk=confgmk$$.tmp diff --git a/debug_counter.h b/debug_counter.h index a8b95edded..481a0727e6 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -356,7 +356,7 @@ RB_DEBUG_COUNTER(load_path_is_not_realpath) enum rb_debug_counter_type { #define RB_DEBUG_COUNTER(name) RB_DEBUG_COUNTER_##name, -#include __FILE__ +#include "debug_counter.h" RB_DEBUG_COUNTER_MAX #undef RB_DEBUG_COUNTER }; diff --git a/defs/gmake.mk b/defs/gmake.mk index c914b39690..b34e8420ba 100644 --- a/defs/gmake.mk +++ b/defs/gmake.mk @@ -37,7 +37,7 @@ TEST_TARGETS := $(patsubst test,test-short,$(TEST_TARGETS)) TEST_DEPENDS := $(filter-out test $(TEST_TARGETS),$(TEST_DEPENDS)) TEST_TARGETS := $(patsubst test-short,btest-ruby test-knownbug test-basic,$(TEST_TARGETS)) TEST_TARGETS := $(patsubst test-basic,test-basic test-leaked-globals,$(TEST_TARGETS)) -TEST_TARGETS := $(patsubst test-bundled-gems,test-bundled-gems-run,$(TEST_TARGETS)) +TEST_TARGETS := $(patsubst test-bundled-gems,test-bundled-gems-spec test-bundled-gems-run,$(TEST_TARGETS)) TEST_TARGETS := $(patsubst test-bundled-gems-run,test-bundled-gems-run $(PREPARE_BUNDLED_GEMS),$(TEST_TARGETS)) TEST_TARGETS := $(patsubst test-bundled-gems-prepare,test-bundled-gems-prepare $(PRECHECK_BUNDLED_GEMS) test-bundled-gems-fetch,$(TEST_TARGETS)) TEST_TARGETS := $(patsubst test-bundler-parallel,test-bundler-parallel $(PREPARE_BUNDLER),$(TEST_TARGETS)) @@ -97,6 +97,7 @@ ORDERED_TEST_TARGETS := $(filter $(TEST_TARGETS), \ test-bundler-prepare test-bundler test-bundler-parallel \ test-bundled-gems-precheck test-bundled-gems-fetch \ test-bundled-gems-prepare test-bundled-gems-run \ + test-bundled-gems-spec \ ) # grep ^yes-test-.*-precheck: template/Makefile.in defs/gmake.mk common.mk diff --git a/defs/id.def b/defs/id.def index 2ddde7be70..71baa4f968 100644 --- a/defs/id.def +++ b/defs/id.def @@ -59,6 +59,7 @@ firstline, predefined = __LINE__+1, %[\ name nil path + pack _ UScore diff --git a/defs/known_errors.def b/defs/known_errors.def index 14b3cff265..23e9e53507 100644 --- a/defs/known_errors.def +++ b/defs/known_errors.def @@ -1,157 +1,157 @@ E2BIG Argument list too long EACCES Permission denied -EADDRINUSE Address in use +EADDRINUSE Address already in use EADDRNOTAVAIL Address not available -EADV +EADV Advertise error EAFNOSUPPORT Address family not supported -EAGAIN Resource unavailable, try again (may be the same value as [EWOULDBLOCK]) +EAGAIN Resource temporarily unavailable, try again (may be the same value as EWOULDBLOCK) EALREADY Connection already in progress -EAUTH -EBADARCH -EBADE -EBADEXEC +EAUTH Authentication error +EBADARCH Bad CPU type in executable +EBADE Bad exchange +EBADEXEC Bad executable EBADF Bad file descriptor -EBADFD -EBADMACHO +EBADFD File descriptor in bad state +EBADMACHO Malformed Macho file EBADMSG Bad message -EBADR -EBADRPC -EBADRQC -EBADSLT -EBFONT +EBADR Invalid request descriptor +EBADRPC RPC struct is bad +EBADRQC Invalid request code +EBADSLT Invalid slot +EBFONT Bad font file format EBUSY Device or resource busy ECANCELED Operation canceled -ECAPMODE +ECAPMODE Not permitted in capability mode ECHILD No child processes -ECHRNG -ECOMM +ECHRNG Channel number out of range +ECOMM Communication error on send ECONNABORTED Connection aborted ECONNREFUSED Connection refused ECONNRESET Connection reset -EDEADLK Resource deadlock would occur -EDEADLOCK +EDEADLK Resource deadlock avoided +EDEADLOCK File locking deadlock error EDESTADDRREQ Destination address required -EDEVERR +EDEVERR Device error; e.g., printer paper out EDOM Mathematics argument out of domain of function -EDOOFUS -EDOTDOT -EDQUOT Reserved +EDOOFUS Improper function use +EDOTDOT RFS specific error +EDQUOT Disk quota exceeded EEXIST File exists EFAULT Bad address EFBIG File too large -EFTYPE -EHOSTDOWN +EFTYPE Invalid file type or format +EHOSTDOWN Host is down EHOSTUNREACH Host is unreachable -EHWPOISON +EHWPOISON Memory page has hardware error EIDRM Identifier removed -EILSEQ Illegal byte sequence +EILSEQ Invalid or incomplete multibyte or wide character EINPROGRESS Operation in progress -EINTR Interrupted function +EINTR Interrupted function call EINVAL Invalid argument -EIO I/O error -EIPSEC +EIO Input/output error +EIPSEC IPsec processing failure EISCONN Socket is connected EISDIR Is a directory -EISNAM -EKEYEXPIRED -EKEYREJECTED -EKEYREVOKED -EL2HLT -EL2NSYNC -EL3HLT -EL3RST -ELIBACC -ELIBBAD -ELIBEXEC -ELIBMAX -ELIBSCN -ELNRNG +EISNAM Is a named file type +EKEYEXPIRED Key has expired +EKEYREJECTED Key was rejected by service +EKEYREVOKED Key has been revoked +EL2HLT Level 2 halted +EL2NSYNC Level 2 not synchronized +EL3HLT Level 3 halted +EL3RST Level 3 reset +ELIBACC Cannot access a needed shared library +ELIBBAD Accessing a corrupted shared library +ELIBEXEC Cannot exec a shared library directly +ELIBMAX Attempting to link in too many shared libraries +ELIBSCN .lib section in a.out corrupted +ELNRNG Link number out of range ELOOP Too many levels of symbolic links -EMEDIUMTYPE -EMFILE File descriptor value too large +EMEDIUMTYPE Wrong medium type +EMFILE Too many open files EMLINK Too many links -EMSGSIZE Message too large -EMULTIHOP Reserved +EMSGSIZE Message too long +EMULTIHOP Multihop attempted ENAMETOOLONG Filename too long -ENAVAIL -ENEEDAUTH +ENAVAIL No XENIX semaphores available +ENEEDAUTH Need authenticator ENETDOWN Network is down ENETRESET Connection aborted by network ENETUNREACH Network unreachable -ENFILE Too many files open in system -ENOANO -ENOATTR +ENFILE Too many open files in system +ENOANO No anode +ENOATTR Attribute not found ENOBUFS No buffer space available -ENOCSI -ENODATA [OB XSR] [Option Start] No message is available on the STREAM head read queue [Option End] +ENOCSI No CSI structure available +ENODATA No data available ENODEV No such device ENOENT No such file or directory -ENOEXEC Executable file format error -ENOKEY +ENOEXEC Exec format error +ENOKEY Required key not available ENOLCK No locks available -ENOLINK Reserved -ENOMEDIUM -ENOMEM Not enough space +ENOLINK Link has been severed +ENOMEDIUM No medium found +ENOMEM Not enough space/cannot allocate memory ENOMSG No message of the desired type -ENONET -ENOPKG -ENOPOLICY +ENONET Machine is not on the network +ENOPKG Package not installed +ENOPOLICY No such policy ENOPROTOOPT Protocol not available ENOSPC No space left on device -ENOSR [OB XSR] [Option Start] No STREAM resources [Option End] -ENOSTR [OB XSR] [Option Start] Not a STREAM [Option End] -ENOSYS Functionality not supported -ENOTBLK -ENOTCAPABLE +ENOSR No STREAM resources +ENOSTR Not a STREAM +ENOSYS Functionality not implemented +ENOTBLK Block device required +ENOTCAPABLE Capabilities insufficient ENOTCONN The socket is not connected -ENOTDIR Not a directory or a symbolic link to a directory +ENOTDIR Not a directory ENOTEMPTY Directory not empty -ENOTNAM +ENOTNAM Not a XENIX named type file ENOTRECOVERABLE State not recoverable ENOTSOCK Not a socket -ENOTSUP Not supported (may be the same value as [EOPNOTSUPP]) +ENOTSUP Operation not supported ENOTTY Inappropriate I/O control operation -ENOTUNIQ +ENOTUNIQ Name not unique on network ENXIO No such device or address -EOPNOTSUPP Operation not supported on socket (may be the same value as [ENOTSUP]) +EOPNOTSUPP Operation not supported on socket EOVERFLOW Value too large to be stored in data type -EOWNERDEAD Previous owner died +EOWNERDEAD Owner died EPERM Operation not permitted -EPFNOSUPPORT +EPFNOSUPPORT Protocol family not supported EPIPE Broken pipe -EPROCLIM -EPROCUNAVAIL -EPROGMISMATCH -EPROGUNAVAIL +EPROCLIM Too many processes +EPROCUNAVAIL Bad procedure for program +EPROGMISMATCH Program version wrong +EPROGUNAVAIL RPC program isn't available EPROTO Protocol error EPROTONOSUPPORT Protocol not supported EPROTOTYPE Protocol wrong type for socket -EPWROFF -EQFULL +EPWROFF Device power is off +EQFULL Interface output queue is full ERANGE Result too large -EREMCHG -EREMOTE -EREMOTEIO -ERESTART -ERFKILL +EREMCHG Remote address changed +EREMOTE Object is remote +EREMOTEIO Remote I/O error +ERESTART Interrupted system call should be restarted +ERFKILL Operation not possible due to RF-kill EROFS Read-only file system -ERPCMISMATCH -ESHLIBVERS -ESHUTDOWN -ESOCKTNOSUPPORT -ESPIPE Invalid seek +ERPCMISMATCH RPC version wrong +ESHLIBVERS Shared library version mismatch +ESHUTDOWN Cannot send after transport endpoint shutdown +ESOCKTNOSUPPORT Socket type not supported +ESPIPE Illegal seek ESRCH No such process -ESRMNT -ESTALE Reserved -ESTRPIPE -ETIME [OB XSR] [Option Start] Stream ioctl() timeout [Option End] +ESRMNT Server mount error +ESTALE Stale file handle +ESTRPIPE Streams pipe error +ETIME Timer expired ETIMEDOUT Connection timed out -ETOOMANYREFS +ETOOMANYREFS Too many references: cannot splice ETXTBSY Text file busy -EUCLEAN -EUNATCH -EUSERS -EWOULDBLOCK Operation would block (may be the same value as [EAGAIN]) -EXDEV Cross-device link -EXFULL -ELAST Largest errno +EUCLEAN Structure needs cleaning +EUNATCH Protocol driver not attached +EUSERS Too many users +EWOULDBLOCK Operation would block +EXDEV Invalid cross-device link +EXFULL Exchange full +ELAST Largest errno value @@ -113,6 +113,7 @@ char *strchr(char*,char); #include "internal/gc.h" #include "internal/io.h" #include "internal/object.h" +#include "internal/imemo.h" #include "internal/vm.h" #include "ruby/encoding.h" #include "ruby/ruby.h" @@ -1389,19 +1390,15 @@ rb_dir_getwd_ospath(void) VALUE cwd; VALUE path_guard; -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - path_guard = Data_Wrap_Struct((VALUE)0, NULL, RUBY_DEFAULT_FREE, NULL); + path_guard = rb_imemo_tmpbuf_auto_free_pointer(); path = ruby_getcwd(); - DATA_PTR(path_guard) = path; + rb_imemo_tmpbuf_set_ptr(path_guard, path); #ifdef __APPLE__ cwd = rb_str_normalize_ospath(path, strlen(path)); #else cwd = rb_str_new2(path); #endif - DATA_PTR(path_guard) = 0; - - xfree(path); + rb_free_tmp_buffer(&path_guard); return cwd; } #endif diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 96cee40cb4..ce844b5026 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -8,28 +8,30 @@ For RubyGems, you will also need: - * OpenSSL 1.1.x or 3.0.x / LibreSSL - * libyaml 0.1.7 or later - * zlib + * [OpenSSL] 1.1.x or 3.0.x / [LibreSSL] + * [libyaml] 0.1.7 or later + * [zlib] If you want to build from the git repository, you will also need: - * autoconf - 2.67 or later - * gperf - 3.1 or later + * [autoconf] - 2.67 or later + * [gperf] - 3.1 or later * Usually unneeded; only if you edit some source files using gperf * ruby - 3.0 or later - * We can upgrade this version to system ruby version of the latest Ubuntu LTS. + * We can upgrade this version to system ruby version of the latest + Ubuntu LTS. 2. Install optional, recommended dependencies: - * libffi (to build fiddle) - * gmp (if you with to accelerate Bignum operations) - * libexecinfo (FreeBSD) - * rustc - 1.58.0 or later, if you wish to build - [YJIT](https://docs.ruby-lang.org/en/master/RubyVM/YJIT.html). + * [libffi] (to build fiddle) + * [gmp] (if you with to accelerate Bignum operations) + * [rustc] - 1.58.0 or later, if you wish to build + [YJIT](rdoc-ref:RubyVM::YJIT). - If you installed the libraries needed for extensions (openssl, readline, libyaml, zlib) into other than the OS default place, - typically using Homebrew on macOS, add `--with-EXTLIB-dir` options to `CONFIGURE_ARGS` environment variable. + If you installed the libraries needed for extensions (openssl, readline, + libyaml, zlib) into other than the OS default place, typically using + Homebrew on macOS, add `--with-EXTLIB-dir` options to `CONFIGURE_ARGS` + environment variable. ``` shell export CONFIGURE_ARGS="" @@ -38,6 +40,16 @@ done ``` +[OpenSSL]: https://www.openssl.org +[LibreSSL]: https://www.libressl.org +[libyaml]: https://github.com/yaml/libyaml/ +[zlib]: https://www.zlib.net +[autoconf]: https://www.gnu.org/software/autoconf/ +[gperf]: https://www.gnu.org/software/gperf/ +[libffi]: https://sourceware.org/libffi/ +[gmp]: https://gmplib.org +[rustc]: https://www.rust-lang.org + ## Quick start guide 1. Download ruby source code: @@ -46,8 +58,8 @@ 1. Build from the tarball: - Download the latest tarball from [ruby-lang.org](https://www.ruby-lang.org/en/downloads/) and - extract it. Example for Ruby 3.0.2: + Download the latest tarball from [Download Ruby] page and extract + it. Example for Ruby 3.0.2: ``` shell tar -xzf ruby-3.0.2.tar.gz @@ -75,7 +87,8 @@ mkdir build && cd build ``` - While it's not necessary to build in a separate directory, it's good practice to do so. + While it's not necessary to build in a separate directory, it's good + practice to do so. 3. We'll install Ruby in `~/.rubies/ruby-master`, so create the directory: @@ -89,7 +102,8 @@ ../configure --prefix="${HOME}/.rubies/ruby-master" ``` - - Also `-C` (or `--config-cache`) would reduce time to configure from the next time. + - Also `-C` (or `--config-cache`) would reduce time to configure from the + next time. 5. Build Ruby: @@ -105,16 +119,24 @@ make install ``` - - If you need to run `make install` with `sudo` and want to avoid document generation with different permissions, you can use - `make SUDO=sudo install`. + - If you need to run `make install` with `sudo` and want to avoid document + generation with different permissions, you can use `make SUDO=sudo + install`. + +[Download Ruby]: https://www.ruby-lang.org/en/downloads/ ### Unexplainable Build Errors -If you are having unexplainable build errors, after saving all your work, try running `git clean -xfd` in the source root to remove all git ignored local files. If you are working from a source directory that's been updated several times, you may have temporary build artifacts from previous releases which can cause build failures. +If you are having unexplainable build errors, after saving all your work, try +running `git clean -xfd` in the source root to remove all git ignored local +files. If you are working from a source directory that's been updated several +times, you may have temporary build artifacts from previous releases which can +cause build failures. ## Building on Windows -The documentation for building on Windows can be found [here](../windows.md). +The documentation for building on Windows can be found in [the separated +file](../windows.md). ## More details @@ -123,8 +145,9 @@ about Ruby's build to help out. ### Running make scripts in parallel -In GNU make and BSD make implementations, to run a specific make script in parallel, pass the flag `-j<number of processes>`. For instance, -to run tests on 8 processes, use: +In GNU make[^caution-gmake-3] and BSD make implementations, to run a specific make script in +parallel, pass the flag `-j<number of processes>`. For instance, to run tests +on 8 processes, use: ``` shell make test-all -j8 @@ -132,7 +155,9 @@ make test-all -j8 We can also set `MAKEFLAGS` to run _all_ `make` commands in parallel. -Having the right `--jobs` flag will ensure all processors are utilized when building software projects. To do this effectively, you can set `MAKEFLAGS` in your shell configuration/profile: +Having the right `--jobs` flag will ensure all processors are utilized when +building software projects. To do this effectively, you can set `MAKEFLAGS` in +your shell configuration/profile: ``` shell # On macOS with Fish shell: @@ -148,11 +173,15 @@ export MAKEFLAGS="--jobs "(nproc) export MAKEFLAGS="--jobs $(nproc)" ``` +[^caution-gmake-3]: **CAUTION**: GNU make 3 is missing some features for parallel execution, we +recommend to upgrade to GNU make 4 or later. + ### Miniruby vs Ruby -Miniruby is a version of Ruby which has no external dependencies and lacks certain features. -It can be useful in Ruby development because it allows for faster build times. Miniruby is -built before Ruby. A functional Miniruby is required to build Ruby. To build Miniruby: +Miniruby is a version of Ruby which has no external dependencies and lacks +certain features. It can be useful in Ruby development because it allows for +faster build times. Miniruby is built before Ruby. A functional Miniruby is +required to build Ruby. To build Miniruby: ``` shell make miniruby @@ -160,8 +189,9 @@ make miniruby ## Debugging -You can use either lldb or gdb for debugging. Before debugging, you need to create a `test.rb` -with the Ruby script you'd like to run. You can use the following make targets: +You can use either lldb or gdb for debugging. Before debugging, you need to +create a `test.rb` with the Ruby script you'd like to run. You can use the +following make targets: * `make run`: Runs `test.rb` using Miniruby * `make lldb`: Runs `test.rb` using Miniruby in lldb @@ -172,7 +202,8 @@ with the Ruby script you'd like to run. You can use the following make targets: ### Compiling for Debugging -You should configure Ruby without optimization and other flags that may interfere with debugging: +You should configure Ruby without optimization and other flags that may +interfere with debugging: ``` shell ./configure --enable-debug-env optflags="-O0 -fno-omit-frame-pointer" @@ -180,15 +211,23 @@ You should configure Ruby without optimization and other flags that may interfer ### Building with Address Sanitizer -Using the address sanitizer (ASAN) is a great way to detect memory issues. It can detect memory safety issues in Ruby itself, and also in any C extensions compiled with and loaded into a Ruby compiled with ASAN. +Using the address sanitizer (ASAN) is a great way to detect memory issues. It +can detect memory safety issues in Ruby itself, and also in any C extensions +compiled with and loaded into a Ruby compiled with ASAN. ``` shell ./autogen.sh mkdir build && cd build -../configure CC=clang cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like +../configure CC=clang-18 cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like make ``` -The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. To run Ruby's test suite under ASAN, issue the following command. Note that this will take quite a long time (over two hours on my laptop); the `RUBY_TEST_TIMEOUT_SCALE` and `SYNTAX_SUGEST_TIMEOUT` variables are required to make sure tests don't spuriously fail with timeouts when in fact they're just slow. + +The compiled Ruby will now automatically crash with a report and a backtrace +if ASAN detects a memory safety issue. To run Ruby's test suite under ASAN, +issue the following command. Note that this will take quite a long time (over +two hours on my laptop); the `RUBY_TEST_TIMEOUT_SCALE` and +`SYNTAX_SUGEST_TIMEOUT` variables are required to make sure tests don't +spuriously fail with timeouts when in fact they're just slow. ``` shell RUBY_TEST_TIMEOUT_SCALE=5 SYNTAX_SUGGEST_TIMEOUT=600 make check @@ -196,11 +235,30 @@ RUBY_TEST_TIMEOUT_SCALE=5 SYNTAX_SUGGEST_TIMEOUT=600 make check Please note, however, the following caveats! -* ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch (and the whole test suite passes only as of commit [9d0a5148ae062a0481a4a18fbeb9cfd01dc10428](https://bugs.ruby-lang.org/projects/ruby-master/repository/git/revisions/9d0a5148ae062a0481a4a18fbeb9cfd01dc10428)) -* Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now (with the `-DUSE_MN_THREADS=0` configure argument). -* Currently, ASAN will only work correctly when using a recent head build of LLVM/Clang - it requires [this bugfix](https://github.com/llvm/llvm-project/pull/75290) related to multithreaded `fork`, which is not yet in any released version. See [here](https://llvm.org/docs/CMake.html) for instructions on how to build LLVM/Clang from source (note you will need at least the `clang` and `compiler-rt` projects enabled). Then, you will need to replace `CC=clang` in the instructions with an explicit path to your built Clang binary. -* ASAN has only been tested so far with Clang on Linux. It may or may not work with other compilers or on other platforms - please file an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if you run into problems with such configurations (or, to report that they actually work properly!) -* In particular, although I have not yet tried it, I have reason to believe ASAN will _not_ work properly on macOS yet - the fix for the multithreaded fork issue was actually reverted for macOS (see [here](https://github.com/llvm/llvm-project/commit/2a03854e4ce9bb1bcd79a211063bc63c4657f92c)). Please open an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if this is a problem for you. +* ASAN will not work properly on any currently released version of Ruby; the + necessary support is currently only present on Ruby's master branch (and the + whole test suite passes only as of commit [Revision 9d0a5148]). +* Due to [Bug #20243], Clang generates code for threadlocal variables which + doesn't work with M:N threading. Thus, it's necessary to disable M:N + threading support at build time for now (with the `-DUSE_MN_THREADS=0` + configure argument). +* ASAN will only work when using Clang version 18 or later - it requires + [llvm/llvm-project#75290] related to multithreaded `fork`. +* ASAN has only been tested so far with Clang on Linux. It may or may not work + with other compilers or on other platforms - please file an issue on + [Ruby Issue Tracking System] if you run into problems with such configurations + (or, to report that they actually work properly!) +* In particular, although I have not yet tried it, I have reason to believe + ASAN will _not_ work properly on macOS yet - the fix for the multithreaded + fork issue was actually reverted for macOS (see [llvm/llvm-project#75659]). + Please open an issue on [Ruby Issue Tracking System] if this is a problem for + you. + +[Revision 9d0a5148]: https://bugs.ruby-lang.org/projects/ruby-master/repository/git/revisions/9d0a5148ae062a0481a4a18fbeb9cfd01dc10428 +[Bug #20243]: https://bugs.ruby-lang.org/issues/20243 +[llvm/llvm-project#75290]: https://github.com/llvm/llvm-project/pull/75290 +[llvm/llvm-project#75659]: https://github.com/llvm/llvm-project/pull/75659#issuecomment-1861584777 +[Ruby Issue Tracking System]: https://bugs.ruby-lang.org ## How to measure coverage of C and Ruby code @@ -217,11 +275,12 @@ make lcov open lcov-out/index.html ``` -If you need only C code coverage, you can remove `COVERAGE=true` from the above process. -You can also use `gcov` command directly to get per-file coverage. +If you need only C code coverage, you can remove `COVERAGE=true` from the +above process. You can also use `gcov` command directly to get per-file +coverage. -If you need only Ruby code coverage, you can remove `--enable-gcov`. -Note that `test-coverage.dat` accumulates all runs of `make test-all`. -Make sure that you remove the file if you want to measure one test run. +If you need only Ruby code coverage, you can remove `--enable-gcov`. Note +that `test-coverage.dat` accumulates all runs of `make test-all`. Make sure +that you remove the file if you want to measure one test run. You can see the coverage result of CI: https://rubyci.org/coverage diff --git a/doc/encodings.rdoc b/doc/encodings.rdoc index 97c0d22616..d85099cdbc 100644 --- a/doc/encodings.rdoc +++ b/doc/encodings.rdoc @@ -419,7 +419,7 @@ These keyword-value pairs specify encoding options: hash = {"\u3042" => 'xyzzy'} hash.default = 'XYZZY' - s.encode('ASCII', fallback: h) # => "xyzzyfooXYZZY" + s.encode('ASCII', fallback: hash) # => "xyzzyfooXYZZY" def (fallback = "U+%.4X").escape(x) self % x.unpack("U") diff --git a/doc/exceptions.md b/doc/exceptions.md new file mode 100644 index 0000000000..4db2f26c18 --- /dev/null +++ b/doc/exceptions.md @@ -0,0 +1,362 @@ +# Exceptions + +Ruby code can raise exceptions. + +Most often, a raised exception is meant to alert the running program +that an unusual (i.e., _exceptional_) situation has arisen, +and may need to be handled. + +Code throughout the Ruby core, Ruby standard library, and Ruby gems generates exceptions +in certain circumstances: + +``` +File.open('nope.txt') # Raises Errno::ENOENT: "No such file or directory" +``` + +## Raised Exceptions + +A raised exception transfers program execution, one way or another. + +### Unrescued Exceptions + +If an exception not _rescued_ +(see [Rescued Exceptions](#label-Rescued+Exceptions) below), +execution transfers to code in the Ruby interpreter +that prints a message and exits the program (or thread): + +``` +$ ruby -e "raise" +-e:1:in `<main>': unhandled exception +``` + +### Rescued Exceptions + +An <i>exception handler</i> may determine what is to happen +when an exception is raised; +the handler may _rescue_ an exception, +and may prevent the program from exiting. + +A simple example: + +``` +begin + raise 'Boom!' # Raises an exception, transfers control. + puts 'Will not get here.' +rescue + puts 'Rescued an exception.' # Control transferred to here; program does not exit. +end +puts 'Got here.' +``` + +Output: + +``` +Rescued an exception. +Got here. +``` + +An exception handler has several elements: + +| Element | Use | +|-----------------------------|------------------------------------------------------------------------------------------| +| Begin clause. | Begins the handler and contains the code whose raised exception, if any, may be rescued. | +| One or more rescue clauses. | Each contains "rescuing" code, which is to be executed for certain exceptions. | +| Else clause (optional). | Contains code to be executed if no exception is raised. | +| Ensure clause (optional). | Contains code to be executed whether or not an exception is raised, or is rescued. | +| <tt>end</tt> statement. | Ends the handler. ` | + +#### Begin Clause + +The begin clause begins the exception handler: + +- May start with a `begin` statement; + see also [Begin-Less Exception Handlers](#label-Begin-Less+Exception+Handlers). +- Contains code whose raised exception (if any) is covered + by the handler. +- Ends with the first following `rescue` statement. + +#### Rescue Clauses + +A rescue clause: + +- Starts with a `rescue` statement. +- Contains code that is to be executed for certain raised exceptions. +- Ends with the first following `rescue`, + `else`, `ensure`, or `end` statement. + +A `rescue` statement may include one or more classes +that are to be rescued; +if none is given, StandardError is assumed. + +The rescue clause rescues both the specified class +(or StandardError if none given) or any of its subclasses; +(see [Built-In Exception Classes](rdoc-ref:Exception@Built-In+Exception+Classes) +for the hierarchy of Ruby built-in exception classes): + + +``` +begin + 1 / 0 # Raises ZeroDivisionError, a subclass of StandardError. +rescue + puts "Rescued #{$!.class}" +end +``` + +Output: + +``` +Rescued ZeroDivisionError +``` + +If the `rescue` statement specifies an exception class, +only that class (or one of its subclasses) is rescued; +this example exits with a ZeroDivisionError, +which was not rescued because it is not ArgumentError or one of its subclasses: + +``` +begin + 1 / 0 +rescue ArgumentError + puts "Rescued #{$!.class}" +end +``` + +A `rescue` statement may specify multiple classes, +which means that its code rescues an exception +of any of the given classes (or their subclasses): + +``` +begin + 1 / 0 +rescue FloatDomainError, ZeroDivisionError + puts "Rescued #{$!.class}" +end +``` + +An exception handler may contain multiple rescue clauses; +in that case, the first clause that rescues the exception does so, +and those before and after are ignored: + +``` +begin + Dir.open('nosuch') +rescue Errno::ENOTDIR + puts "Rescued #{$!.class}" +rescue Errno::ENOENT + puts "Rescued #{$!.class}" +end +``` + +Output: + +``` +Rescued Errno::ENOENT +``` + +A `rescue` statement may specify a variable +whose value becomes the rescued exception +(an instance of Exception or one of its subclasses: + +``` +begin + 1 / 0 +rescue => x + puts x.class + puts x.message +end +``` + +Output: + +``` +ZeroDivisionError +divided by 0 +``` + +In the rescue clause, these global variables are defined: + +- `$!`": the current exception instance. +- `$@`: its backtrace. + +#### Else Clause + +The `else` clause: + +- Starts with an `else` statement. +- Contains code that is to be executed if no exception is raised in the begin clause. +- Ends with the first following `ensure` or `end` statement. + +``` +begin + puts 'Begin.' +rescue + puts 'Rescued an exception!' +else + puts 'No exception raised.' +end +``` + +Output: + +``` +Begin. +No exception raised. +``` + +#### Ensure Clause + +The ensure clause: + +- Starts with an `ensure` statement. +- Contains code that is to be executed + regardless of whether an exception is raised, + and regardless of whether a raised exception is handled. +- Ends with the first following `end` statement. + +``` +def foo(boom: false) + puts 'Begin.' + raise 'Boom!' if boom +rescue + puts 'Rescued an exception!' +else + puts 'No exception raised.' +ensure + puts 'Always do this.' +end + +foo(boom: true) +foo(boom: false) +``` + +Output: + +``` +Begin. +Rescued an exception! +Always do this. +Begin. +No exception raised. +Always do this. +``` + +#### End Statement + +The `end` statement ends the handler. + +Code following it is reached only if any raised exception is rescued. + +#### Begin-Less \Exception Handlers + +As seen above, an exception handler may be implemented with `begin` and `end`. + +An exception handler may also be implemented as: + +- A method body: + + ``` + def foo(boom: false) # Serves as beginning of exception handler. + puts 'Begin.' + raise 'Boom!' if boom + rescue + puts 'Rescued an exception!' + else + puts 'No exception raised.' + end # Serves as end of exception handler. + ``` + +- A block: + + ``` + Dir.chdir('.') do |dir| # Serves as beginning of exception handler. + raise 'Boom!' + rescue + puts 'Rescued an exception!' + end # Serves as end of exception handler. + ``` + +#### Re-Raising an \Exception + +It can be useful to rescue an exception, but allow its eventual effect; +for example, a program can rescue an exception, log data about it, +and then "reinstate" the exception. + +This may be done via the `raise` method, but in a special way; +a rescuing clause: + + - Captures an exception. + - Does whatever is needed concerning the exception (such as logging it). + - Calls method `raise` with no argument, + which raises the rescued exception: + +``` +begin + 1 / 0 +rescue ZeroDivisionError + # Do needful things (like logging). + raise # Raised exception will be ZeroDivisionError, not RuntimeError. +end +``` + +Output: + +``` +ruby t.rb +t.rb:2:in `/': divided by 0 (ZeroDivisionError) + from t.rb:2:in `<main>' +``` + +#### Retrying + +It can be useful to retry a begin clause; +for example, if it must access a possibly-volatile resource +(such as a web page), +it can be useful to try the access more than once +(in the hope that it may become available): + +``` +retries = 0 +begin + puts "Try ##{retries}." + raise 'Boom' +rescue + puts "Rescued retry ##{retries}." + if (retries += 1) < 3 + puts 'Retrying' + retry + else + puts 'Giving up.' + raise + end +end +``` + +``` +Try #0. +Rescued retry #0. +Retrying +Try #1. +Rescued retry #1. +Retrying +Try #2. +Rescued retry #2. +Giving up. +# RuntimeError ('Boom') raised. +``` + +Note that the retry re-executes the entire begin clause, +not just the part after the point of failure. + +## Raising an \Exception + +Raise an exception with method Kernel#raise. + +## Custom Exceptions + +To provide additional or alternate information, +you may create custom exception classes; +each should be a subclass of one of the built-in exception classes: + +``` +class MyException < StandardError; end +``` diff --git a/doc/format_specifications.rdoc b/doc/format_specifications.rdoc index 1111575e74..bdfdc24953 100644 --- a/doc/format_specifications.rdoc +++ b/doc/format_specifications.rdoc @@ -233,6 +233,8 @@ Format +argument+ as a single character: sprintf('%c', 'A') # => "A" sprintf('%c', 65) # => "A" +This behaves like String#<<, except for raising ArgumentError instead of RangeError. + === Specifier +d+ Format +argument+ as a decimal integer: diff --git a/doc/strscan/helper_methods.md b/doc/strscan/helper_methods.md new file mode 100644 index 0000000000..6555a2ce66 --- /dev/null +++ b/doc/strscan/helper_methods.md @@ -0,0 +1,128 @@ +## Helper Methods + +These helper methods display values returned by scanner's methods. + +### `put_situation(scanner)` + +Display scanner's situation: + +- Byte position (`#pos`). +- Character position (`#charpos`) +- Target string (`#rest`) and size (`#rest_size`). + +``` +scanner = StringScanner.new('foobarbaz') +scanner.scan(/foo/) +put_situation(scanner) +# Situation: +# pos: 3 +# charpos: 3 +# rest: "barbaz" +# rest_size: 6 +``` + +### `put_match_values(scanner)` + +Display the scanner's match values: + +``` +scanner = StringScanner.new('Fri Dec 12 1975 14:39') +pattern = /(?<wday>\w+) (?<month>\w+) (?<day>\d+) / +scanner.match?(pattern) +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 11 +# pre_match: "" +# matched : "Fri Dec 12 " +# post_match: "1975 14:39" +# Captured match values: +# size: 4 +# captures: ["Fri", "Dec", "12"] +# named_captures: {"wday"=>"Fri", "month"=>"Dec", "day"=>"12"} +# values_at: ["Fri Dec 12 ", "Fri", "Dec", "12", nil] +# []: +# [0]: "Fri Dec 12 " +# [1]: "Fri" +# [2]: "Dec" +# [3]: "12" +# [4]: nil +``` + +### `match_values_cleared?(scanner)` + +Returns whether the scanner's match values are all properly cleared: + +``` +scanner = StringScanner.new('foobarbaz') +match_values_cleared?(scanner) # => true +put_match_values(scanner) +# Basic match values: +# matched?: false +# matched_size: nil +# pre_match: nil +# matched : nil +# post_match: nil +# Captured match values: +# size: nil +# captures: nil +# named_captures: {} +# values_at: nil +# [0]: nil +scanner.scan(/foo/) +match_values_cleared?(scanner) # => false +``` + +## The Code + +``` +def put_situation(scanner) + puts '# Situation:' + puts "# pos: #{scanner.pos}" + puts "# charpos: #{scanner.charpos}" + puts "# rest: #{scanner.rest.inspect}" + puts "# rest_size: #{scanner.rest_size}" +end +``` + +``` +def put_match_values(scanner) + puts '# Basic match values:' + puts "# matched?: #{scanner.matched?}" + value = scanner.matched_size || 'nil' + puts "# matched_size: #{value}" + puts "# pre_match: #{scanner.pre_match.inspect}" + puts "# matched : #{scanner.matched.inspect}" + puts "# post_match: #{scanner.post_match.inspect}" + puts '# Captured match values:' + puts "# size: #{scanner.size}" + puts "# captures: #{scanner.captures}" + puts "# named_captures: #{scanner.named_captures}" + if scanner.size.nil? + puts "# values_at: #{scanner.values_at(0)}" + puts "# [0]: #{scanner[0]}" + else + puts "# values_at: #{scanner.values_at(*(0..scanner.size))}" + puts "# []:" + scanner.size.times do |i| + puts "# [#{i}]: #{scanner[i].inspect}" + end + end +end +``` + +``` +def match_values_cleared?(scanner) + scanner.matched? == false && + scanner.matched_size.nil? && + scanner.matched.nil? && + scanner.pre_match.nil? && + scanner.post_match.nil? && + scanner.size.nil? && + scanner[0].nil? && + scanner.captures.nil? && + scanner.values_at(0..1).nil? && + scanner.named_captures == {} +end +``` + diff --git a/doc/strscan/link_refs.txt b/doc/strscan/link_refs.txt new file mode 100644 index 0000000000..19f6f7ce5c --- /dev/null +++ b/doc/strscan/link_refs.txt @@ -0,0 +1,17 @@ +[1]: rdoc-ref:StringScanner@Stored+String +[2]: rdoc-ref:StringScanner@Byte+Position+-28Position-29 +[3]: rdoc-ref:StringScanner@Target+Substring +[4]: rdoc-ref:StringScanner@Setting+the+Target+Substring +[5]: rdoc-ref:StringScanner@Traversing+the+Target+Substring +[6]: https://docs.ruby-lang.org/en/master/Regexp.html +[7]: rdoc-ref:StringScanner@Character+Position +[8]: https://docs.ruby-lang.org/en/master/String.html#method-i-5B-5D +[9]: rdoc-ref:StringScanner@Match+Values +[10]: rdoc-ref:StringScanner@Fixed-Anchor+Property +[11]: rdoc-ref:StringScanner@Positions +[13]: rdoc-ref:StringScanner@Captured+Match+Values +[14]: rdoc-ref:StringScanner@Querying+the+Target+Substring +[15]: rdoc-ref:StringScanner@Searching+the+Target+Substring +[16]: https://docs.ruby-lang.org/en/master/Regexp.html#class-Regexp-label-Groups+and+Captures +[17]: rdoc-ref:StringScanner@Matching +[18]: rdoc-ref:StringScanner@Basic+Match+Values diff --git a/doc/strscan/methods/get_byte.md b/doc/strscan/methods/get_byte.md new file mode 100644 index 0000000000..2f23be1899 --- /dev/null +++ b/doc/strscan/methods/get_byte.md @@ -0,0 +1,30 @@ +call-seq: + get_byte -> byte_as_character or nil + +Returns the next byte, if available: + +- If the [position][2] + is not at the end of the [stored string][1]: + + - Returns the next byte. + - Increments the [byte position][2]. + - Adjusts the [character position][7]. + + ``` + scanner = StringScanner.new(HIRAGANA_TEXT) + # => #<StringScanner 0/15 @ "\xE3\x81\x93\xE3\x82..."> + scanner.string # => "こんにちは" + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\xE3", 1, 1] + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\x81", 2, 2] + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\x93", 3, 1] + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\xE3", 4, 2] + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\x82", 5, 3] + [scanner.get_byte, scanner.pos, scanner.charpos] # => ["\x93", 6, 2] + ``` + +- Otherwise, returns `nil`, and does not change the positions. + + ``` + scanner.terminate + [scanner.get_byte, scanner.pos, scanner.charpos] # => [nil, 15, 5] + ``` diff --git a/doc/strscan/methods/get_charpos.md b/doc/strscan/methods/get_charpos.md new file mode 100644 index 0000000000..f77563c860 --- /dev/null +++ b/doc/strscan/methods/get_charpos.md @@ -0,0 +1,19 @@ +call-seq: + charpos -> character_position + +Returns the [character position][7] (initially zero), +which may be different from the [byte position][2] +given by method #pos: + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.getch # => "こ" # 3-byte character. +scanner.getch # => "ん" # 3-byte character. +put_situation(scanner) +# Situation: +# pos: 6 +# charpos: 2 +# rest: "にちは" +# rest_size: 9 +``` diff --git a/doc/strscan/methods/get_pos.md b/doc/strscan/methods/get_pos.md new file mode 100644 index 0000000000..56bcef3274 --- /dev/null +++ b/doc/strscan/methods/get_pos.md @@ -0,0 +1,14 @@ +call-seq: + pos -> byte_position + +Returns the integer [byte position][2], +which may be different from the [character position][7]: + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos # => 0 +scanner.getch # => "こ" # 3-byte character. +scanner.charpos # => 1 +scanner.pos # => 3 +``` diff --git a/doc/strscan/methods/getch.md b/doc/strscan/methods/getch.md new file mode 100644 index 0000000000..b57732ad7c --- /dev/null +++ b/doc/strscan/methods/getch.md @@ -0,0 +1,43 @@ +call-seq: + getch -> character or nil + +Returns the next (possibly multibyte) character, +if available: + +- If the [position][2] + is at the beginning of a character: + + - Returns the character. + - Increments the [character position][7] by 1. + - Increments the [byte position][2] + by the size (in bytes) of the character. + + ``` + scanner = StringScanner.new(HIRAGANA_TEXT) + scanner.string # => "こんにちは" + [scanner.getch, scanner.pos, scanner.charpos] # => ["こ", 3, 1] + [scanner.getch, scanner.pos, scanner.charpos] # => ["ん", 6, 2] + [scanner.getch, scanner.pos, scanner.charpos] # => ["に", 9, 3] + [scanner.getch, scanner.pos, scanner.charpos] # => ["ち", 12, 4] + [scanner.getch, scanner.pos, scanner.charpos] # => ["は", 15, 5] + [scanner.getch, scanner.pos, scanner.charpos] # => [nil, 15, 5] + ``` + +- If the [position][2] is within a multi-byte character + (that is, not at its beginning), + behaves like #get_byte (returns a 1-byte character): + + ``` + scanner.pos = 1 + [scanner.getch, scanner.pos, scanner.charpos] # => ["\x81", 2, 2] + [scanner.getch, scanner.pos, scanner.charpos] # => ["\x93", 3, 1] + [scanner.getch, scanner.pos, scanner.charpos] # => ["ん", 6, 2] + ``` + +- If the [position][2] is at the end of the [stored string][1], + returns `nil` and does not modify the positions: + + ``` + scanner.terminate + [scanner.getch, scanner.pos, scanner.charpos] # => [nil, 15, 5] + ``` diff --git a/doc/strscan/methods/scan.md b/doc/strscan/methods/scan.md new file mode 100644 index 0000000000..714fa9910a --- /dev/null +++ b/doc/strscan/methods/scan.md @@ -0,0 +1,51 @@ +call-seq: + scan(pattern) -> substring or nil + +Attempts to [match][17] the given `pattern` +at the beginning of the [target substring][3]. + +If the match succeeds: + +- Returns the matched substring. +- Increments the [byte position][2] by <tt>substring.bytesize</tt>, + and may increment the [character position][7]. +- Sets [match values][9]. + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos = 6 +scanner.scan(/に/) # => "に" +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 3 +# pre_match: "こん" +# matched : "に" +# post_match: "ちは" +# Captured match values: +# size: 1 +# captures: [] +# named_captures: {} +# values_at: ["に", nil] +# []: +# [0]: "に" +# [1]: nil +put_situation(scanner) +# Situation: +# pos: 9 +# charpos: 3 +# rest: "ちは" +# rest_size: 6 +``` + +If the match fails: + +- Returns `nil`. +- Does not increment byte and character positions. +- Clears match values. + +``` +scanner.scan(/nope/) # => nil +match_values_cleared?(scanner) # => true +``` diff --git a/doc/strscan/methods/scan_until.md b/doc/strscan/methods/scan_until.md new file mode 100644 index 0000000000..3b7ff2c3a9 --- /dev/null +++ b/doc/strscan/methods/scan_until.md @@ -0,0 +1,52 @@ +call-seq: + scan_until(pattern) -> substring or nil + +Attempts to [match][17] the given `pattern` +anywhere (at any [position][2]) in the [target substring][3]. + +If the match attempt succeeds: + +- Sets [match values][9]. +- Sets the [byte position][2] to the end of the matched substring; + may adjust the [character position][7]. +- Returns the matched substring. + + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos = 6 +scanner.scan_until(/ち/) # => "にち" +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 3 +# pre_match: "こんに" +# matched : "ち" +# post_match: "は" +# Captured match values: +# size: 1 +# captures: [] +# named_captures: {} +# values_at: ["ち", nil] +# []: +# [0]: "ち" +# [1]: nil +put_situation(scanner) +# Situation: +# pos: 12 +# charpos: 4 +# rest: "は" +# rest_size: 3 +``` + +If the match attempt fails: + +- Clears match data. +- Returns `nil`. +- Does not update positions. + +``` +scanner.scan_until(/nope/) # => nil +match_values_cleared?(scanner) # => true +``` diff --git a/doc/strscan/methods/set_pos.md b/doc/strscan/methods/set_pos.md new file mode 100644 index 0000000000..230177109c --- /dev/null +++ b/doc/strscan/methods/set_pos.md @@ -0,0 +1,27 @@ +call-seq: + pos = n -> n + pointer = n -> n + +Sets the [byte position][2] and the [character position][11]; +returns `n`. + +Does not affect [match values][9]. + +For non-negative `n`, sets the position to `n`: + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos = 3 # => 3 +scanner.rest # => "んにちは" +scanner.charpos # => 1 +``` + +For negative `n`, counts from the end of the [stored string][1]: + +``` +scanner.pos = -9 # => -9 +scanner.pos # => 6 +scanner.rest # => "にちは" +scanner.charpos # => 2 +``` diff --git a/doc/strscan/methods/skip.md b/doc/strscan/methods/skip.md new file mode 100644 index 0000000000..656f134c5a --- /dev/null +++ b/doc/strscan/methods/skip.md @@ -0,0 +1,43 @@ +call-seq: + skip(pattern) match_size or nil + +Attempts to [match][17] the given `pattern` +at the beginning of the [target substring][3]; + +If the match succeeds: + +- Increments the [byte position][2] by substring.bytesize, + and may increment the [character position][7]. +- Sets [match values][9]. +- Returns the size (bytes) of the matched substring. + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos = 6 +scanner.skip(/に/) # => 3 +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 3 +# pre_match: "こん" +# matched : "に" +# post_match: "ちは" +# Captured match values: +# size: 1 +# captures: [] +# named_captures: {} +# values_at: ["に", nil] +# []: +# [0]: "に" +# [1]: nil +put_situation(scanner) +# Situation: +# pos: 9 +# charpos: 3 +# rest: "ちは" +# rest_size: 6 + +scanner.skip(/nope/) # => nil +match_values_cleared?(scanner) # => true +``` diff --git a/doc/strscan/methods/skip_until.md b/doc/strscan/methods/skip_until.md new file mode 100644 index 0000000000..5187a4826f --- /dev/null +++ b/doc/strscan/methods/skip_until.md @@ -0,0 +1,49 @@ +call-seq: + skip_until(pattern) -> matched_substring_size or nil + +Attempts to [match][17] the given `pattern` +anywhere (at any [position][2]) in the [target substring][3]; +does not modify the positions. + +If the match attempt succeeds: + +- Sets [match values][9]. +- Returns the size of the matched substring. + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.pos = 6 +scanner.skip_until(/ち/) # => 6 +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 3 +# pre_match: "こんに" +# matched : "ち" +# post_match: "は" +# Captured match values: +# size: 1 +# captures: [] +# named_captures: {} +# values_at: ["ち", nil] +# []: +# [0]: "ち" +# [1]: nil +put_situation(scanner) +# Situation: +# pos: 12 +# charpos: 4 +# rest: "は" +# rest_size: 3 +``` + +If the match attempt fails: + +- Clears match values. +- Returns `nil`. + +``` +scanner.skip_until(/nope/) # => nil +match_values_cleared?(scanner) # => true +``` diff --git a/doc/strscan/methods/terminate.md b/doc/strscan/methods/terminate.md new file mode 100644 index 0000000000..fd55727099 --- /dev/null +++ b/doc/strscan/methods/terminate.md @@ -0,0 +1,30 @@ +call-seq: + terminate -> self + +Sets the scanner to end-of-string; +returns +self+: + +- Sets both [positions][11] to end-of-stream. +- Clears [match values][9]. + +``` +scanner = StringScanner.new(HIRAGANA_TEXT) +scanner.string # => "こんにちは" +scanner.scan_until(/に/) +put_situation(scanner) +# Situation: +# pos: 9 +# charpos: 3 +# rest: "ちは" +# rest_size: 6 +match_values_cleared?(scanner) # => false + +scanner.terminate # => #<StringScanner fin> +put_situation(scanner) +# Situation: +# pos: 15 +# charpos: 5 +# rest: "" +# rest_size: 0 +match_values_cleared?(scanner) # => true +``` diff --git a/doc/strscan/strscan.md b/doc/strscan/strscan.md new file mode 100644 index 0000000000..465cebd4cb --- /dev/null +++ b/doc/strscan/strscan.md @@ -0,0 +1,543 @@ +\Class `StringScanner` supports processing a stored string as a stream; +this code creates a new `StringScanner` object with string `'foobarbaz'`: + +``` +require 'strscan' +scanner = StringScanner.new('foobarbaz') +``` + +## About the Examples + +All examples here assume that `StringScanner` has been required: + +``` +require 'strscan' +``` + +Some examples here assume that these constants are defined: + +``` +MULTILINE_TEXT = <<~EOT +Go placidly amid the noise and haste, +and remember what peace there may be in silence. +EOT + +HIRAGANA_TEXT = 'こんにちは' + +ENGLISH_TEXT = 'Hello' +``` + +Some examples here assume that certain helper methods are defined: + +- `put_situation(scanner)`: + Displays the values of the scanner's + methods #pos, #charpos, #rest, and #rest_size. +- `put_match_values(scanner)`: + Displays the scanner's [match values][9]. +- `match_values_cleared?(scanner)`: + Returns whether the scanner's [match values][9] are cleared. + +See examples [here][ext/strscan/helper_methods_md.html]. + +## The `StringScanner` \Object + +This code creates a `StringScanner` object +(we'll call it simply a _scanner_), +and shows some of its basic properties: + +``` +scanner = StringScanner.new('foobarbaz') +scanner.string # => "foobarbaz" +put_situation(scanner) +# Situation: +# pos: 0 +# charpos: 0 +# rest: "foobarbaz" +# rest_size: 9 +``` + +The scanner has: + +* A <i>stored string</i>, which is: + + * Initially set by StringScanner.new(string) to the given `string` + (`'foobarbaz'` in the example above). + * Modifiable by methods #string=(new_string) and #concat(more_string). + * Returned by method #string. + + More at [Stored String][1] below. + +* A _position_; + a zero-based index into the bytes of the stored string (_not_ into its characters): + + * Initially set by StringScanner.new to `0`. + * Returned by method #pos. + * Modifiable explicitly by methods #reset, #terminate, and #pos=(new_pos). + * Modifiable implicitly (various traversing methods, among others). + + More at [Byte Position][2] below. + +* A <i>target substring</i>, + which is a trailing substring of the stored string; + it extends from the current position to the end of the stored string: + + * Initially set by StringScanner.new(string) to the given `string` + (`'foobarbaz'` in the example above). + * Returned by method #rest. + * Modified by any modification to either the stored string or the position. + + <b>Most importantly</b>: + the searching and traversing methods operate on the target substring, + which may be (and often is) less than the entire stored string. + + More at [Target Substring][3] below. + +## Stored \String + +The <i>stored string</i> is the string stored in the `StringScanner` object. + +Each of these methods sets, modifies, or returns the stored string: + +| Method | Effect | +|----------------------|-------------------------------------------------| +| ::new(string) | Creates a new scanner for the given string. | +| #string=(new_string) | Replaces the existing stored string. | +| #concat(more_string) | Appends a string to the existing stored string. | +| #string | Returns the stored string. | + +## Positions + +A `StringScanner` object maintains a zero-based <i>byte position</i> +and a zero-based <i>character position</i>. + +Each of these methods explicitly sets positions: + +| Method | Effect | +|--------------------------|----------------------------------------------------------| +| #reset | Sets both positions to zero (begining of stored string). | +| #terminate | Sets both positions to the end of the stored string. | +| #pos=(new_byte_position) | Sets byte position; adjusts character position. | + +### Byte Position (Position) + +The byte position (or simply _position_) +is a zero-based index into the bytes in the scanner's stored string; +for a new `StringScanner` object, the byte position is zero. + +When the byte position is: + +* Zero (at the beginning), the target substring is the entire stored string. +* Equal to the size of the stored string (at the end), + the target substring is the empty string `''`. + +To get or set the byte position: + +* \#pos: returns the byte position. +* \#pos=(new_pos): sets the byte position. + +Many methods use the byte position as the basis for finding matches; +many others set, increment, or decrement the byte position: + +``` +scanner = StringScanner.new('foobar') +scanner.pos # => 0 +scanner.scan(/foo/) # => "foo" # Match found. +scanner.pos # => 3 # Byte position incremented. +scanner.scan(/foo/) # => nil # Match not found. +scanner.pos # => 3 # Byte position not changed. +``` + +Some methods implicitly modify the byte position; +see: + +* [Setting the Target Substring][4]. +* [Traversing the Target Substring][5]. + +The values of these methods are derived directly from the values of #pos and #string: + +- \#charpos: the [character position][7]. +- \#rest: the [target substring][3]. +- \#rest_size: `rest.size`. + +### Character Position + +The character position is a zero-based index into the _characters_ +in the stored string; +for a new `StringScanner` object, the character position is zero. + +\Method #charpos returns the character position; +its value may not be reset explicitly. + +Some methods change (increment or reset) the character position; +see: + +* [Setting the Target Substring][4]. +* [Traversing the Target Substring][5]. + +Example (string includes multi-byte characters): + +``` +scanner = StringScanner.new(ENGLISH_TEXT) # Five 1-byte characters. +scanner.concat(HIRAGANA_TEXT) # Five 3-byte characters +scanner.string # => "Helloこんにちは" # Twenty bytes in all. +put_situation(scanner) +# Situation: +# pos: 0 +# charpos: 0 +# rest: "Helloこんにちは" +# rest_size: 20 +scanner.scan(/Hello/) # => "Hello" # Five 1-byte characters. +put_situation(scanner) +# Situation: +# pos: 5 +# charpos: 5 +# rest: "こんにちは" +# rest_size: 15 +scanner.getch # => "こ" # One 3-byte character. +put_situation(scanner) +# Situation: +# pos: 8 +# charpos: 6 +# rest: "んにちは" +# rest_size: 12``` + +## Target Substring + +The target substring is the the part of the [stored string][1] +that extends from the current [byte position][2] to the end of the stored string; +it is always either: + +- The entire stored string (byte position is zero). +- A trailing substring of the stored string (byte position positive). + +The target substring is returned by method #rest, +and its size is returned by method #rest_size. + +Examples: + +``` +scanner = StringScanner.new('foobarbaz') +put_situation(scanner) +# Situation: +# pos: 0 +# charpos: 0 +# rest: "foobarbaz" +# rest_size: 9 +scanner.pos = 3 +put_situation(scanner) +# Situation: +# pos: 3 +# charpos: 3 +# rest: "barbaz" +# rest_size: 6 +scanner.pos = 9 +put_situation(scanner) +# Situation: +# pos: 9 +# charpos: 9 +# rest: "" +# rest_size: 0 +``` + +### Setting the Target Substring + +The target substring is set whenever: + +* The [stored string][1] is set (position reset to zero; target substring set to stored string). +* The [byte position][2] is set (target substring adjusted accordingly). + +### Querying the Target Substring + +This table summarizes (details and examples at the links): + +| Method | Returns | +|------------|-----------------------------------| +| #rest | Target substring. | +| #rest_size | Size (bytes) of target substring. | + +### Searching the Target Substring + +A _search_ method examines the target substring, +but does not advance the [positions][11] +or (by implication) shorten the target substring. + +This table summarizes (details and examples at the links): + +| Method | Returns | Sets Match Values? | +|-----------------------|-----------------------------------------------|--------------------| +| #check(pattern) | Matched leading substring or +nil+. | Yes. | +| #check_until(pattern) | Matched substring (anywhere) or +nil+. | Yes. | +| #exist?(pattern) | Matched substring (anywhere) end index. | Yes. | +| #match?(pattern) | Size of matched leading substring or +nil+. | Yes. | +| #peek(size) | Leading substring of given length (bytes). | No. | +| #peek_byte | Integer leading byte or +nil+. | No. | +| #rest | Target substring (from byte position to end). | No. | + +### Traversing the Target Substring + +A _traversal_ method examines the target substring, +and, if successful: + +- Advances the [positions][11]. +- Shortens the target substring. + + +This table summarizes (details and examples at links): + +| Method | Returns | Sets Match Values? | +|----------------------|------------------------------------------------------|--------------------| +| #get_byte | Leading byte or +nil+. | No. | +| #getch | Leading character or +nil+. | No. | +| #scan(pattern) | Matched leading substring or +nil+. | Yes. | +| #scan_byte | Integer leading byte or +nil+. | No. | +| #scan_until(pattern) | Matched substring (anywhere) or +nil+. | Yes. | +| #skip(pattern) | Matched leading substring size or +nil+. | Yes. | +| #skip_until(pattern) | Position delta to end-of-matched-substring or +nil+. | Yes. | +| #unscan | +self+. | No. | + +## Querying the Scanner + +Each of these methods queries the scanner object +without modifying it (details and examples at links) + +| Method | Returns | +|---------------------|----------------------------------| +| #beginning_of_line? | +true+ or +false+. | +| #charpos | Character position. | +| #eos? | +true+ or +false+. | +| #fixed_anchor? | +true+ or +false+. | +| #inspect | String representation of +self+. | +| #pos | Byte position. | +| #rest | Target substring. | +| #rest_size | Size of target substring. | +| #string | Stored string. | + +## Matching + +`StringScanner` implements pattern matching via Ruby class [Regexp][6], +and its matching behaviors are the same as Ruby's +except for the [fixed-anchor property][10]. + +### Matcher Methods + +Each <i>matcher method</i> takes a single argument `pattern`, +and attempts to find a matching substring in the [target substring][3]. + +| Method | Pattern Type | Matches Target Substring | Success Return | May Update Positions? | +|--------------|-------------------|--------------------------|--------------------|-----------------------| +| #check | Regexp or String. | At beginning. | Matched substring. | No. | +| #check_until | Regexp. | Anywhere. | Substring. | No. | +| #match? | Regexp or String. | At beginning. | Updated position. | No. | +| #exist? | Regexp. | Anywhere. | Updated position. | No. | +| #scan | Regexp or String. | At beginning. | Matched substring. | Yes. | +| #scan_until | Regexp. | Anywhere. | Substring. | Yes. | +| #skip | Regexp or String. | At beginning. | Match size. | Yes. | +| #skip_until | Regexp. | Anywhere. | Position delta. | Yes. | + +<br> + +Which matcher you choose will depend on: + +- Where you want to find a match: + + - Only at the beginning of the target substring: + #check, #match?, #scan, #skip. + - Anywhere in the target substring: + #check_until, #exist?, #scan_until, #skip_until. + +- Whether you want to: + + - Traverse, by advancing the positions: + #scan, #scan_until, #skip, #skip_until. + - Keep the positions unchanged: + #check, #check_until, #exist?, #match?. + +- What you want for the return value: + + - The matched substring: #check, #check_until, #scan, #scan_until. + - The updated position: #exist?, #match?. + - The position delta: #skip_until. + - The match size: #skip. + +### Match Values + +The <i>match values</i> in a `StringScanner` object +generally contain the results of the most recent attempted match. + +Each match value may be thought of as: + +* _Clear_: Initially, or after an unsuccessful match attempt: + usually, `false`, `nil`, or `{}`. +* _Set_: After a successful match attempt: + `true`, string, array, or hash. + +Each of these methods clears match values: + +- ::new(string). +- \#reset. +- \#terminate. + +Each of these methods attempts a match based on a pattern, +and either sets match values (if successful) or clears them (if not); + +- \#check(pattern) +- \#check_until(pattern) +- \#exist?(pattern) +- \#match?(pattern) +- \#scan(pattern) +- \#scan_until(pattern) +- \#skip(pattern) +- \#skip_until(pattern) + +#### Basic Match Values + +Basic match values are those not related to captures. + +Each of these methods returns a basic match value: + +| Method | Return After Match | Return After No Match | +|-----------------|----------------------------------------|-----------------------| +| #matched? | +true+. | +false+. | +| #matched_size | Size of matched substring. | +nil+. | +| #matched | Matched substring. | +nil+. | +| #pre_match | Substring preceding matched substring. | +nil+. | +| #post_match | Substring following matched substring. | +nil+. | + +<br> + +See examples below. + +#### Captured Match Values + +Captured match values are those related to [captures][16]. + +Each of these methods returns a captured match value: + +| Method | Return After Match | Return After No Match | +|-----------------|-----------------------------------------|-----------------------| +| #size | Count of captured substrings. | +nil+. | +| #[](n) | <tt>n</tt>th captured substring. | +nil+. | +| #captures | Array of all captured substrings. | +nil+. | +| #values_at(*n) | Array of specified captured substrings. | +nil+. | +| #named_captures | Hash of named captures. | <tt>{}</tt>. | + +<br> + +See examples below. + +#### Match Values Examples + +Successful basic match attempt (no captures): + +``` +scanner = StringScanner.new('foobarbaz') +scanner.exist?(/bar/) +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 3 +# pre_match: "foo" +# matched : "bar" +# post_match: "baz" +# Captured match values: +# size: 1 +# captures: [] +# named_captures: {} +# values_at: ["bar", nil] +# []: +# [0]: "bar" +# [1]: nil +``` + +Failed basic match attempt (no captures); + +``` +scanner = StringScanner.new('foobarbaz') +scanner.exist?(/nope/) +match_values_cleared?(scanner) # => true +``` + +Successful unnamed capture match attempt: + +``` +scanner = StringScanner.new('foobarbazbatbam') +scanner.exist?(/(foo)bar(baz)bat(bam)/) +put_match_values(scanner) +# Basic match values: +# matched?: true +# matched_size: 15 +# pre_match: "" +# matched : "foobarbazbatbam" +# post_match: "" +# Captured match values: +# size: 4 +# captures: ["foo", "baz", "bam"] +# named_captures: {} +# values_at: ["foobarbazbatbam", "foo", "baz", "bam", nil] +# []: +# [0]: "foobarbazbatbam" +# [1]: "foo" +# [2]: "baz" +# [3]: "bam" +# [4]: nil +``` + +Successful named capture match attempt; +same as unnamed above, except for #named_captures: + +``` +scanner = StringScanner.new('foobarbazbatbam') +scanner.exist?(/(?<x>foo)bar(?<y>baz)bat(?<z>bam)/) +scanner.named_captures # => {"x"=>"foo", "y"=>"baz", "z"=>"bam"} +``` + +Failed unnamed capture match attempt: + +``` +scanner = StringScanner.new('somestring') +scanner.exist?(/(foo)bar(baz)bat(bam)/) +match_values_cleared?(scanner) # => true +``` + +Failed named capture match attempt; +same as unnamed above, except for #named_captures: + +``` +scanner = StringScanner.new('somestring') +scanner.exist?(/(?<x>foo)bar(?<y>baz)bat(?<z>bam)/) +match_values_cleared?(scanner) # => false +scanner.named_captures # => {"x"=>nil, "y"=>nil, "z"=>nil} +``` + +## Fixed-Anchor Property + +Pattern matching in `StringScanner` is the same as in Ruby's, +except for its fixed-anchor property, +which determines the meaning of `'\A'`: + +* `false` (the default): matches the current byte position. + + ``` + scanner = StringScanner.new('foobar') + scanner.scan(/\A./) # => "f" + scanner.scan(/\A./) # => "o" + scanner.scan(/\A./) # => "o" + scanner.scan(/\A./) # => "b" + ``` + +* `true`: matches the beginning of the target substring; + never matches unless the byte position is zero: + + ``` + scanner = StringScanner.new('foobar', fixed_anchor: true) + scanner.scan(/\A./) # => "f" + scanner.scan(/\A./) # => nil + scanner.reset + scanner.scan(/\A./) # => "f" + ``` + +The fixed-anchor property is set when the `StringScanner` object is created, +and may not be modified +(see StringScanner.new); +method #fixed_anchor? returns the setting. + diff --git a/doc/syntax/literals.rdoc b/doc/syntax/literals.rdoc index 0c1e4a434b..6d681419a2 100644 --- a/doc/syntax/literals.rdoc +++ b/doc/syntax/literals.rdoc @@ -138,19 +138,18 @@ Also \Rational numbers may be imaginary numbers. == Strings -=== \String Literals - -The most common way of writing strings is using <tt>"</tt>: - - "This is a string." - -The string may be many lines long. - -Any internal <tt>"</tt> must be escaped: - - "This string has a quote: \". As you can see, it is escaped" - -Double-quote strings allow escaped characters such as <tt>\n</tt> for +=== Escape Sequences + +Some characters can be represented as escape sequences in +double-quoted strings, +character literals, +here document literals (non-quoted, double-quoted, and with backticks), +double-quoted symbols, +double-quoted symbol keys in Hash literals, +Regexp literals, and +several percent literals (<tt>%</tt>, <tt>%Q,</tt> <tt>%W</tt>, <tt>%I</tt>, <tt>%r</tt>, <tt>%x</tt>). + +They allow escape sequences such as <tt>\n</tt> for newline, <tt>\t</tt> for tab, etc. The full list of supported escape sequences are as follows: @@ -174,11 +173,31 @@ sequences are as follows: \M-\cx same as above \c\M-x same as above \c? or \C-? delete, ASCII 7Fh (DEL) + \<newline> continuation line (empty string) + +The last one, <tt>\<newline></tt>, represents an empty string instead of a character. +It is used to fold a line in a string. + +=== Double-quoted \String Literals -Any other character following a backslash is interpreted as the +The most common way of writing strings is using <tt>"</tt>: + + "This is a string." + +The string may be many lines long. + +Any internal <tt>"</tt> must be escaped: + + "This string has a quote: \". As you can see, it is escaped" + +Double-quoted strings allow escape sequences described in +{Escape Sequences}[#label-Escape+Sequences]. + +In a double-quoted string, +any other character following a backslash is interpreted as the character itself. -Double-quote strings allow interpolation of other values using +Double-quoted strings allow interpolation of other values using <tt>#{...}</tt>: "One plus one is two: #{1 + 1}" @@ -190,8 +209,14 @@ You can also use <tt>#@foo</tt>, <tt>#@@foo</tt> and <tt>#$foo</tt> as a shorthand for, respectively, <tt>#{ @foo }</tt>, <tt>#{ @@foo }</tt> and <tt>#{ $foo }</tt>. +See also: + +* {% and %Q: Interpolable String Literals}[#label-25+and+-25Q-3A+Interpolable+String+Literals] + +=== Single-quoted \String Literals + Interpolation may be disabled by escaping the "#" character or using -single-quote strings: +single-quoted strings: '#{1 + 1}' #=> "\#{1 + 1}" @@ -199,6 +224,16 @@ In addition to disabling interpolation, single-quoted strings also disable all escape sequences except for the single-quote (<tt>\'</tt>) and backslash (<tt>\\\\</tt>). +In a single-quoted string, +any other character following a backslash is interpreted as is: +a backslash and the character itself. + +See also: + +* {%q: Non-Interpolable String Literals}[#label-25q-3A+Non-Interpolable+String+Literals] + +=== Literal String Concatenation + Adjacent string literals are automatically concatenated by the interpreter: "con" "cat" "en" "at" "ion" #=> "concatenation" @@ -211,10 +246,12 @@ be concatenated as long as a percent-string is not last. %q{a} 'b' "c" #=> "abc" "a" 'b' %q{c} #=> NameError: uninitialized constant q +=== Character Literal + There is also a character literal notation to represent single character strings, which syntax is a question mark (<tt>?</tt>) -followed by a single character or escape sequence that corresponds to -a single codepoint in the script encoding: +followed by a single character or escape sequence (except continuation line) +that corresponds to a single codepoint in the script encoding: ?a #=> "a" ?abc #=> SyntaxError @@ -228,11 +265,6 @@ a single codepoint in the script encoding: ?\C-\M-a #=> "\x81", same as above ?あ #=> "あ" -See also: - -* {%q: Non-Interpolable String Literals}[#label-25q-3A+Non-Interpolable+String+Literals] -* {% and %Q: Interpolable String Literals}[#label-25+and+-25Q-3A+Interpolable+String+Literals] - === Here Document Literals If you are writing a large block of text you may use a "here document" or @@ -283,9 +315,10 @@ its end is a multiple of eight. The amount to be removed is counted in terms of the number of spaces. If the boundary appears in the middle of a tab, that tab is not removed. -A heredoc allows interpolation and escaped characters. You may disable -interpolation and escaping by surrounding the opening identifier with single -quotes: +A heredoc allows interpolation and the escape sequences described in +{Escape Sequences}[#label-Escape+Sequences]. +You may disable interpolation and the escaping by surrounding the opening +identifier with single quotes: expected_result = <<-'EXPECTED' One plus one is #{1 + 1} @@ -326,12 +359,15 @@ details on what symbols are and when ruby creates them internally. You may reference a symbol using a colon: <tt>:my_symbol</tt>. -You may also create symbols by interpolation: +You may also create symbols by interpolation and escape sequences described in +{Escape Sequences}[#label-Escape+Sequences] with double-quotes: :"my_symbol1" :"my_symbol#{1 + 1}" + :"foo\sbar" -Like strings, a single-quote may be used to disable interpolation: +Like strings, a single-quote may be used to disable interpolation and +escape sequences: :'my_symbol#{1 + 1}' #=> :"my_symbol\#{1 + 1}" @@ -451,7 +487,12 @@ may use these paired delimiters: * <tt>(</tt> and <tt>)</tt>. * <tt>{</tt> and <tt>}</tt>. * <tt><</tt> and <tt>></tt>. -* Any other character, as both beginning and ending delimiters. +* Non-alphanumeric ASCII character except above, as both beginning and ending delimiters. + +The delimiters can be escaped with a backslash. +However, the first four pairs (brackets, parenthesis, braces, and +angle brackets) are allowed without backslash as far as they are correctly +paired. These are demonstrated in the next section. @@ -460,13 +501,20 @@ These are demonstrated in the next section. You can write a non-interpolable string with <tt>%q</tt>. The created string is the same as if you created it with single quotes: - %[foo bar baz] # => "foo bar baz" # Using []. - %(foo bar baz) # => "foo bar baz" # Using (). - %{foo bar baz} # => "foo bar baz" # Using {}. - %<foo bar baz> # => "foo bar baz" # Using <>. - %|foo bar baz| # => "foo bar baz" # Using two |. - %:foo bar baz: # => "foo bar baz" # Using two :. + %q[foo bar baz] # => "foo bar baz" # Using []. + %q(foo bar baz) # => "foo bar baz" # Using (). + %q{foo bar baz} # => "foo bar baz" # Using {}. + %q<foo bar baz> # => "foo bar baz" # Using <>. + %q|foo bar baz| # => "foo bar baz" # Using two |. + %q:foo bar baz: # => "foo bar baz" # Using two :. %q(1 + 1 is #{1 + 1}) # => "1 + 1 is \#{1 + 1}" # No interpolation. + %q[foo[bar]baz] # => "foo[bar]baz" # brackets can be nested. + %q(foo(bar)baz) # => "foo(bar)baz" # parenthesis can be nested. + %q{foo{bar}baz} # => "foo{bar}baz" # braces can be nested. + %q<foo<bar>baz> # => "foo<bar>baz" # angle brackets can be nested. + +This is similar to single-quoted string but only backslashs and +the specified delimiters can be escaped with a backslash. === <tt>% and %Q</tt>: Interpolable String Literals @@ -476,30 +524,63 @@ or with its alias <tt>%</tt>: %[foo bar baz] # => "foo bar baz" %(1 + 1 is #{1 + 1}) # => "1 + 1 is 2" # Interpolation. +This is similar to double-quoted string. +It allow escape sequences described in +{Escape Sequences}[#label-Escape+Sequences]. +Other escaped characters (a backslash followed by a character) are +interpreted as the character. + === <tt>%w and %W</tt>: String-Array Literals -You can write an array of strings with <tt>%w</tt> (non-interpolable) -or <tt>%W</tt> (interpolable): +You can write an array of strings as whitespace-separated words +with <tt>%w</tt> (non-interpolable) or <tt>%W</tt> (interpolable): %w[foo bar baz] # => ["foo", "bar", "baz"] %w[1 % *] # => ["1", "%", "*"] # Use backslash to embed spaces in the strings. %w[foo\ bar baz\ bat] # => ["foo bar", "baz bat"] + %W[foo\ bar baz\ bat] # => ["foo bar", "baz bat"] %w(#{1 + 1}) # => ["\#{1", "+", "1}"] %W(#{1 + 1}) # => ["2"] + # The nested delimiters evaluated to a flat array of strings + # (not nested array). + %w[foo[bar baz]qux] # => ["foo[bar", "baz]qux"] + +The following characters are considered as white spaces to separate words: + +* space, ASCII 20h (SPC) +* form feed, ASCII 0Ch (FF) +* newline (line feed), ASCII 0Ah (LF) +* carriage return, ASCII 0Dh (CR) +* horizontal tab, ASCII 09h (TAB) +* vertical tab, ASCII 0Bh (VT) + +The white space characters can be escaped with a backslash to make them +part of a word. + +<tt>%W</tt> allow escape sequences described in +{Escape Sequences}[#label-Escape+Sequences]. +However the continuation line <tt>\<newline></tt> is not usable because +it is interpreted as the escaped newline described above. + === <tt>%i and %I</tt>: Symbol-Array Literals -You can write an array of symbols with <tt>%i</tt> (non-interpolable) -or <tt>%I</tt> (interpolable): +You can write an array of symbols as whitespace-separated words +with <tt>%i</tt> (non-interpolable) or <tt>%I</tt> (interpolable): %i[foo bar baz] # => [:foo, :bar, :baz] %i[1 % *] # => [:"1", :%, :*] # Use backslash to embed spaces in the symbols. %i[foo\ bar baz\ bat] # => [:"foo bar", :"baz bat"] + %I[foo\ bar baz\ bat] # => [:"foo bar", :"baz bat"] %i(#{1 + 1}) # => [:"\#{1", :+, :"1}"] %I(#{1 + 1}) # => [:"2"] +The white space characters and its escapes are interpreted as the same as +string-array literals described in +{%w and %W: String-Array Literals}[#label-25w+and+-25W-3A+String-Array+Literals]. + === <tt>%s</tt>: Symbol Literals You can write a symbol with <tt>%s</tt>: @@ -507,6 +588,10 @@ You can write a symbol with <tt>%s</tt>: %s[foo] # => :foo %s[foo bar] # => :"foo bar" +This is non-interpolable. +No interpolation allowed. +Only backslashs and the specified delimiters can be escaped with a backslash. + === <tt>%r</tt>: Regexp Literals You can write a regular expression with <tt>%r</tt>; @@ -531,4 +616,10 @@ See {Regexp modes}[rdoc-ref:Regexp@Modes] for details. You can write and execute a shell command with <tt>%x</tt>: - %x(echo 1) # => "1\n" + %x(echo 1) # => "1\n" + %x[echo #{1 + 2}] # => "3\n" + %x[echo \u0030] # => "0\n" + +This is interpolable. +<tt>%x</tt> allow escape sequences described in +{Escape Sequences}[#label-Escape+Sequences]. diff --git a/doc/syntax/pattern_matching.rdoc b/doc/syntax/pattern_matching.rdoc index e49c09a1f8..6a30380f46 100644 --- a/doc/syntax/pattern_matching.rdoc +++ b/doc/syntax/pattern_matching.rdoc @@ -422,7 +422,8 @@ These core and library classes implement deconstruction: == Guard clauses -+if+ can be used to attach an additional condition (guard clause) when the pattern matches. This condition may use bound variables: ++if+ can be used to attach an additional condition (guard clause) when the pattern matches in +case+/+in+ expressions. +This condition may use bound variables: case [1, 2] in a, b if b == a*2 @@ -450,6 +451,11 @@ These core and library classes implement deconstruction: end #=> "matched" +Note that <code>=></code> and +in+ operator can not have a guard clause. +The following examples is parsed as a standalone expression with modifier +if+. + + [1, 2] in a, b if b == a*2 + == Appendix A. Pattern syntax Approximate syntax is: diff --git a/enc/Makefile.in b/enc/Makefile.in index 6920bc9520..ce93fdd22d 100644 --- a/enc/Makefile.in +++ b/enc/Makefile.in @@ -52,7 +52,7 @@ optflags = @optflags@ debugflags = @debugflags@ warnflags = @warnflags@ CCDLFLAGS = @CCDLFLAGS@ -INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(top_srcdir) +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(top_srcdir) @incflags@ DEFS = @DEFS@ CPPFLAGS = @CPPFLAGS@ -DONIG_ENC_REGISTER=rb_enc_register LDFLAGS = @LDFLAGS@ @@ -386,18 +386,28 @@ warn_vsprintf(rb_encoding *enc, const char *file, int line, const char *fmt, va_ return rb_str_cat2(str, "\n"); } -#define with_warn_vsprintf(file, line, fmt) \ +#define with_warn_vsprintf(enc, file, line, fmt) \ VALUE str; \ va_list args; \ va_start(args, fmt); \ - str = warn_vsprintf(NULL, file, line, fmt, args); \ + str = warn_vsprintf(enc, file, line, fmt, args); \ va_end(args); void rb_compile_warn(const char *file, int line, const char *fmt, ...) { if (!NIL_P(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { + rb_write_warning_str(str); + } + } +} + +void +rb_enc_compile_warn(rb_encoding *enc, const char *file, int line, const char *fmt, ...) +{ + if (!NIL_P(ruby_verbose)) { + with_warn_vsprintf(enc, file, line, fmt) { rb_write_warning_str(str); } } @@ -408,7 +418,18 @@ void rb_compile_warning(const char *file, int line, const char *fmt, ...) { if (RTEST(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { + rb_write_warning_str(str); + } + } +} + +/* rb_enc_compile_warning() reports only in verbose mode */ +void +rb_enc_compile_warning(rb_encoding *enc, const char *file, int line, const char *fmt, ...) +{ + if (RTEST(ruby_verbose)) { + with_warn_vsprintf(enc, file, line, fmt) { rb_write_warning_str(str); } } @@ -418,7 +439,7 @@ void rb_category_compile_warn(rb_warning_category_t category, const char *file, int line, const char *fmt, ...) { if (!NIL_P(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { rb_warn_category(str, rb_warning_category_to_name(category)); } } @@ -2698,8 +2719,18 @@ syntax_error_with_path(VALUE exc, VALUE path, VALUE *mesg, rb_encoding *enc) rb_ivar_set(exc, id_i_path, path); } else { - if (rb_attr_get(exc, id_i_path) != path) { - rb_raise(rb_eArgError, "SyntaxError#path changed"); + VALUE old_path = rb_attr_get(exc, id_i_path); + if (old_path != path) { + if (rb_str_equal(path, old_path)) { + rb_raise(rb_eArgError, "SyntaxError#path changed: %+"PRIsVALUE" (%p->%p)", + old_path, (void *)old_path, (void *)path); + } + else { + rb_raise(rb_eArgError, "SyntaxError#path changed: %+"PRIsVALUE"(%s%s)->%+"PRIsVALUE"(%s)", + old_path, rb_enc_name(rb_enc_get(old_path)), + (FL_TEST(old_path, RSTRING_FSTR) ? ":FSTR" : ""), + path, rb_enc_name(rb_enc_get(path))); + } } VALUE s = *mesg = rb_attr_get(exc, idMesg); if (RSTRING_LEN(s) > 0 && *(RSTRING_END(s)-1) != '\n') @@ -2710,32 +2741,47 @@ syntax_error_with_path(VALUE exc, VALUE path, VALUE *mesg, rb_encoding *enc) /* * Document-module: Errno + + * When an operating system encounters an error, + * it typically reports the error as an integer error code: + * + * $ ls nosuch.txt + * ls: cannot access 'nosuch.txt': No such file or directory + * $ echo $? # Code for last error. + * 2 + * + * When the Ruby interpreter interacts with the operating system + * and receives such an error code (e.g., +2+), + * it maps the code to a particular Ruby exception class (e.g., +Errno::ENOENT+): + * + * File.open('nosuch.txt') + * # => No such file or directory @ rb_sysopen - nosuch.txt (Errno::ENOENT) + * + * Each such class is: * - * Ruby exception objects are subclasses of Exception. However, - * operating systems typically report errors using plain - * integers. Module Errno is created dynamically to map these - * operating system errors to Ruby classes, with each error number - * generating its own subclass of SystemCallError. As the subclass - * is created in module Errno, its name will start - * <code>Errno::</code>. + * - A nested class in this module, +Errno+. + * - A subclass of class SystemCallError. + * - Associated with an error code. * - * The names of the <code>Errno::</code> classes depend on the - * environment in which Ruby runs. On a typical Unix or Windows - * platform, there are Errno classes such as Errno::EACCES, - * Errno::EAGAIN, Errno::EINTR, and so on. + * Thus: * - * The integer operating system error number corresponding to a - * particular error is available as the class constant - * <code>Errno::</code><em>error</em><code>::Errno</code>. + * Errno::ENOENT.superclass # => SystemCallError + * Errno::ENOENT::Errno # => 2 * - * Errno::EACCES::Errno #=> 13 - * Errno::EAGAIN::Errno #=> 11 - * Errno::EINTR::Errno #=> 4 + * The names of nested classes are returned by method +Errno.constants+: * - * The full list of operating system errors on your particular platform - * are available as the constants of Errno. + * Errno.constants.size # => 158 + * Errno.constants.sort.take(5) # => [:E2BIG, :EACCES, :EADDRINUSE, :EADDRNOTAVAIL, :EADV] + * + * As seen above, the error code associated with each class + * is available as the value of a constant; + * the value for a particular class may vary among operating systems. + * If the class is not needed for the particular operating system, + * the value is zero: + * + * Errno::ENOENT::Errno # => 2 + * Errno::ENOTCAPABLE::Errno # => 0 * - * Errno.constants #=> :E2BIG, :EACCES, :EADDRINUSE, :EADDRNOTAVAIL, ... */ static st_table *syserr_tbl; @@ -3861,11 +3907,6 @@ rb_error_frozen_object(VALUE frozen_obj) { rb_yjit_lazy_push_frame(GET_EC()->cfp->pc); - if (CHILLED_STRING_P(frozen_obj)) { - CHILLED_STRING_MUTATED(frozen_obj); - return; - } - VALUE debug_info; const ID created_info = id_debug_created_info; VALUE mesg = rb_sprintf("can't modify frozen %"PRIsVALUE": ", @@ -3888,14 +3929,20 @@ rb_error_frozen_object(VALUE frozen_obj) void rb_check_frozen(VALUE obj) { - rb_check_frozen_internal(obj); + if (RB_UNLIKELY(RB_OBJ_FROZEN(obj))) { + rb_error_frozen_object(obj); + } + + if (RB_UNLIKELY(CHILLED_STRING_P(obj))) { + rb_str_modify(obj); + } } void rb_check_copyable(VALUE obj, VALUE orig) { if (!FL_ABLE(obj)) return; - rb_check_frozen_internal(obj); + rb_check_frozen(obj); if (!FL_ABLE(orig)) return; } diff --git a/ext/-test-/integer/my_integer.c b/ext/-test-/integer/my_integer.c index d86474bd7d..94f14d2765 100644 --- a/ext/-test-/integer/my_integer.c +++ b/ext/-test-/integer/my_integer.c @@ -1,9 +1,13 @@ #include "ruby.h" +static const rb_data_type_t my_integer_type = { + "MyInteger", {0}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY +}; + static VALUE my_integer_s_new(VALUE klass) { - return Data_Wrap_Struct(klass, 0, 0, 0); + return TypedData_Wrap_Struct(klass, &my_integer_type, 0); } void diff --git a/ext/-test-/load/resolve_symbol_target/resolve_symbol_target.def b/ext/-test-/load/resolve_symbol_target/resolve_symbol_target.def deleted file mode 100644 index c2ed3610fe..0000000000 --- a/ext/-test-/load/resolve_symbol_target/resolve_symbol_target.def +++ /dev/null @@ -1,4 +0,0 @@ -LIBRARY resolve_symbol_target -EXPORTS - Init_resolve_symbol_target - rst_any_method diff --git a/ext/-test-/load/stringify_target/stringify_target.def b/ext/-test-/load/stringify_target/stringify_target.def deleted file mode 100644 index 89c2b762de..0000000000 --- a/ext/-test-/load/stringify_target/stringify_target.def +++ /dev/null @@ -1,4 +0,0 @@ -LIBRARY stringify_target -EXPORTS - Init_stringify_target - stt_any_method diff --git a/ext/-test-/public_header_warnings/extconf.rb b/ext/-test-/public_header_warnings/extconf.rb new file mode 100644 index 0000000000..f6a8a51f63 --- /dev/null +++ b/ext/-test-/public_header_warnings/extconf.rb @@ -0,0 +1,23 @@ +# +# Under compilers with WERRORFLAG, MakeMakefile.try_compile treats warnings as errors, so we can +# use append_cflags to test whether the public header files emit warnings with certain flags turned +# on. +# +def check_append_cflags(flag, msg = nil) + msg ||= "flag #{flag} is not acceptable" + !$CFLAGS.include?(flag) or raise("flag #{flag} already present in $CFLAGS") + append_cflags(flag) + $CFLAGS.include?(flag) or raise(msg) +end + +if %w[gcc clang].include?(RbConfig::CONFIG['CC']) + config_string("WERRORFLAG") or raise("expected WERRORFLAG to be defined") + + # should be acceptable on all modern C compilers + check_append_cflags("-D_TEST_OK", "baseline compiler warning test failed") + + # Feature #20507: Allow compilation of C extensions with -Wconversion + check_append_cflags("-Wconversion", "-Wconversion raising warnings in public headers") +end + +create_makefile("-test-/public_header_warnings") diff --git a/ext/-test-/string/chilled.c b/ext/-test-/string/chilled.c deleted file mode 100644 index c98fc72c47..0000000000 --- a/ext/-test-/string/chilled.c +++ /dev/null @@ -1,13 +0,0 @@ -#include "ruby.h" - -static VALUE -bug_s_rb_str_chilled_p(VALUE self, VALUE str) -{ - return rb_str_chilled_p(str) ? Qtrue : Qfalse; -} - -void -Init_string_chilled(VALUE klass) -{ - rb_define_singleton_method(klass, "rb_str_chilled_p", bug_s_rb_str_chilled_p, 1); -} diff --git a/ext/-test-/thread/id/extconf.rb b/ext/-test-/thread/id/extconf.rb new file mode 100644 index 0000000000..a0ae0eff15 --- /dev/null +++ b/ext/-test-/thread/id/extconf.rb @@ -0,0 +1,3 @@ +if have_func("gettid") + create_makefile("-test-/thread/id") +end diff --git a/ext/-test-/thread/id/id.c b/ext/-test-/thread/id/id.c new file mode 100644 index 0000000000..b46a5955e2 --- /dev/null +++ b/ext/-test-/thread/id/id.c @@ -0,0 +1,15 @@ +#include <ruby.h> + +static VALUE +bug_gettid(VALUE self) +{ + pid_t tid = gettid(); + return PIDT2NUM(tid); +} + +void +Init_id(void) +{ + VALUE klass = rb_define_module_under(rb_define_module("Bug"), "ThreadID"); + rb_define_module_function(klass, "gettid", bug_gettid, 0); +} diff --git a/ext/date/date_core.c b/ext/date/date_core.c index 0fcefd671b..ca9449f0e9 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -6326,9 +6326,11 @@ minus_dd(VALUE self, VALUE other) * call-seq: * d - other -> date or rational * - * Returns the difference between the two dates if the other is a date - * object. If the other is a numeric value, returns a date object - * pointing +other+ days before self. If the other is a fractional number, + * If the other is a date object, returns a Rational + * whose value is the difference between the two dates in days. + * If the other is a numeric value, returns a date object + * pointing +other+ days before self. + * If the other is a fractional number, * assumes its precision is at most nanosecond. * * Date.new(2001,2,3) - 1 #=> #<Date: 2001-02-02 ...> @@ -8955,18 +8957,22 @@ time_to_datetime(VALUE self) static VALUE date_to_time(VALUE self) { + VALUE t; + get_d1a(self); if (m_julian_p(adat)) { - VALUE tmp = d_lite_gregorian(self); - get_d1b(tmp); + self = d_lite_gregorian(self); + get_d1b(self); adat = bdat; } - return f_local3(rb_cTime, + t = f_local3(rb_cTime, m_real_year(adat), INT2FIX(m_mon(adat)), INT2FIX(m_mday(adat))); + RB_GC_GUARD(self); /* may be the converted gregorian */ + return t; } /* @@ -9055,6 +9061,7 @@ datetime_to_time(VALUE self) f_add(INT2FIX(m_sec(dat)), m_sf_in_sec(dat)), INT2FIX(m_of(dat))); + RB_GC_GUARD(self); /* may be the converted gregorian */ return t; } } diff --git a/ext/digest/digest.c b/ext/digest/digest.c index 68837a674c..bd8d3e815f 100644 --- a/ext/digest/digest.c +++ b/ext/digest/digest.c @@ -534,9 +534,39 @@ rb_digest_class_init(VALUE self) * * * rb_ivar_set(cDigest_SHA1, rb_intern("metadata"), - * Data_Wrap_Struct(0, 0, 0, (void *)&sha1)); + * rb_digest_make_metadata(&sha1)); */ +#ifdef DIGEST_USE_RB_EXT_RESOLVE_SYMBOL +static const rb_data_type_t metadata_type = { + "digest/metadata", + {0}, +}; + +RUBY_FUNC_EXPORTED VALUE +rb_digest_wrap_metadata(const rb_digest_metadata_t *meta) +{ + return rb_obj_freeze(TypedData_Wrap_Struct(0, &metadata_type, (void *)meta)); +} +#endif + +static rb_digest_metadata_t * +get_metadata_ptr(VALUE obj) +{ + rb_digest_metadata_t *algo; + +#ifdef DIGEST_USE_RB_EXT_RESOLVE_SYMBOL + if (!rb_typeddata_is_kind_of(obj, &metadata_type)) return 0; + algo = RTYPEDDATA_DATA(obj); +#else +# undef RUBY_UNTYPED_DATA_WARNING +# define RUBY_UNTYPED_DATA_WARNING 0 + Data_Get_Struct(obj, rb_digest_metadata_t, algo); +#endif + + return algo; +} + static rb_digest_metadata_t * get_digest_base_metadata(VALUE klass) { @@ -554,8 +584,8 @@ get_digest_base_metadata(VALUE klass) if (NIL_P(p)) rb_raise(rb_eRuntimeError, "Digest::Base cannot be directly inherited in Ruby"); - if (!RB_TYPE_P(obj, T_DATA) || RTYPEDDATA_P(obj)) { - wrong: + algo = get_metadata_ptr(obj); + if (!algo) { if (p == klass) rb_raise(rb_eTypeError, "%"PRIsVALUE"::metadata is not initialized properly", klass); @@ -564,12 +594,6 @@ get_digest_base_metadata(VALUE klass) klass, p); } -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - Data_Get_Struct(obj, rb_digest_metadata_t, algo); - - if (!algo) goto wrong; - switch (algo->api_version) { case 3: break; diff --git a/ext/digest/digest.h b/ext/digest/digest.h index 68a3da5dd2..4503929bab 100644 --- a/ext/digest/digest.h +++ b/ext/digest/digest.h @@ -64,10 +64,29 @@ rb_id_metadata(void) return rb_intern_const("metadata"); } +#if !defined(HAVE_RB_EXT_RESOLVE_SYMBOL) +#elif !defined(RUBY_UNTYPED_DATA_WARNING) +# error RUBY_UNTYPED_DATA_WARNING is not defined +#elif RUBY_UNTYPED_DATA_WARNING +/* rb_ext_resolve_symbol() has been defined since Ruby 3.3, but digest + * bundled with 3.3 didn't use it. */ +# define DIGEST_USE_RB_EXT_RESOLVE_SYMBOL 1 +#endif + static inline VALUE rb_digest_make_metadata(const rb_digest_metadata_t *meta) { +#ifdef DIGEST_USE_RB_EXT_RESOLVE_SYMBOL + typedef VALUE (*wrapper_func_type)(const rb_digest_metadata_t *meta); + static wrapper_func_type wrapper; + if (!wrapper) { + wrapper = (wrapper_func_type)rb_ext_resolve_symbol("digest.so", "rb_digest_wrap_metadata"); + if (!wrapper) rb_raise(rb_eLoadError, "rb_digest_wrap_metadata not found"); + } + return wrapper(meta); +#else #undef RUBY_UNTYPED_DATA_WARNING #define RUBY_UNTYPED_DATA_WARNING 0 return rb_obj_freeze(Data_Wrap_Struct(0, 0, 0, (void *)meta)); +#endif } diff --git a/ext/json/generator/extconf.rb b/ext/json/generator/extconf.rb index 8627c5f4bd..cf8d5f2bda 100644 --- a/ext/json/generator/extconf.rb +++ b/ext/json/generator/extconf.rb @@ -1,4 +1,9 @@ require 'mkmf' -$defs << "-DJSON_GENERATOR" -create_makefile 'json/ext/generator' +if RUBY_ENGINE == 'truffleruby' + # The pure-Ruby generator is faster on TruffleRuby, so skip compiling the generator extension + File.write('Makefile', dummy_makefile("").join) +else + $defs << "-DJSON_GENERATOR" + create_makefile 'json/ext/generator' +end diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c index 6d78284bc4..e968619205 100644 --- a/ext/json/generator/generator.c +++ b/ext/json/generator/generator.c @@ -636,16 +636,12 @@ static size_t State_memsize(const void *ptr) # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif -#ifdef NEW_TYPEDDATA_WRAPPER static const rb_data_type_t JSON_Generator_State_type = { "JSON/Generator/State", {NULL, State_free, State_memsize,}, -#ifdef RUBY_TYPED_FREE_IMMEDIATELY 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE, -#endif + RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE, }; -#endif static VALUE cState_s_allocate(VALUE klass) { diff --git a/ext/json/generator/generator.h b/ext/json/generator/generator.h index 1a736b84dd..98b7d833f3 100644 --- a/ext/json/generator/generator.h +++ b/ext/json/generator/generator.h @@ -166,12 +166,7 @@ static inline void *ruby_zalloc(size_t n) return p; } #endif -#ifdef TypedData_Make_Struct + static const rb_data_type_t JSON_Generator_State_type; -#define NEW_TYPEDDATA_WRAPPER 1 -#else -#define TypedData_Make_Struct(klass, type, ignore, json) Data_Make_Struct(klass, type, NULL, State_free, json) -#define TypedData_Get_Struct(self, JSON_Generator_State, ignore, json) Data_Get_Struct(self, JSON_Generator_State, json) -#endif #endif diff --git a/ext/json/lib/json/ext.rb b/ext/json/lib/json/ext.rb index 7264a857fa..b62e231712 100644 --- a/ext/json/lib/json/ext.rb +++ b/ext/json/lib/json/ext.rb @@ -4,11 +4,19 @@ module JSON # This module holds all the modules/classes that implement JSON's # functionality as C extensions. module Ext - require 'json/ext/parser' - require 'json/ext/generator' - $DEBUG and warn "Using Ext extension for JSON." - JSON.parser = Parser - JSON.generator = Generator + if RUBY_ENGINE == 'truffleruby' + require 'json/ext/parser' + require 'json/pure' + $DEBUG and warn "Using Ext extension for JSON parser and Pure library for JSON generator." + JSON.parser = Parser + JSON.generator = JSON::Pure::Generator + else + require 'json/ext/parser' + require 'json/ext/generator' + $DEBUG and warn "Using Ext extension for JSON." + JSON.parser = Parser + JSON.generator = Generator + end end JSON_LOADED = true unless defined?(::JSON::JSON_LOADED) diff --git a/ext/json/parser/parser.c b/ext/json/parser/parser.c index 57f87432b6..7e44d469f8 100644 --- a/ext/json/parser/parser.c +++ b/ext/json/parser/parser.c @@ -2097,12 +2097,12 @@ case 9: static void JSON_mark(void *ptr) { JSON_Parser *json = ptr; - rb_gc_mark_maybe(json->Vsource); - rb_gc_mark_maybe(json->create_id); - rb_gc_mark_maybe(json->object_class); - rb_gc_mark_maybe(json->array_class); - rb_gc_mark_maybe(json->decimal_class); - rb_gc_mark_maybe(json->match_string); + rb_gc_mark(json->Vsource); + rb_gc_mark(json->create_id); + rb_gc_mark(json->object_class); + rb_gc_mark(json->array_class); + rb_gc_mark(json->decimal_class); + rb_gc_mark(json->match_string); } static void JSON_free(void *ptr) @@ -2118,16 +2118,12 @@ static size_t JSON_memsize(const void *ptr) return sizeof(*json) + FBUFFER_CAPA(json->fbuffer); } -#ifdef NEW_TYPEDDATA_WRAPPER static const rb_data_type_t JSON_Parser_type = { "JSON/Parser", {JSON_mark, JSON_free, JSON_memsize,}, -#ifdef RUBY_TYPED_FREE_IMMEDIATELY 0, 0, RUBY_TYPED_FREE_IMMEDIATELY, -#endif }; -#endif static VALUE cJSON_parser_s_allocate(VALUE klass) { diff --git a/ext/json/parser/parser.h b/ext/json/parser/parser.h index 92ed3fdc5d..651d40c074 100644 --- a/ext/json/parser/parser.h +++ b/ext/json/parser/parser.h @@ -85,12 +85,7 @@ static inline void *ruby_zalloc(size_t n) return p; } #endif -#ifdef TypedData_Make_Struct + static const rb_data_type_t JSON_Parser_type; -#define NEW_TYPEDDATA_WRAPPER 1 -#else -#define TypedData_Make_Struct(klass, type, ignore, json) Data_Make_Struct(klass, type, NULL, JSON_free, json) -#define TypedData_Get_Struct(self, JSON_Parser, ignore, json) Data_Get_Struct(self, JSON_Parser, json) -#endif #endif diff --git a/ext/json/parser/parser.rl b/ext/json/parser/parser.rl index af190e7500..e2522918dc 100644 --- a/ext/json/parser/parser.rl +++ b/ext/json/parser/parser.rl @@ -857,12 +857,12 @@ static VALUE cParser_parse(VALUE self) static void JSON_mark(void *ptr) { JSON_Parser *json = ptr; - rb_gc_mark_maybe(json->Vsource); - rb_gc_mark_maybe(json->create_id); - rb_gc_mark_maybe(json->object_class); - rb_gc_mark_maybe(json->array_class); - rb_gc_mark_maybe(json->decimal_class); - rb_gc_mark_maybe(json->match_string); + rb_gc_mark(json->Vsource); + rb_gc_mark(json->create_id); + rb_gc_mark(json->object_class); + rb_gc_mark(json->array_class); + rb_gc_mark(json->decimal_class); + rb_gc_mark(json->match_string); } static void JSON_free(void *ptr) @@ -878,16 +878,12 @@ static size_t JSON_memsize(const void *ptr) return sizeof(*json) + FBUFFER_CAPA(json->fbuffer); } -#ifdef NEW_TYPEDDATA_WRAPPER static const rb_data_type_t JSON_Parser_type = { "JSON/Parser", {JSON_mark, JSON_free, JSON_memsize,}, -#ifdef RUBY_TYPED_FREE_IMMEDIATELY 0, 0, RUBY_TYPED_FREE_IMMEDIATELY, -#endif }; -#endif static VALUE cJSON_parser_s_allocate(VALUE klass) { diff --git a/ext/openssl/extconf.rb b/ext/openssl/extconf.rb index dd3732d0a8..8d2eac0262 100644 --- a/ext/openssl/extconf.rb +++ b/ext/openssl/extconf.rb @@ -8,7 +8,7 @@ = Licence This program is licensed under the same licence as Ruby. - (See the file 'LICENCE'.) + (See the file 'COPYING'.) =end require "mkmf" @@ -149,6 +149,9 @@ engines.each { |name| have_func("ENGINE_load_#{name}()", "openssl/engine.h") } +# missing in libressl < 3.5 +have_func("i2d_re_X509_tbs(NULL, NULL)", x509_h) + # added in 1.1.0 if !have_struct_member("SSL", "ctx", "openssl/ssl.h") || is_libressl $defs.push("-DHAVE_OPAQUE_OPENSSL") diff --git a/ext/openssl/lib/openssl.rb b/ext/openssl/lib/openssl.rb index 8a342f15b6..f5ca956d07 100644 --- a/ext/openssl/lib/openssl.rb +++ b/ext/openssl/lib/openssl.rb @@ -7,7 +7,7 @@ = Licence This program is licensed under the same licence as Ruby. - (See the file 'LICENCE'.) + (See the file 'COPYING'.) =end require 'openssl.so' diff --git a/ext/openssl/lib/openssl/bn.rb b/ext/openssl/lib/openssl/bn.rb index 0a5e11b4c2..e4889a140c 100644 --- a/ext/openssl/lib/openssl/bn.rb +++ b/ext/openssl/lib/openssl/bn.rb @@ -10,7 +10,7 @@ # # = Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ module OpenSSL diff --git a/ext/openssl/lib/openssl/buffering.rb b/ext/openssl/lib/openssl/buffering.rb index 216c51e3be..d0b4b18038 100644 --- a/ext/openssl/lib/openssl/buffering.rb +++ b/ext/openssl/lib/openssl/buffering.rb @@ -8,7 +8,7 @@ # #= Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ ## diff --git a/ext/openssl/lib/openssl/cipher.rb b/ext/openssl/lib/openssl/cipher.rb index 8ad8c35dd3..ab75ac8e1a 100644 --- a/ext/openssl/lib/openssl/cipher.rb +++ b/ext/openssl/lib/openssl/cipher.rb @@ -9,7 +9,7 @@ # # = Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ module OpenSSL diff --git a/ext/openssl/lib/openssl/digest.rb b/ext/openssl/lib/openssl/digest.rb index 0f35ddadd3..5cda1e931c 100644 --- a/ext/openssl/lib/openssl/digest.rb +++ b/ext/openssl/lib/openssl/digest.rb @@ -9,7 +9,7 @@ # # = Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ module OpenSSL diff --git a/ext/openssl/lib/openssl/marshal.rb b/ext/openssl/lib/openssl/marshal.rb index af5647192a..eb8eda4748 100644 --- a/ext/openssl/lib/openssl/marshal.rb +++ b/ext/openssl/lib/openssl/marshal.rb @@ -9,7 +9,7 @@ # # = Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ module OpenSSL module Marshal diff --git a/ext/openssl/lib/openssl/ssl.rb b/ext/openssl/lib/openssl/ssl.rb index d28bf1a374..2186f5f43a 100644 --- a/ext/openssl/lib/openssl/ssl.rb +++ b/ext/openssl/lib/openssl/ssl.rb @@ -7,7 +7,7 @@ = Licence This program is licensed under the same licence as Ruby. - (See the file 'LICENCE'.) + (See the file 'COPYING'.) =end require "openssl/buffering" diff --git a/ext/openssl/lib/openssl/x509.rb b/ext/openssl/lib/openssl/x509.rb index f973f4f4dc..b66727420e 100644 --- a/ext/openssl/lib/openssl/x509.rb +++ b/ext/openssl/lib/openssl/x509.rb @@ -9,7 +9,7 @@ # # = Licence # This program is licensed under the same licence as Ruby. -# (See the file 'LICENCE'.) +# (See the file 'COPYING'.) #++ require_relative 'marshal' diff --git a/ext/openssl/openssl.gemspec b/ext/openssl/openssl.gemspec index 2765f55401..e692e661c4 100644 --- a/ext/openssl/openssl.gemspec +++ b/ext/openssl/openssl.gemspec @@ -6,14 +6,14 @@ Gem::Specification.new do |spec| spec.summary = %q{SSL/TLS and general-purpose cryptography for Ruby} spec.description = %q{OpenSSL for Ruby provides access to SSL/TLS and general-purpose cryptography based on the OpenSSL library.} spec.homepage = "https://github.com/ruby/openssl" - spec.license = "Ruby" + spec.licenses = ["Ruby", "BSD-2-Clause"] if Gem::Platform === spec.platform and spec.platform =~ 'java' or RUBY_ENGINE == 'jruby' spec.platform = "java" spec.files = [] spec.add_runtime_dependency('jruby-openssl', '~> 0.14') else - spec.files = Dir["lib/**/*.rb", "ext/**/*.{c,h,rb}", "*.md", "BSDL", "LICENSE.txt"] + spec.files = Dir["lib/**/*.rb", "ext/**/*.{c,h,rb}", "*.md", "BSDL", "COPYING"] spec.require_paths = ["lib"] spec.extensions = ["ext/openssl/extconf.rb"] end diff --git a/ext/openssl/openssl_missing.c b/ext/openssl/openssl_missing.c index 4415703db4..5a6d23e106 100644 --- a/ext/openssl/openssl_missing.c +++ b/ext/openssl/openssl_missing.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include RUBY_EXTCONF_H diff --git a/ext/openssl/openssl_missing.h b/ext/openssl/openssl_missing.h index 8629bfe505..0711f924e5 100644 --- a/ext/openssl/openssl_missing.h +++ b/ext/openssl/openssl_missing.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_OPENSSL_MISSING_H_) #define _OSSL_OPENSSL_MISSING_H_ diff --git a/ext/openssl/ossl.c b/ext/openssl/ossl.c index 00eded55cb..59ad7d19a4 100644 --- a/ext/openssl/ossl.c +++ b/ext/openssl/ossl.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" #include <stdarg.h> /* for ossl_raise */ diff --git a/ext/openssl/ossl.h b/ext/openssl/ossl.h index 68d42b71e2..c3140ac3ef 100644 --- a/ext/openssl/ossl.h +++ b/ext/openssl/ossl.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_H_) #define _OSSL_H_ diff --git a/ext/openssl/ossl_asn1.c b/ext/openssl/ossl_asn1.c index 0533342077..fb47684347 100644 --- a/ext/openssl/ossl_asn1.c +++ b/ext/openssl/ossl_asn1.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_asn1.h b/ext/openssl/ossl_asn1.h index 939a96ce74..f47e353948 100644 --- a/ext/openssl/ossl_asn1.h +++ b/ext/openssl/ossl_asn1.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_ASN1_H_) #define _OSSL_ASN1_H_ diff --git a/ext/openssl/ossl_bio.c b/ext/openssl/ossl_bio.c index 42833d901a..2ef2080507 100644 --- a/ext/openssl/ossl_bio.c +++ b/ext/openssl/ossl_bio.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_bio.h b/ext/openssl/ossl_bio.h index da68c5e5a2..1b871f1cd7 100644 --- a/ext/openssl/ossl_bio.h +++ b/ext/openssl/ossl_bio.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_BIO_H_) #define _OSSL_BIO_H_ diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c index ce0d3ec7ee..7393fdea56 100644 --- a/ext/openssl/ossl_bn.c +++ b/ext/openssl/ossl_bn.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ /* modified by Michal Rokos <m.rokos@sh.cvut.cz> */ #include "ossl.h" diff --git a/ext/openssl/ossl_bn.h b/ext/openssl/ossl_bn.h index 1cc041fc22..800f84cb1e 100644 --- a/ext/openssl/ossl_bn.h +++ b/ext/openssl/ossl_bn.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_BN_H_) #define _OSSL_BN_H_ diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c index 6f74c925c0..cc0114f579 100644 --- a/ext/openssl/ossl_cipher.c +++ b/ext/openssl/ossl_cipher.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_cipher.h b/ext/openssl/ossl_cipher.h index 2392d41c6a..07b50c3bd5 100644 --- a/ext/openssl/ossl_cipher.h +++ b/ext/openssl/ossl_cipher.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_CIPHER_H_) #define _OSSL_CIPHER_H_ diff --git a/ext/openssl/ossl_config.c b/ext/openssl/ossl_config.c index 0e598b4d51..55875028b2 100644 --- a/ext/openssl/ossl_config.c +++ b/ext/openssl/ossl_config.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_config.h b/ext/openssl/ossl_config.h index 4e604f1aed..a254360c2c 100644 --- a/ext/openssl/ossl_config.h +++ b/ext/openssl/ossl_config.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #ifndef OSSL_CONFIG_H #define OSSL_CONFIG_H diff --git a/ext/openssl/ossl_digest.c b/ext/openssl/ossl_digest.c index 1ae26a2355..00ec8931ab 100644 --- a/ext/openssl/ossl_digest.c +++ b/ext/openssl/ossl_digest.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_digest.h b/ext/openssl/ossl_digest.h index 50bf5666a3..99771b8ae1 100644 --- a/ext/openssl/ossl_digest.h +++ b/ext/openssl/ossl_digest.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_DIGEST_H_) #define _OSSL_DIGEST_H_ diff --git a/ext/openssl/ossl_engine.c b/ext/openssl/ossl_engine.c index 9e86321d06..294d58adef 100644 --- a/ext/openssl/ossl_engine.c +++ b/ext/openssl/ossl_engine.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_engine.h b/ext/openssl/ossl_engine.h index cd548beea3..f6f4ff4c1f 100644 --- a/ext/openssl/ossl_engine.h +++ b/ext/openssl/ossl_engine.h @@ -6,7 +6,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(OSSL_ENGINE_H) #define OSSL_ENGINE_H diff --git a/ext/openssl/ossl_hmac.c b/ext/openssl/ossl_hmac.c index c485ba7e67..b1163f6127 100644 --- a/ext/openssl/ossl_hmac.c +++ b/ext/openssl/ossl_hmac.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_hmac.h b/ext/openssl/ossl_hmac.h index 7c51f4722d..17427587b2 100644 --- a/ext/openssl/ossl_hmac.h +++ b/ext/openssl/ossl_hmac.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_HMAC_H_) #define _OSSL_HMAC_H_ diff --git a/ext/openssl/ossl_ns_spki.c b/ext/openssl/ossl_ns_spki.c index 9d70b5d87a..e822d5e0a9 100644 --- a/ext/openssl/ossl_ns_spki.c +++ b/ext/openssl/ossl_ns_spki.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_ns_spki.h b/ext/openssl/ossl_ns_spki.h index 62ba8cb163..20d6857682 100644 --- a/ext/openssl/ossl_ns_spki.h +++ b/ext/openssl/ossl_ns_spki.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_NS_SPKI_H_) #define _OSSL_NS_SPKI_H_ diff --git a/ext/openssl/ossl_ocsp.c b/ext/openssl/ossl_ocsp.c index df986bb3ee..9796d44a26 100644 --- a/ext/openssl/ossl_ocsp.c +++ b/ext/openssl/ossl_ocsp.c @@ -6,7 +6,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_ocsp.h b/ext/openssl/ossl_ocsp.h index 6d2aac8657..07da7d1684 100644 --- a/ext/openssl/ossl_ocsp.h +++ b/ext/openssl/ossl_ocsp.h @@ -6,7 +6,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_OCSP_H_) #define _OSSL_OCSP_H_ diff --git a/ext/openssl/ossl_pkcs12.c b/ext/openssl/ossl_pkcs12.c index 164b2da465..1fcb1a88d3 100644 --- a/ext/openssl/ossl_pkcs12.c +++ b/ext/openssl/ossl_pkcs12.c @@ -1,6 +1,6 @@ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" @@ -134,6 +134,10 @@ ossl_pkcs12_s_create(int argc, VALUE *argv, VALUE self) if (!NIL_P(keytype)) ktype = NUM2INT(keytype); + if (ktype != 0 && ktype != KEY_SIG && ktype != KEY_EX) { + ossl_raise(rb_eArgError, "Unknown key usage type %"PRIsVALUE, INT2NUM(ktype)); + } + obj = NewPKCS12(cPKCS12); x509s = NIL_P(ca) ? NULL : ossl_x509_ary2sk(ca); p12 = PKCS12_create(passphrase, friendlyname, key, x509, x509s, @@ -272,4 +276,8 @@ Init_ossl_pkcs12(void) rb_attr(cPKCS12, rb_intern("ca_certs"), 1, 0, Qfalse); rb_define_method(cPKCS12, "initialize", ossl_pkcs12_initialize, -1); rb_define_method(cPKCS12, "to_der", ossl_pkcs12_to_der, 0); + + /* MSIE specific PKCS12 key usage extensions */ + rb_define_const(cPKCS12, "KEY_EX", INT2NUM(KEY_EX)); + rb_define_const(cPKCS12, "KEY_SIG", INT2NUM(KEY_SIG)); } diff --git a/ext/openssl/ossl_pkcs12.h b/ext/openssl/ossl_pkcs12.h index fe4f15ef60..d4003e81c9 100644 --- a/ext/openssl/ossl_pkcs12.h +++ b/ext/openssl/ossl_pkcs12.h @@ -1,6 +1,6 @@ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_PKCS12_H_) #define _OSSL_PKCS12_H_ diff --git a/ext/openssl/ossl_pkcs7.c b/ext/openssl/ossl_pkcs7.c index aeeb4bf5f4..b7e6d330b2 100644 --- a/ext/openssl/ossl_pkcs7.c +++ b/ext/openssl/ossl_pkcs7.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" @@ -848,6 +848,25 @@ ossl_pkcs7_to_der(VALUE self) } static VALUE +ossl_pkcs7_to_text(VALUE self) +{ + PKCS7 *pkcs7; + BIO *out; + VALUE str; + + GetPKCS7(self, pkcs7); + if(!(out = BIO_new(BIO_s_mem()))) + ossl_raise(ePKCS7Error, NULL); + if(!PKCS7_print_ctx(out, pkcs7, 0, NULL)) { + BIO_free(out); + ossl_raise(ePKCS7Error, NULL); + } + str = ossl_membio2str(out); + + return str; +} + +static VALUE ossl_pkcs7_to_pem(VALUE self) { PKCS7 *pkcs7; @@ -1056,6 +1075,7 @@ Init_ossl_pkcs7(void) rb_define_method(cPKCS7, "to_pem", ossl_pkcs7_to_pem, 0); rb_define_alias(cPKCS7, "to_s", "to_pem"); rb_define_method(cPKCS7, "to_der", ossl_pkcs7_to_der, 0); + rb_define_method(cPKCS7, "to_text", ossl_pkcs7_to_text, 0); cPKCS7Signer = rb_define_class_under(cPKCS7, "SignerInfo", rb_cObject); rb_define_const(cPKCS7, "Signer", cPKCS7Signer); diff --git a/ext/openssl/ossl_pkcs7.h b/ext/openssl/ossl_pkcs7.h index 3e1b094670..4cbbc6a1ae 100644 --- a/ext/openssl/ossl_pkcs7.h +++ b/ext/openssl/ossl_pkcs7.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_PKCS7_H_) #define _OSSL_PKCS7_H_ diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index 013412c27f..6af2245f39 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_pkey.h b/ext/openssl/ossl_pkey.h index 10669b824c..37d828e048 100644 --- a/ext/openssl/ossl_pkey.h +++ b/ext/openssl/ossl_pkey.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(OSSL_PKEY_H) #define OSSL_PKEY_H diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c index a231814a99..00699b9b07 100644 --- a/ext/openssl/ossl_pkey_dh.c +++ b/ext/openssl/ossl_pkey_dh.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c index 058ce73888..a7598d1e80 100644 --- a/ext/openssl/ossl_pkey_dsa.c +++ b/ext/openssl/ossl_pkey_dsa.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c index 389f76f309..7d986989e5 100644 --- a/ext/openssl/ossl_pkey_rsa.c +++ b/ext/openssl/ossl_pkey_rsa.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_provider.c b/ext/openssl/ossl_provider.c index 981c6ccdc7..d1f6c5d427 100644 --- a/ext/openssl/ossl_provider.c +++ b/ext/openssl/ossl_provider.c @@ -1,6 +1,6 @@ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_rand.c b/ext/openssl/ossl_rand.c index 659dc818b6..774e7836dc 100644 --- a/ext/openssl/ossl_rand.c +++ b/ext/openssl/ossl_rand.c @@ -5,7 +5,7 @@ * All rights reserved. * * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_rand.h b/ext/openssl/ossl_rand.h index 8f77a3b239..874ab539b8 100644 --- a/ext/openssl/ossl_rand.h +++ b/ext/openssl/ossl_rand.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_RAND_H_) #define _OSSL_RAND_H_ diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c index d68c64d5fb..457630ddc8 100644 --- a/ext/openssl/ossl_ssl.c +++ b/ext/openssl/ossl_ssl.c @@ -7,7 +7,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_ssl.h b/ext/openssl/ossl_ssl.h index 535c56097c..a92985c601 100644 --- a/ext/openssl/ossl_ssl.h +++ b/ext/openssl/ossl_ssl.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_SSL_H_) #define _OSSL_SSL_H_ diff --git a/ext/openssl/ossl_ts.c b/ext/openssl/ossl_ts.c index f698bdc7ff..d6a5fc9892 100644 --- a/ext/openssl/ossl_ts.c +++ b/ext/openssl/ossl_ts.c @@ -5,7 +5,7 @@ */ /* * This program is licenced under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" @@ -504,6 +504,25 @@ ossl_ts_req_to_der(VALUE self) } static VALUE +ossl_ts_req_to_text(VALUE self) +{ + TS_REQ *req; + BIO *out; + + GetTSRequest(self, req); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_REQ_print_bio(out, req)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + +static VALUE ossl_ts_resp_alloc(VALUE klass) { TS_RESP *resp; @@ -757,6 +776,25 @@ ossl_ts_resp_to_der(VALUE self) return asn1_to_der((void *)resp, (int (*)(void *, unsigned char **))i2d_TS_RESP); } +static VALUE +ossl_ts_resp_to_text(VALUE self) +{ + TS_RESP *resp; + BIO *out; + + GetTSResponse(self, resp); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_RESP_print_bio(out, resp)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + /* * Verifies a timestamp token by checking the signature, validating the * certificate chain implied by tsa_certificate and by checking conformance to @@ -1073,6 +1111,25 @@ ossl_ts_token_info_to_der(VALUE self) return asn1_to_der((void *)info, (int (*)(void *, unsigned char **))i2d_TS_TST_INFO); } +static VALUE +ossl_ts_token_info_to_text(VALUE self) +{ + TS_TST_INFO *info; + BIO *out; + + GetTSTokenInfo(self, info); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_TST_INFO_print_bio(out, info)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + static ASN1_INTEGER * ossl_tsfac_serial_cb(struct TS_resp_ctx *ctx, void *data) { @@ -1356,6 +1413,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampResponse, "token_info", ossl_ts_resp_get_token_info, 0); rb_define_method(cTimestampResponse, "tsa_certificate", ossl_ts_resp_get_tsa_certificate, 0); rb_define_method(cTimestampResponse, "to_der", ossl_ts_resp_to_der, 0); + rb_define_method(cTimestampResponse, "to_text", ossl_ts_resp_to_text, 0); rb_define_method(cTimestampResponse, "verify", ossl_ts_resp_verify, -1); /* Document-class: OpenSSL::Timestamp::TokenInfo @@ -1374,6 +1432,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampTokenInfo, "ordering", ossl_ts_token_info_get_ordering, 0); rb_define_method(cTimestampTokenInfo, "nonce", ossl_ts_token_info_get_nonce, 0); rb_define_method(cTimestampTokenInfo, "to_der", ossl_ts_token_info_to_der, 0); + rb_define_method(cTimestampTokenInfo, "to_text", ossl_ts_token_info_to_text, 0); /* Document-class: OpenSSL::Timestamp::Request * Allows to create timestamp requests or parse existing ones. A Request is @@ -1399,6 +1458,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampRequest, "cert_requested=", ossl_ts_req_set_cert_requested, 1); rb_define_method(cTimestampRequest, "cert_requested?", ossl_ts_req_get_cert_requested, 0); rb_define_method(cTimestampRequest, "to_der", ossl_ts_req_to_der, 0); + rb_define_method(cTimestampRequest, "to_text", ossl_ts_req_to_text, 0); /* * Indicates a successful response. Equal to +0+. diff --git a/ext/openssl/ossl_ts.h b/ext/openssl/ossl_ts.h index 25fb0e1d64..eeca3046eb 100644 --- a/ext/openssl/ossl_ts.h +++ b/ext/openssl/ossl_ts.h @@ -5,7 +5,7 @@ */ /* * This program is licenced under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_TS_H_) diff --git a/ext/openssl/ossl_x509.c b/ext/openssl/ossl_x509.c index f8470703fc..9686fc1a9c 100644 --- a/ext/openssl/ossl_x509.c +++ b/ext/openssl/ossl_x509.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509.h b/ext/openssl/ossl_x509.h index 4fadfa6b82..88e3f16a1a 100644 --- a/ext/openssl/ossl_x509.h +++ b/ext/openssl/ossl_x509.h @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #if !defined(_OSSL_X509_H_) #define _OSSL_X509_H_ diff --git a/ext/openssl/ossl_x509attr.c b/ext/openssl/ossl_x509attr.c index d1d8bb5e95..be525c9e7c 100644 --- a/ext/openssl/ossl_x509attr.c +++ b/ext/openssl/ossl_x509attr.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509cert.c b/ext/openssl/ossl_x509cert.c index aa6b9bb7ce..dbf83ff525 100644 --- a/ext/openssl/ossl_x509cert.c +++ b/ext/openssl/ossl_x509cert.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" @@ -707,6 +707,38 @@ ossl_x509_eq(VALUE self, VALUE other) return !X509_cmp(a, b) ? Qtrue : Qfalse; } +#ifdef HAVE_I2D_RE_X509_TBS +/* + * call-seq: + * cert.tbs_bytes => string + * + * Returns the DER-encoded bytes of the certificate's to be signed certificate. + * This is mainly useful for validating embedded certificate transparency signatures. + */ +static VALUE +ossl_x509_tbs_bytes(VALUE self) +{ + X509 *x509; + int len; + unsigned char *p0; + VALUE str; + + GetX509(self, x509); + len = i2d_re_X509_tbs(x509, NULL); + if (len <= 0) { + ossl_raise(eX509CertError, "i2d_re_X509_tbs"); + } + str = rb_str_new(NULL, len); + p0 = (unsigned char *)RSTRING_PTR(str); + if (i2d_re_X509_tbs(x509, &p0) <= 0) { + ossl_raise(eX509CertError, "i2d_re_X509_tbs"); + } + ossl_str_adjust(str, p0); + + return str; +} +#endif + struct load_chained_certificates_arguments { VALUE certificates; X509 *certificate; @@ -999,4 +1031,7 @@ Init_ossl_x509cert(void) rb_define_method(cX509Cert, "add_extension", ossl_x509_add_extension, 1); rb_define_method(cX509Cert, "inspect", ossl_x509_inspect, 0); rb_define_method(cX509Cert, "==", ossl_x509_eq, 1); +#ifdef HAVE_I2D_RE_X509_TBS + rb_define_method(cX509Cert, "tbs_bytes", ossl_x509_tbs_bytes, 0); +#endif } diff --git a/ext/openssl/ossl_x509crl.c b/ext/openssl/ossl_x509crl.c index 80e29f9df2..368270ce11 100644 --- a/ext/openssl/ossl_x509crl.c +++ b/ext/openssl/ossl_x509crl.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509ext.c b/ext/openssl/ossl_x509ext.c index 192d09bd3f..7f47cd7cce 100644 --- a/ext/openssl/ossl_x509ext.c +++ b/ext/openssl/ossl_x509ext.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509name.c b/ext/openssl/ossl_x509name.c index 9591912f70..5060be92cc 100644 --- a/ext/openssl/ossl_x509name.c +++ b/ext/openssl/ossl_x509name.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509req.c b/ext/openssl/ossl_x509req.c index f058185151..37ba03728f 100644 --- a/ext/openssl/ossl_x509req.c +++ b/ext/openssl/ossl_x509req.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509revoked.c b/ext/openssl/ossl_x509revoked.c index 108447c868..5b82470c83 100644 --- a/ext/openssl/ossl_x509revoked.c +++ b/ext/openssl/ossl_x509revoked.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/openssl/ossl_x509store.c b/ext/openssl/ossl_x509store.c index f27381ca90..31328ec47f 100644 --- a/ext/openssl/ossl_x509store.c +++ b/ext/openssl/ossl_x509store.c @@ -5,7 +5,7 @@ */ /* * This program is licensed under the same licence as Ruby. - * (See the file 'LICENCE'.) + * (See the file 'COPYING'.) */ #include "ossl.h" diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl index 21f29bd79a..fc98c067b8 100644 --- a/ext/ripper/ripper_init.c.tmpl +++ b/ext/ripper/ripper_init.c.tmpl @@ -665,8 +665,4 @@ InitVM_ripper(void) */ rb_define_global_const("SCRIPT_LINES__", Qnil); #endif - rb_ripper_none = rb_obj_alloc(rb_cObject); - rb_obj_freeze(rb_ripper_none); - rb_gc_register_mark_object(rb_ripper_none); - } diff --git a/ext/ripper/ripper_init.h b/ext/ripper/ripper_init.h index 664bb7bce3..9d228107d1 100644 --- a/ext/ripper/ripper_init.h +++ b/ext/ripper/ripper_init.h @@ -1,7 +1,6 @@ #ifndef RIPPER_INIT_H #define RIPPER_INIT_H -extern VALUE rb_ripper_none; PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3); #endif /* RIPPER_INIT_H */ diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb index d0002d1ec3..38f859dd97 100644 --- a/ext/ripper/tools/dsl.rb +++ b/ext/ripper/tools/dsl.rb @@ -20,83 +20,158 @@ class DSL NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source - def self.line?(line, lineno = nil) - if %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/> =~ line - new($2, $1&.split(",") || [], lineno) + def self.line?(line, lineno = nil, indent: nil) + if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line + new(code, comma_split(option), lineno, indent: indent || space) end end - def initialize(code, options, lineno = nil) + def self.comma_split(str) + str or return [] + str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first) + end + + using Module.new { + refine Array do + def to_s + if empty? + "rb_ary_new()" + else + "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})" + end + end + end + } + + class Var + class Table < Hash + def initialize(&block) + super() {|tbl, arg| + tbl.fetch(arg, &block) + } + end + + def fetch(arg, &block) + super { + self[arg] = Var.new(self, arg, &block) + } + end + + def add(&block) + v = new_var + self[v] = Var.new(self, v, &block) + end + + def defined?(name) + name = name.to_s + any? {|_, v| v.var == name} + end + + def new_var + "v#{size+1}" + end + end + + attr_reader :var, :value + + PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze + + def pretty_print_instance_variables + PRETTY_PRINT_INSTANCE_VARIABLES + end + + alias to_s var + + def initialize(table, arg, &block) + @var = table.new_var + @value = yield arg + @table = table + end + + # Indexing. + # + # $:1 -> v1=get_value($:1) + # $:1[0] -> rb_ary_entry(v1, 0) + # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)] + # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1) + # + # Splat needs `[range]` because `Var` does not have the length info. + def [](idx) + if ::Range === idx + idx.map {|i| self[i]} + else + @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"} + end + end + end + + def initialize(code, options, lineno = nil, indent: "\t\t\t") @lineno = lineno + @indent = indent @events = {} @error = options.include?("error") - @brace = options.include?("brace") if options.include?("final") @final = "p->result" else @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue") end - @vars = 0 - - # struct parser_params *p - p = p = "p" - @code = +"" - code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"') - @last_value = eval(code) + bind = dsl_binding + @var_table = Var::Table.new {|arg| "get_value(#{arg})"} + code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) { + if (arg = $&) == "$:$" + '"p->s_lvalue"' + elsif arg.start_with?("$:") + "(#{@var_table[arg]}=@var_table[#{arg.dump}])" + else + arg.dump + end + } + @last_value = bind.eval(code) rescue SyntaxError $stderr.puts "error on line #{@lineno}" if @lineno raise end + def dsl_binding(p = "p") + # struct parser_params *p + binding + end + attr_reader :events undef lambda undef hash - undef class + undef :class def generate - s = "#@code#@final=#@last_value;" - s = "{VALUE #{ (1..@vars).map {|v| "v#{ v }" }.join(",") };#{ s }}" if @vars > 0 + s = "#@final=#@last_value;" s << "ripper_error(p);" if @error - s = "{#{ s }}" if @brace - "\t\t\t#{s}" - end - - def new_var - "v#{ @vars += 1 }" - end - - def opt_event(event, default, addend) - add_event(event, [default, addend], true) + unless @var_table.empty? + vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ") + s = "VALUE #{ vars }; #{ s }" + end + "#{@indent}{#{s}}" end - def add_event(event, args, qundef_check = false) + def add_event(event, args) event = event.to_s.sub(/!\z/, "") @events[event] = args.size vars = [] args.each do |arg| - vars << v = new_var - if arg =~ /\A\$:#{NAME_PATTERN}\z/ - @code << "#{ v }=get_value(#{arg});" - else - @code << "#{ v }=#{ arg };" - end + arg = @var_table.add {arg} unless Var === arg + vars << arg end - v = new_var - d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })" - d = "#{ vars.last }==rb_ripper_none ? #{ vars.first } : #{ d }" if qundef_check - @code << "#{ v }=#{ d };" - v + @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"} end def method_missing(event, *args) if event.to_s =~ /!\z/ add_event(event, args) - elsif args.empty? and /\Aid[A-Z_]/ =~ event.to_s + elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event)) event else - "#{ event }(#{ args.join(", ") })" + "#{ event }(#{ args.map(&:to_s).join(", ") })" end end diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb index 92ced37f04..57ecac0b39 100644 --- a/ext/ripper/tools/generate.rb +++ b/ext/ripper/tools/generate.rb @@ -75,6 +75,7 @@ def generate_eventids1_h(ids) buf << %Q[#ifndef RIPPER_EVENTIDS1\n] buf << %Q[#define RIPPER_EVENTIDS1\n] buf << %Q[\n] + buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n] buf << %Q[void ripper_init_eventids1(void);\n] buf << %Q[void ripper_init_eventids1_table(VALUE self);\n] buf << %Q[\n] @@ -84,9 +85,6 @@ def generate_eventids1_h(ids) end buf << %Q[};\n] buf << %Q[\n] - ids.each do |id, arity| - buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n] - end buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n] buf << %Q[\n] end @@ -101,7 +99,7 @@ def generate_eventids1(ids) buf << %Q[void\n] buf << %Q[ripper_init_eventids1(void)\n] buf << %Q[{\n] - buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n] + buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n] ids.each do |id, arity| buf << %Q[ set_id1(#{id});\n] end diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb index a92be93d5b..5e8a6e0cb5 100644 --- a/ext/ripper/tools/preproc.rb +++ b/ext/ripper/tools/preproc.rb @@ -56,7 +56,7 @@ require_relative 'dsl' def generate_line(f, out) while line = f.gets case - when gen = DSL.line?(line) + when gen = DSL.line?(line, f.lineno) out << gen.generate << "\n" when line.start_with?("%%") out << "%%\n" diff --git a/ext/socket/raddrinfo.c b/ext/socket/raddrinfo.c index e79bcfa332..090ba1a0c0 100644 --- a/ext/socket/raddrinfo.c +++ b/ext/socket/raddrinfo.c @@ -481,7 +481,7 @@ rb_getaddrinfo(const char *hostp, const char *portp, const struct addrinfo *hint { int retry; struct getaddrinfo_arg *arg; - int err, gai_errno = 0; + int err = 0, gai_errno = 0; start: retry = 0; @@ -493,8 +493,10 @@ start: pthread_t th; if (do_pthread_create(&th, do_getaddrinfo, arg) != 0) { + int err = errno; free_getaddrinfo_arg(arg); - return EAI_AGAIN; + errno = err; + return EAI_SYSTEM; } pthread_detach(th); @@ -712,8 +714,10 @@ start: pthread_t th; if (do_pthread_create(&th, do_getnameinfo, arg) != 0) { + int err = errno; free_getnameinfo_arg(arg); - return EAI_AGAIN; + errno = err; + return EAI_SYSTEM; } pthread_detach(th); diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index 820b8228a3..c149811e05 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -51,18 +51,11 @@ static long strio_write(VALUE self, VALUE str); #define IS_STRIO(obj) (rb_typeddata_is_kind_of((obj), &strio_data_type)) #define error_inval(msg) (rb_syserr_fail(EINVAL, msg)) #define get_enc(ptr) ((ptr)->enc ? (ptr)->enc : !NIL_P((ptr)->string) ? rb_enc_get((ptr)->string) : NULL) -#ifndef HAVE_RB_STR_CHILLED_P -static bool -rb_str_chilled_p(VALUE str) -{ - return false; -} -#endif static bool readonly_string_p(VALUE string) { - return OBJ_FROZEN_RAW(string) && !rb_str_chilled_p(string); + return OBJ_FROZEN_RAW(string); } static struct StringIO * @@ -184,9 +177,6 @@ check_modifiable(struct StringIO *ptr) if (NIL_P(ptr->string)) { /* Null device StringIO */ } - else if (rb_str_chilled_p(ptr->string)) { - rb_str_modify(ptr->string); - } else if (OBJ_FROZEN_RAW(ptr->string)) { rb_raise(rb_eIOError, "not modifiable string"); } diff --git a/ext/strscan/strscan.c b/ext/strscan/strscan.c index 70a3ce5260..fad35925a8 100644 --- a/ext/strscan/strscan.c +++ b/ext/strscan/strscan.c @@ -218,16 +218,28 @@ strscan_s_allocate(VALUE klass) } /* - * call-seq: - * StringScanner.new(string, fixed_anchor: false) - * StringScanner.new(string, dup = false) - * - * Creates a new StringScanner object to scan over the given +string+. + * :markup: markdown + * :include: strscan/link_refs.txt * - * If +fixed_anchor+ is +true+, +\A+ always matches the beginning of - * the string. Otherwise, +\A+ always matches the current position. + * call-seq: + * StringScanner.new(string, fixed_anchor: false) -> string_scanner + * + * Returns a new `StringScanner` object whose [stored string][1] + * is the given `string`; + * sets the [fixed-anchor property][10]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.string # => "foobarbaz" + * scanner.fixed_anchor? # => false + * put_situation(scanner) + * # Situation: + * # pos: 0 + * # charpos: 0 + * # rest: "foobarbaz" + * # rest_size: 9 + * ``` * - * +dup+ argument is obsolete and not used now. */ static VALUE strscan_initialize(int argc, VALUE *argv, VALUE self) @@ -266,11 +278,14 @@ check_strscan(VALUE obj) } /* + * :markup: markdown + * :include: strscan/link_refs.txt + * * call-seq: - * dup - * clone + * dup -> shallow_copy * - * Duplicates a StringScanner object. + * Returns a shallow copy of `self`; + * the [stored string][1] in the copy is the same string as in `self`. */ static VALUE strscan_init_copy(VALUE vself, VALUE vorig) @@ -297,10 +312,13 @@ strscan_init_copy(VALUE vself, VALUE vorig) ======================================================================= */ /* - * call-seq: StringScanner.must_C_version + * call-seq: + * StringScanner.must_C_version -> self * - * This method is defined for backward compatibility. + * Returns +self+; defined for backward compatibility. */ + + /* :nodoc: */ static VALUE strscan_s_mustc(VALUE self) { @@ -308,7 +326,30 @@ strscan_s_mustc(VALUE self) } /* - * Reset the scan pointer (index 0) and clear matching data. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * reset -> self + * + * Sets both [byte position][2] and [character position][7] to zero, + * and clears [match values][9]; + * returns +self+: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.exist?(/bar/) # => 6 + * scanner.reset # => #<StringScanner 0/9 @ "fooba..."> + * put_situation(scanner) + * # Situation: + * # pos: 0 + * # charpos: 0 + * # rest: "foobarbaz" + * # rest_size: 9 + * # => nil + * match_values_cleared?(scanner) # => true + * ``` + * */ static VALUE strscan_reset(VALUE self) @@ -322,11 +363,9 @@ strscan_reset(VALUE self) } /* - * call-seq: - * terminate - * clear - * - * Sets the scan pointer to the end of the string and clear matching data. + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/terminate.md */ static VALUE strscan_terminate(VALUE self) @@ -340,9 +379,13 @@ strscan_terminate(VALUE self) } /* - * Equivalent to #terminate. - * This method is obsolete; use #terminate instead. + * call-seq: + * clear -> self + * + * This method is obsolete; use the equivalent method StringScanner#terminate. */ + + /* :nodoc: */ static VALUE strscan_clear(VALUE self) { @@ -351,7 +394,21 @@ strscan_clear(VALUE self) } /* - * Returns the string being scanned. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * string -> stored_string + * + * Returns the [stored string][1]: + * + * ``` + * scanner = StringScanner.new('foobar') + * scanner.string # => "foobar" + * scanner.concat('baz') + * scanner.string # => "foobarbaz" + * ``` + * */ static VALUE strscan_get_string(VALUE self) @@ -363,10 +420,39 @@ strscan_get_string(VALUE self) } /* - * call-seq: string=(str) + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * string = other_string -> other_string + * + * Replaces the [stored string][1] with the given `other_string`: + * + * - Sets both [positions][11] to zero. + * - Clears [match values][9]. + * - Returns `other_string`. + * + * ``` + * scanner = StringScanner.new('foobar') + * scanner.scan(/foo/) + * put_situation(scanner) + * # Situation: + * # pos: 3 + * # charpos: 3 + * # rest: "bar" + * # rest_size: 3 + * match_values_cleared?(scanner) # => false + * + * scanner.string = 'baz' # => "baz" + * put_situation(scanner) + * # Situation: + * # pos: 0 + * # charpos: 0 + * # rest: "baz" + * # rest_size: 3 + * match_values_cleared?(scanner) # => true + * ``` * - * Changes the string being scanned to +str+ and resets the scanner. - * Returns +str+. */ static VALUE strscan_set_string(VALUE self, VALUE str) @@ -381,18 +467,33 @@ strscan_set_string(VALUE self, VALUE str) } /* - * call-seq: - * concat(str) - * <<(str) + * :markup: markdown + * :include: strscan/link_refs.txt * - * Appends +str+ to the string being scanned. - * This method does not affect scan pointer. + * call-seq: + * concat(more_string) -> self + * + * - Appends the given `more_string` + * to the [stored string][1]. + * - Returns `self`. + * - Does not affect the [positions][11] + * or [match values][9]. + * + * + * ``` + * scanner = StringScanner.new('foo') + * scanner.string # => "foo" + * scanner.terminate + * scanner.concat('barbaz') # => #<StringScanner 3/9 "foo" @ "barba..."> + * scanner.string # => "foobarbaz" + * put_situation(scanner) + * # Situation: + * # pos: 3 + * # charpos: 3 + * # rest: "barbaz" + * # rest_size: 6 + * ``` * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan(/Fri /) - * s << " +1000 GMT" - * s.string # -> "Fri Dec 12 1975 14:39 +1000 GMT" - * s.scan(/Dec/) # -> "Dec" */ static VALUE strscan_concat(VALUE self, VALUE str) @@ -406,18 +507,9 @@ strscan_concat(VALUE self, VALUE str) } /* - * Returns the byte position of the scan pointer. In the 'reset' position, this - * value is zero. In the 'terminated' position (i.e. the string is exhausted), - * this value is the bytesize of the string. - * - * In short, it's a 0-based index into bytes of the string. - * - * s = StringScanner.new('test string') - * s.pos # -> 0 - * s.scan_until /str/ # -> "test str" - * s.pos # -> 8 - * s.terminate # -> #<StringScanner fin> - * s.pos # -> 11 + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/get_pos.md */ static VALUE strscan_get_pos(VALUE self) @@ -429,17 +521,9 @@ strscan_get_pos(VALUE self) } /* - * Returns the character position of the scan pointer. In the 'reset' position, this - * value is zero. In the 'terminated' position (i.e. the string is exhausted), - * this value is the size of the string. - * - * In short, it's a 0-based index into the string. - * - * s = StringScanner.new("abc\u00e4def\u00f6ghi") - * s.charpos # -> 0 - * s.scan_until(/\u00e4/) # -> "abc\u00E4" - * s.pos # -> 5 - * s.charpos # -> 4 + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/get_charpos.md */ static VALUE strscan_get_charpos(VALUE self) @@ -452,13 +536,9 @@ strscan_get_charpos(VALUE self) } /* - * call-seq: pos=(n) - * - * Sets the byte position of the scan pointer. - * - * s = StringScanner.new('test string') - * s.pos = 7 # -> 7 - * s.rest # -> "ring" + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/set_pos.md */ static VALUE strscan_set_pos(VALUE self, VALUE v) @@ -662,20 +742,9 @@ strscan_do_scan(VALUE self, VALUE pattern, int succptr, int getstr, int headonly } /* - * call-seq: scan(pattern) => String - * - * Tries to match with +pattern+ at the current position. If there's a match, - * the scanner advances the "scan pointer" and returns the matched string. - * Otherwise, the scanner returns +nil+. - * - * s = StringScanner.new('test string') - * p s.scan(/\w+/) # -> "test" - * p s.scan(/\w+/) # -> nil - * p s.scan(/\s+/) # -> " " - * p s.scan("str") # -> "str" - * p s.scan(/\w+/) # -> "ing" - * p s.scan(/./) # -> nil - * + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/scan.md */ static VALUE strscan_scan(VALUE self, VALUE re) @@ -684,16 +753,60 @@ strscan_scan(VALUE self, VALUE re) } /* - * call-seq: match?(pattern) + * :markup: markdown + * :include: strscan/link_refs.txt * - * Tests whether the given +pattern+ is matched from the current scan pointer. - * Returns the length of the match, or +nil+. The scan pointer is not advanced. + * call-seq: + * match?(pattern) -> updated_position or nil + * + * Attempts to [match][17] the given `pattern` + * at the beginning of the [target substring][3]; + * does not modify the [positions][11]. + * + * If the match succeeds: + * + * - Sets [match values][9]. + * - Returns the size in bytes of the matched substring. + * + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.pos = 3 + * scanner.match?(/bar/) => 3 + * put_match_values(scanner) + * # Basic match values: + * # matched?: true + * # matched_size: 3 + * # pre_match: "foo" + * # matched : "bar" + * # post_match: "baz" + * # Captured match values: + * # size: 1 + * # captures: [] + * # named_captures: {} + * # values_at: ["bar", nil] + * # []: + * # [0]: "bar" + * # [1]: nil + * put_situation(scanner) + * # Situation: + * # pos: 3 + * # charpos: 3 + * # rest: "barbaz" + * # rest_size: 6 + * ``` + * + * If the match fails: + * + * - Clears match values. + * - Returns `nil`. + * - Does not increment positions. + * + * ``` + * scanner.match?(/nope/) # => nil + * match_values_cleared?(scanner) # => true + * ``` * - * s = StringScanner.new('test string') - * p s.match?(/\w+/) # -> 4 - * p s.match?(/\w+/) # -> 4 - * p s.match?("test") # -> 4 - * p s.match?(/\s+/) # -> nil */ static VALUE strscan_match_p(VALUE self, VALUE re) @@ -702,22 +815,9 @@ strscan_match_p(VALUE self, VALUE re) } /* - * call-seq: skip(pattern) - * - * Attempts to skip over the given +pattern+ beginning with the scan pointer. - * If it matches, the scan pointer is advanced to the end of the match, and the - * length of the match is returned. Otherwise, +nil+ is returned. - * - * It's similar to #scan, but without returning the matched string. - * - * s = StringScanner.new('test string') - * p s.skip(/\w+/) # -> 4 - * p s.skip(/\w+/) # -> nil - * p s.skip(/\s+/) # -> 1 - * p s.skip("st") # -> 2 - * p s.skip(/\w+/) # -> 4 - * p s.skip(/./) # -> nil - * + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/skip.md */ static VALUE strscan_skip(VALUE self, VALUE re) @@ -726,19 +826,59 @@ strscan_skip(VALUE self, VALUE re) } /* - * call-seq: check(pattern) - * - * This returns the value that #scan would return, without advancing the scan - * pointer. The match register is affected, though. + * :markup: markdown + * :include: strscan/link_refs.txt * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.check /Fri/ # -> "Fri" - * s.pos # -> 0 - * s.matched # -> "Fri" - * s.check /12/ # -> nil - * s.matched # -> nil + * call-seq: + * check(pattern) -> matched_substring or nil + * + * Attempts to [match][17] the given `pattern` + * at the beginning of the [target substring][3]; + * does not modify the [positions][11]. + * + * If the match succeeds: + * + * - Returns the matched substring. + * - Sets all [match values][9]. + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.pos = 3 + * scanner.check('bar') # => "bar" + * put_match_values(scanner) + * # Basic match values: + * # matched?: true + * # matched_size: 3 + * # pre_match: "foo" + * # matched : "bar" + * # post_match: "baz" + * # Captured match values: + * # size: 1 + * # captures: [] + * # named_captures: {} + * # values_at: ["bar", nil] + * # []: + * # [0]: "bar" + * # [1]: nil + * # => 0..1 + * put_situation(scanner) + * # Situation: + * # pos: 3 + * # charpos: 3 + * # rest: "barbaz" + * # rest_size: 6 + * ``` + * + * If the match fails: + * + * - Returns `nil`. + * - Clears all [match values][9]. + * + * ``` + * scanner.check(/nope/) # => nil + * match_values_cleared?(scanner) # => true + * ``` * - * Mnemonic: it "checks" to see whether a #scan will return a value. */ static VALUE strscan_check(VALUE self, VALUE re) @@ -747,15 +887,24 @@ strscan_check(VALUE self, VALUE re) } /* - * call-seq: scan_full(pattern, advance_pointer_p, return_string_p) + * call-seq: + * scan_full(pattern, advance_pointer_p, return_string_p) -> matched_substring or nil + * + * Equivalent to one of the following: + * + * - +advance_pointer_p+ +true+: * - * Tests whether the given +pattern+ is matched from the current scan pointer. - * Advances the scan pointer if +advance_pointer_p+ is true. - * Returns the matched string if +return_string_p+ is true. - * The match register is affected. + * - +return_string_p+ +true+: StringScanner#scan(pattern). + * - +return_string_p+ +false+: StringScanner#skip(pattern). + * + * - +advance_pointer_p+ +false+: + * + * - +return_string_p+ +true+: StringScanner#check(pattern). + * - +return_string_p+ +false+: StringScanner#match?(pattern). * - * "full" means "#scan with full parameters". */ + + /* :nodoc: */ static VALUE strscan_scan_full(VALUE self, VALUE re, VALUE s, VALUE f) { @@ -763,16 +912,9 @@ strscan_scan_full(VALUE self, VALUE re, VALUE s, VALUE f) } /* - * call-seq: scan_until(pattern) - * - * Scans the string _until_ the +pattern+ is matched. Returns the substring up - * to and including the end of the match, advancing the scan pointer to that - * location. If there is no match, +nil+ is returned. - * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan_until(/1/) # -> "Fri Dec 1" - * s.pre_match # -> "Fri Dec " - * s.scan_until(/XYZ/) # -> nil + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/scan_until.md */ static VALUE strscan_scan_until(VALUE self, VALUE re) @@ -781,17 +923,61 @@ strscan_scan_until(VALUE self, VALUE re) } /* - * call-seq: exist?(pattern) + * :markup: markdown + * :include: strscan/link_refs.txt * - * Looks _ahead_ to see if the +pattern+ exists _anywhere_ in the string, - * without advancing the scan pointer. This predicates whether a #scan_until - * will return a value. + * call-seq: + * exist?(pattern) -> byte_offset or nil + * + * Attempts to [match][17] the given `pattern` + * anywhere (at any [position][2]) + * n the [target substring][3]; + * does not modify the [positions][11]. + * + * If the match succeeds: + * + * - Returns a byte offset: + * the distance in bytes between the current [position][2] + * and the end of the matched substring. + * - Sets all [match values][9]. + * + * ``` + * scanner = StringScanner.new('foobarbazbatbam') + * scanner.pos = 6 + * scanner.exist?(/bat/) # => 6 + * put_match_values(scanner) + * # Basic match values: + * # matched?: true + * # matched_size: 3 + * # pre_match: "foobarbaz" + * # matched : "bat" + * # post_match: "bam" + * # Captured match values: + * # size: 1 + * # captures: [] + * # named_captures: {} + * # values_at: ["bat", nil] + * # []: + * # [0]: "bat" + * # [1]: nil + * put_situation(scanner) + * # Situation: + * # pos: 6 + * # charpos: 6 + * # rest: "bazbatbam" + * # rest_size: 9 + * ``` + * + * If the match fails: + * + * - Returns `nil`. + * - Clears all [match values][9]. + * + * ``` + * scanner.exist?(/nope/) # => nil + * match_values_cleared?(scanner) # => true + * ``` * - * s = StringScanner.new('test string') - * s.exist? /s/ # -> 3 - * s.scan /test/ # -> "test" - * s.exist? /s/ # -> 2 - * s.exist? /e/ # -> nil */ static VALUE strscan_exist_p(VALUE self, VALUE re) @@ -800,20 +986,9 @@ strscan_exist_p(VALUE self, VALUE re) } /* - * call-seq: skip_until(pattern) - * - * Advances the scan pointer until +pattern+ is matched and consumed. Returns - * the number of bytes advanced, or +nil+ if no match was found. - * - * Look ahead to match +pattern+, and advance the scan pointer to the _end_ - * of the match. Return the number of characters advanced, or +nil+ if the - * match was unsuccessful. - * - * It's similar to #scan_until, but without returning the intervening string. - * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.skip_until /12/ # -> 10 - * s # + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/skip_until.md */ static VALUE strscan_skip_until(VALUE self, VALUE re) @@ -822,17 +997,61 @@ strscan_skip_until(VALUE self, VALUE re) } /* - * call-seq: check_until(pattern) - * - * This returns the value that #scan_until would return, without advancing the - * scan pointer. The match register is affected, though. + * :markup: markdown + * :include: strscan/link_refs.txt * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.check_until /12/ # -> "Fri Dec 12" - * s.pos # -> 0 - * s.matched # -> 12 + * call-seq: + * check_until(pattern) -> substring or nil + * + * Attempts to [match][17] the given `pattern` + * anywhere (at any [position][2]) + * in the [target substring][3]; + * does not modify the [positions][11]. + * + * If the match succeeds: + * + * - Sets all [match values][9]. + * - Returns the matched substring, + * which extends from the current [position][2] + * to the end of the matched substring. + * + * ``` + * scanner = StringScanner.new('foobarbazbatbam') + * scanner.pos = 6 + * scanner.check_until(/bat/) # => "bazbat" + * put_match_values(scanner) + * # Basic match values: + * # matched?: true + * # matched_size: 3 + * # pre_match: "foobarbaz" + * # matched : "bat" + * # post_match: "bam" + * # Captured match values: + * # size: 1 + * # captures: [] + * # named_captures: {} + * # values_at: ["bat", nil] + * # []: + * # [0]: "bat" + * # [1]: nil + * put_situation(scanner) + * # Situation: + * # pos: 6 + * # charpos: 6 + * # rest: "bazbatbam" + * # rest_size: 9 + * ``` + * + * If the match fails: + * + * - Clears all [match values][9]. + * - Returns `nil`. + * + * ``` + * scanner.check_until(/nope/) # => nil + * match_values_cleared?(scanner) # => true + * ``` * - * Mnemonic: it "checks" to see whether a #scan_until will return a value. */ static VALUE strscan_check_until(VALUE self, VALUE re) @@ -841,14 +1060,24 @@ strscan_check_until(VALUE self, VALUE re) } /* - * call-seq: search_full(pattern, advance_pointer_p, return_string_p) + * call-seq: + * search_full(pattern, advance_pointer_p, return_string_p) -> matched_substring or position_delta or nil + * + * Equivalent to one of the following: + * + * - +advance_pointer_p+ +true+: + * + * - +return_string_p+ +true+: StringScanner#scan_until(pattern). + * - +return_string_p+ +false+: StringScanner#skip_until(pattern). + * + * - +advance_pointer_p+ +false+: + * + * - +return_string_p+ +true+: StringScanner#check_until(pattern). + * - +return_string_p+ +false+: StringScanner#exist?(pattern). * - * Scans the string _until_ the +pattern+ is matched. - * Advances the scan pointer if +advance_pointer_p+, otherwise not. - * Returns the matched string if +return_string_p+ is true, otherwise - * returns the number of bytes advanced. - * This method does affect the match register. */ + + /* :nodoc: */ static VALUE strscan_search_full(VALUE self, VALUE re, VALUE s, VALUE f) { @@ -868,17 +1097,9 @@ adjust_registers_to_matched(struct strscanner *p) } /* - * Scans one character and returns it. - * This method is multibyte character sensitive. - * - * s = StringScanner.new("ab") - * s.getch # => "a" - * s.getch # => "b" - * s.getch # => nil - * - * s = StringScanner.new("\244\242".force_encoding("euc-jp")) - * s.getch # => "\x{A4A2}" # Japanese hira-kana "A" in EUC-JP - * s.getch # => nil + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/getch.md */ static VALUE strscan_getch(VALUE self) @@ -903,19 +1124,13 @@ strscan_getch(VALUE self) } /* + * call-seq: + * scan_byte -> integer_byte + * * Scans one byte and returns it as an integer. * This method is not multibyte character sensitive. * See also: #getch. * - * s = StringScanner.new('ab') - * s.scan_byte # => 97 - * s.scan_byte # => 98 - * s.scan_byte # => nil - * - * s = StringScanner.new("\244\242".force_encoding("euc-jp")) - * s.scan_byte # => 0xA4 - * s.scan_byte # => 0xA2 - * s.scan_byte # => nil */ static VALUE strscan_scan_byte(VALUE self) @@ -954,19 +1169,9 @@ strscan_peek_byte(VALUE self) } /* - * Scans one byte and returns it. - * This method is not multibyte character sensitive. - * See also: #getch. - * - * s = StringScanner.new('ab') - * s.get_byte # => "a" - * s.get_byte # => "b" - * s.get_byte # => nil - * - * s = StringScanner.new("\244\242".force_encoding("euc-jp")) - * s.get_byte # => "\xA4" - * s.get_byte # => "\xA2" - * s.get_byte # => nil + * :markup: markdown + * :include: strscan/link_refs.txt + * :include: strscan/methods/get_byte.md */ static VALUE strscan_get_byte(VALUE self) @@ -988,9 +1193,14 @@ strscan_get_byte(VALUE self) } /* + * call-seq: + * getbyte + * * Equivalent to #get_byte. * This method is obsolete; use #get_byte instead. */ + + /* :nodoc: */ static VALUE strscan_getbyte(VALUE self) { @@ -999,14 +1209,22 @@ strscan_getbyte(VALUE self) } /* - * call-seq: peek(len) + * :markup: markdown + * :include: strscan/link_refs.txt * - * Extracts a string corresponding to <tt>string[pos,len]</tt>, without - * advancing the scan pointer. + * call-seq: + * peek(length) -> substring * - * s = StringScanner.new('test string') - * s.peek(7) # => "test st" - * s.peek(7) # => "test st" + * Returns the substring `string[pos, length]`; + * does not update [match values][9] or [positions][11]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.pos = 3 + * scanner.peek(3) # => "bar" + * scanner.terminate + * scanner.peek(3) # => "" + * ``` * */ static VALUE @@ -1026,9 +1244,14 @@ strscan_peek(VALUE self, VALUE vlen) } /* + * call-seq: + * peep + * * Equivalent to #peek. * This method is obsolete; use #peek instead. */ + + /* :nodoc: */ static VALUE strscan_peep(VALUE self, VALUE vlen) { @@ -1037,15 +1260,42 @@ strscan_peep(VALUE self, VALUE vlen) } /* - * Sets the scan pointer to the previous position. Only one previous position is - * remembered, and it changes with each scanning operation. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * unscan -> self + * + * Sets the [position][2] to its value previous to the recent successful + * [match][17] attempt: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.scan(/foo/) + * put_situation(scanner) + * # Situation: + * # pos: 3 + * # charpos: 3 + * # rest: "barbaz" + * # rest_size: 6 + * scanner.unscan + * # => #<StringScanner 0/9 @ "fooba..."> + * put_situation(scanner) + * # Situation: + * # pos: 0 + * # charpos: 0 + * # rest: "foobarbaz" + * # rest_size: 9 + * ``` + * + * Raises an exception if match values are clear: + * + * ``` + * scanner.scan(/nope/) # => nil + * match_values_cleared?(scanner) # => true + * scanner.unscan # Raises StringScanner::Error. + * ``` * - * s = StringScanner.new('test string') - * s.scan(/\w+/) # => "test" - * s.unscan - * s.scan(/../) # => "te" - * s.scan(/\d/) # => nil - * s.unscan # ScanError: unscan failed: previous match record not exist */ static VALUE strscan_unscan(VALUE self) @@ -1061,16 +1311,37 @@ strscan_unscan(VALUE self) } /* - * Returns +true+ if and only if the scan pointer is at the beginning of the line. - * - * s = StringScanner.new("test\ntest\n") - * s.bol? # => true - * s.scan(/te/) - * s.bol? # => false - * s.scan(/st\n/) - * s.bol? # => true - * s.terminate - * s.bol? # => true + * + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * beginning_of_line? -> true or false + * + * Returns whether the [position][2] is at the beginning of a line; + * that is, at the beginning of the [stored string][1] + * or immediately after a newline: + * + * scanner = StringScanner.new(MULTILINE_TEXT) + * scanner.string + * # => "Go placidly amid the noise and haste,\nand remember what peace there may be in silence.\n" + * scanner.pos # => 0 + * scanner.beginning_of_line? # => true + * + * scanner.scan_until(/,/) # => "Go placidly amid the noise and haste," + * scanner.beginning_of_line? # => false + * + * scanner.scan(/\n/) # => "\n" + * scanner.beginning_of_line? # => true + * + * scanner.terminate + * scanner.beginning_of_line? # => true + * + * scanner.concat('x') + * scanner.terminate + * scanner.beginning_of_line? # => false + * + * StringScanner#bol? is an alias for StringScanner#beginning_of_line?. */ static VALUE strscan_bol_p(VALUE self) @@ -1084,14 +1355,24 @@ strscan_bol_p(VALUE self) } /* - * Returns +true+ if the scan pointer is at the end of the string. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * eos? -> true or false + * + * Returns whether the [position][2] + * is at the end of the [stored string][1]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.eos? # => false + * pos = 3 + * scanner.eos? # => false + * scanner.terminate + * scanner.eos? # => true + * ``` * - * s = StringScanner.new('test string') - * p s.eos? # => false - * s.scan(/test/) - * p s.eos? # => false - * s.terminate - * p s.eos? # => true */ static VALUE strscan_eos_p(VALUE self) @@ -1103,9 +1384,14 @@ strscan_eos_p(VALUE self) } /* + * call-seq: + * empty? + * * Equivalent to #eos?. * This method is obsolete, use #eos? instead. */ + + /* :nodoc: */ static VALUE strscan_empty_p(VALUE self) { @@ -1114,6 +1400,9 @@ strscan_empty_p(VALUE self) } /* + * call-seq: + * rest? + * * Returns true if and only if there is more data in the string. See #eos?. * This method is obsolete; use #eos? instead. * @@ -1122,6 +1411,8 @@ strscan_empty_p(VALUE self) * s.eos? # => false * s.rest? # => true */ + + /* :nodoc: */ static VALUE strscan_rest_p(VALUE self) { @@ -1132,13 +1423,26 @@ strscan_rest_p(VALUE self) } /* - * Returns +true+ if and only if the last match was successful. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * matched? -> true or false + * + * Returns `true` of the most recent [match attempt][17] was successful, + * `false` otherwise; + * see [Basic Matched Values][18]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.matched? # => false + * scanner.pos = 3 + * scanner.exist?(/baz/) # => 6 + * scanner.matched? # => true + * scanner.exist?(/nope/) # => nil + * scanner.matched? # => false + * ``` * - * s = StringScanner.new('test string') - * s.match?(/\w+/) # => 4 - * s.matched? # => true - * s.match?(/\d+/) # => nil - * s.matched? # => false */ static VALUE strscan_matched_p(VALUE self) @@ -1150,11 +1454,27 @@ strscan_matched_p(VALUE self) } /* - * Returns the last matched string. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * matched -> matched_substring or nil + * + * Returns the matched substring from the most recent [match][17] attempt + * if it was successful, + * or `nil` otherwise; + * see [Basic Matched Values][18]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.matched # => nil + * scanner.pos = 3 + * scanner.match?(/bar/) # => 3 + * scanner.matched # => "bar" + * scanner.match?(/nope/) # => nil + * scanner.matched # => nil + * ``` * - * s = StringScanner.new('test string') - * s.match?(/\w+/) # -> 4 - * s.matched # -> "test" */ static VALUE strscan_matched(VALUE self) @@ -1169,15 +1489,29 @@ strscan_matched(VALUE self) } /* - * Returns the size of the most recent match in bytes, or +nil+ if there - * was no recent match. This is different than <tt>matched.size</tt>, - * which will return the size in characters. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * matched_size -> substring_size or nil + * + * Returns the size (in bytes) of the matched substring + * from the most recent match [match attempt][17] if it was successful, + * or `nil` otherwise; + * see [Basic Matched Values][18]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.matched_size # => nil + * + * pos = 3 + * scanner.exist?(/baz/) # => 9 + * scanner.matched_size # => 3 + * + * scanner.exist?(/nope/) # => nil + * scanner.matched_size # => nil + * ``` * - * s = StringScanner.new('test string') - * s.check /\w+/ # -> "test" - * s.matched_size # -> 4 - * s.check /\d+/ # -> nil - * s.matched_size # -> nil */ static VALUE strscan_matched_size(VALUE self) @@ -1208,30 +1542,75 @@ name_to_backref_number(struct re_registers *regs, VALUE regexp, const char* name } /* - * call-seq: [](n) - * - * Returns the n-th subgroup in the most recent match. - * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan(/(\w+) (\w+) (\d+) /) # -> "Fri Dec 12 " - * s[0] # -> "Fri Dec 12 " - * s[1] # -> "Fri" - * s[2] # -> "Dec" - * s[3] # -> "12" - * s.post_match # -> "1975 14:39" - * s.pre_match # -> "" - * - * s.reset - * s.scan(/(?<wday>\w+) (?<month>\w+) (?<day>\d+) /) # -> "Fri Dec 12 " - * s[0] # -> "Fri Dec 12 " - * s[1] # -> "Fri" - * s[2] # -> "Dec" - * s[3] # -> "12" - * s[:wday] # -> "Fri" - * s[:month] # -> "Dec" - * s[:day] # -> "12" - * s.post_match # -> "1975 14:39" - * s.pre_match # -> "" + * + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * [](specifier) -> substring or nil + * + * Returns a captured substring or `nil`; + * see [Captured Match Values][13]. + * + * When there are captures: + * + * ``` + * scanner = StringScanner.new('Fri Dec 12 1975 14:39') + * scanner.scan(/(?<wday>\w+) (?<month>\w+) (?<day>\d+) /) + * ``` + * + * - `specifier` zero: returns the entire matched substring: + * + * ``` + * scanner[0] # => "Fri Dec 12 " + * scanner.pre_match # => "" + * scanner.post_match # => "1975 14:39" + * ``` + * + * - `specifier` positive integer. returns the `n`th capture, or `nil` if out of range: + * + * ``` + * scanner[1] # => "Fri" + * scanner[2] # => "Dec" + * scanner[3] # => "12" + * scanner[4] # => nil + * ``` + * + * - `specifier` negative integer. counts backward from the last subgroup: + * + * ``` + * scanner[-1] # => "12" + * scanner[-4] # => "Fri Dec 12 " + * scanner[-5] # => nil + * ``` + * + * - `specifier` symbol or string. returns the named subgroup, or `nil` if no such: + * + * ``` + * scanner[:wday] # => "Fri" + * scanner['wday'] # => "Fri" + * scanner[:month] # => "Dec" + * scanner[:day] # => "12" + * scanner[:nope] # => nil + * ``` + * + * When there are no captures, only `[0]` returns non-`nil`: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.exist?(/bar/) + * scanner[0] # => "bar" + * scanner[1] # => nil + * ``` + * + * For a failed match, even `[0]` returns `nil`: + * + * ``` + * scanner.scan(/nope/) # => nil + * scanner[0] # => nil + * scanner[1] # => nil + * ``` + * */ static VALUE strscan_aref(VALUE self, VALUE idx) @@ -1268,14 +1647,28 @@ strscan_aref(VALUE self, VALUE idx) } /* - * call-seq: size + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * size -> captures_count + * + * Returns the count of captures if the most recent match attempt succeeded, `nil` otherwise; + * see [Captures Match Values][13]: * - * Returns the amount of subgroups in the most recent match. - * The full match counts as a subgroup. + * ``` + * scanner = StringScanner.new('Fri Dec 12 1975 14:39') + * scanner.size # => nil + * + * pattern = /(?<wday>\w+) (?<month>\w+) (?<day>\d+) / + * scanner.match?(pattern) + * scanner.values_at(*0..scanner.size) # => ["Fri Dec 12 ", "Fri", "Dec", "12", nil] + * scanner.size # => 4 + * + * scanner.match?(/nope/) # => nil + * scanner.size # => nil + * ``` * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan(/(\w+) (\w+) (\d+) /) # -> "Fri Dec 12 " - * s.size # -> 4 */ static VALUE strscan_size(VALUE self) @@ -1288,16 +1681,30 @@ strscan_size(VALUE self) } /* - * call-seq: captures + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * captures -> substring_array or nil + * + * Returns the array of [captured match values][13] at indexes `(1..)` + * if the most recent match attempt succeeded, or `nil` otherwise: + * + * ``` + * scanner = StringScanner.new('Fri Dec 12 1975 14:39') + * scanner.captures # => nil * - * Returns the subgroups in the most recent match (not including the full match). - * If nothing was priorly matched, it returns nil. + * scanner.exist?(/(?<wday>\w+) (?<month>\w+) (?<day>\d+) /) + * scanner.captures # => ["Fri", "Dec", "12"] + * scanner.values_at(*0..4) # => ["Fri Dec 12 ", "Fri", "Dec", "12", nil] + * + * scanner.exist?(/Fri/) + * scanner.captures # => [] + * + * scanner.scan(/nope/) + * scanner.captures # => nil + * ``` * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan(/(\w+) (\w+) (\d+) (1980)?/) # -> "Fri Dec 12 " - * s.captures # -> ["Fri", "Dec", "12", nil] - * s.scan(/(\w+) (\w+) (\d+) (1980)?/) # -> nil - * s.captures # -> nil */ static VALUE strscan_captures(VALUE self) @@ -1327,17 +1734,25 @@ strscan_captures(VALUE self) } /* - * call-seq: - * scanner.values_at( i1, i2, ... iN ) -> an_array + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * values_at(*specifiers) -> array_of_captures or nil * - * Returns the subgroups in the most recent match at the given indices. - * If nothing was priorly matched, it returns nil. + * Returns an array of captured substrings, or `nil` of none. + * + * For each `specifier`, the returned substring is `[specifier]`; + * see #[]. + * + * ``` + * scanner = StringScanner.new('Fri Dec 12 1975 14:39') + * pattern = /(?<wday>\w+) (?<month>\w+) (?<day>\d+) / + * scanner.match?(pattern) + * scanner.values_at(*0..3) # => ["Fri Dec 12 ", "Fri", "Dec", "12"] + * scanner.values_at(*%i[wday month day]) # => ["Fri", "Dec", "12"] + * ``` * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.scan(/(\w+) (\w+) (\d+) /) # -> "Fri Dec 12 " - * s.values_at 0, -1, 5, 2 # -> ["Fri Dec 12 ", "12", nil, "Dec"] - * s.scan(/(\w+) (\w+) (\d+) /) # -> nil - * s.values_at 0, -1, 5, 2 # -> nil */ static VALUE @@ -1359,13 +1774,29 @@ strscan_values_at(int argc, VALUE *argv, VALUE self) } /* - * Returns the <i><b>pre</b>-match</i> (in the regular expression sense) of the last scan. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * pre_match -> substring + * + * Returns the substring that precedes the matched substring + * from the most recent match attempt if it was successful, + * or `nil` otherwise; + * see [Basic Match Values][18]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.pre_match # => nil + * + * scanner.pos = 3 + * scanner.exist?(/baz/) # => 6 + * scanner.pre_match # => "foobar" # Substring of entire string, not just target string. + * + * scanner.exist?(/nope/) # => nil + * scanner.pre_match # => nil + * ``` * - * s = StringScanner.new('test string') - * s.scan(/\w+/) # -> "test" - * s.scan(/\s+/) # -> " " - * s.pre_match # -> "test" - * s.post_match # -> "string" */ static VALUE strscan_pre_match(VALUE self) @@ -1380,13 +1811,29 @@ strscan_pre_match(VALUE self) } /* - * Returns the <i><b>post</b>-match</i> (in the regular expression sense) of the last scan. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * post_match -> substring + * + * Returns the substring that follows the matched substring + * from the most recent match attempt if it was successful, + * or `nil` otherwise; + * see [Basic Match Values][18]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.post_match # => nil + * + * scanner.pos = 3 + * scanner.match?(/bar/) # => 3 + * scanner.post_match # => "baz" + * + * scanner.match?(/nope/) # => nil + * scanner.post_match # => nil + * ``` * - * s = StringScanner.new('test string') - * s.scan(/\w+/) # -> "test" - * s.scan(/\s+/) # -> " " - * s.pre_match # -> "test" - * s.post_match # -> "string" */ static VALUE strscan_post_match(VALUE self) @@ -1401,8 +1848,24 @@ strscan_post_match(VALUE self) } /* - * Returns the "rest" of the string (i.e. everything after the scan pointer). - * If there is no more data (eos? = true), it returns <tt>""</tt>. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * rest -> target_substring + * + * Returns the 'rest' of the [stored string][1] (all after the current [position][2]), + * which is the [target substring][3]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.rest # => "foobarbaz" + * scanner.pos = 3 + * scanner.rest # => "barbaz" + * scanner.terminate + * scanner.rest # => "" + * ``` + * */ static VALUE strscan_rest(VALUE self) @@ -1417,7 +1880,26 @@ strscan_rest(VALUE self) } /* - * <tt>s.rest_size</tt> is equivalent to <tt>s.rest.size</tt>. + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * rest_size -> integer + * + * Returns the size (in bytes) of the #rest of the [stored string][1]: + * + * ``` + * scanner = StringScanner.new('foobarbaz') + * scanner.rest # => "foobarbaz" + * scanner.rest_size # => 9 + * scanner.pos = 3 + * scanner.rest # => "barbaz" + * scanner.rest_size # => 6 + * scanner.terminate + * scanner.rest # => "" + * scanner.rest_size # => 0 + * ``` + * */ static VALUE strscan_rest_size(VALUE self) @@ -1434,9 +1916,14 @@ strscan_rest_size(VALUE self) } /* + * call-seq: + * restsize + * * <tt>s.restsize</tt> is equivalent to <tt>s.rest_size</tt>. * This method is obsolete; use #rest_size instead. */ + + /* :nodoc: */ static VALUE strscan_restsize(VALUE self) { @@ -1447,15 +1934,39 @@ strscan_restsize(VALUE self) #define INSPECT_LENGTH 5 /* - * Returns a string that represents the StringScanner object, showing: - * - the current position - * - the size of the string - * - the characters surrounding the scan pointer - * - * s = StringScanner.new("Fri Dec 12 1975 14:39") - * s.inspect # -> '#<StringScanner 0/21 @ "Fri D...">' - * s.scan_until /12/ # -> "Fri Dec 12" - * s.inspect # -> '#<StringScanner 10/21 "...ec 12" @ " 1975...">' + * :markup: markdown + * :include: strscan/link_refs.txt + * + * call-seq: + * inspect -> string + * + * Returns a string representation of `self` that may show: + * + * 1. The current [position][2]. + * 2. The size (in bytes) of the [stored string][1]. + * 3. The substring preceding the current position. + * 4. The substring following the current position (which is also the [target substring][3]). + * + * ``` + * scanner = StringScanner.new("Fri Dec 12 1975 14:39") + * scanner.pos = 11 + * scanner.inspect # => "#<StringScanner 11/21 \"...c 12 \" @ \"1975 ...\">" + * ``` + * + * If at beginning-of-string, item 4 above (following substring) is omitted: + * + * ``` + * scanner.reset + * scanner.inspect # => "#<StringScanner 0/21 @ \"Fri D...\">" + * ``` + * + * If at end-of-string, all items above are omitted: + * + * ``` + * scanner.terminate + * scanner.inspect # => "#<StringScanner fin>" + * ``` + * */ static VALUE strscan_inspect(VALUE self) @@ -1527,13 +2038,13 @@ inspect2(struct strscanner *p) } /* - * call-seq: - * scanner.fixed_anchor? -> true or false + * :markup: markdown + * :include: strscan/link_refs.txt * - * Whether +scanner+ uses fixed anchor mode or not. + * call-seq: + * fixed_anchor? -> true or false * - * If fixed anchor mode is used, +\A+ always matches the beginning of - * the string. Otherwise, +\A+ always matches the current position. + * Returns whether the [fixed-anchor property][10] is set. */ static VALUE strscan_fixed_anchor_p(VALUE self) @@ -1569,14 +2080,32 @@ named_captures_iter(const OnigUChar *name, } /* + * :markup: markdown + * :include: strscan/link_refs.txt + * * call-seq: - * scanner.named_captures -> hash + * named_captures -> hash * - * Returns a hash of string variables matching the regular expression. + * Returns the array of captured match values at indexes (1..) + * if the most recent match attempt succeeded, or nil otherwise; + * see [Captured Match Values][13]: + * + * ``` + * scanner = StringScanner.new('Fri Dec 12 1975 14:39') + * scanner.named_captures # => {} + * + * pattern = /(?<wday>\w+) (?<month>\w+) (?<day>\d+) / + * scanner.match?(pattern) + * scanner.named_captures # => {"wday"=>"Fri", "month"=>"Dec", "day"=>"12"} + * + * scanner.string = 'nope' + * scanner.match?(pattern) + * scanner.named_captures # => {"wday"=>nil, "month"=>nil, "day"=>nil} + * + * scanner.match?(/nosuch/) + * scanner.named_captures # => {} + * ``` * - * scan = StringScanner.new('foobarbaz') - * scan.match?(/(?<f>foo)(?<r>bar)(?<z>baz)/) - * scan.named_captures # -> {"f"=>"foo", "r"=>"bar", "z"=>"baz"} */ static VALUE strscan_named_captures(VALUE self) @@ -1600,109 +2129,11 @@ strscan_named_captures(VALUE self) /* * Document-class: StringScanner * - * StringScanner provides for lexical scanning operations on a String. Here is - * an example of its usage: - * - * require 'strscan' - * - * s = StringScanner.new('This is an example string') - * s.eos? # -> false - * - * p s.scan(/\w+/) # -> "This" - * p s.scan(/\w+/) # -> nil - * p s.scan(/\s+/) # -> " " - * p s.scan(/\s+/) # -> nil - * p s.scan(/\w+/) # -> "is" - * s.eos? # -> false - * - * p s.scan(/\s+/) # -> " " - * p s.scan(/\w+/) # -> "an" - * p s.scan(/\s+/) # -> " " - * p s.scan(/\w+/) # -> "example" - * p s.scan(/\s+/) # -> " " - * p s.scan(/\w+/) # -> "string" - * s.eos? # -> true - * - * p s.scan(/\s+/) # -> nil - * p s.scan(/\w+/) # -> nil - * - * Scanning a string means remembering the position of a <i>scan pointer</i>, - * which is just an index. The point of scanning is to move forward a bit at - * a time, so matches are sought after the scan pointer; usually immediately - * after it. - * - * Given the string "test string", here are the pertinent scan pointer - * positions: - * - * t e s t s t r i n g - * 0 1 2 ... 1 - * 0 - * - * When you #scan for a pattern (a regular expression), the match must occur - * at the character after the scan pointer. If you use #scan_until, then the - * match can occur anywhere after the scan pointer. In both cases, the scan - * pointer moves <i>just beyond</i> the last character of the match, ready to - * scan again from the next character onwards. This is demonstrated by the - * example above. - * - * == Method Categories - * - * There are other methods besides the plain scanners. You can look ahead in - * the string without actually scanning. You can access the most recent match. - * You can modify the string being scanned, reset or terminate the scanner, - * find out or change the position of the scan pointer, skip ahead, and so on. - * - * === Advancing the Scan Pointer - * - * - #getch - * - #get_byte - * - #scan_byte - * - #scan - * - #scan_until - * - #skip - * - #skip_until - * - * === Looking Ahead - * - * - #check - * - #check_until - * - #exist? - * - #match? - * - #peek - * - #peek_byte - * - * === Finding Where we Are - * - * - #beginning_of_line? (<tt>#bol?</tt>) - * - #eos? - * - #rest? - * - #rest_size - * - #pos - * - * === Setting Where we Are - * - * - #reset - * - #terminate - * - #pos= - * - * === Match Data - * - * - #matched - * - #matched? - * - #matched_size - * - <tt>#[]</tt> - * - #pre_match - * - #post_match - * - * === Miscellaneous - * - * - <tt><<</tt> - * - #concat - * - #string - * - #string= - * - #unscan - * - * There are aliases to several of the methods. + * :markup: markdown + * + * :include: strscan/link_refs.txt + * :include: strscan/strscan.md + * */ void Init_strscan(void) diff --git a/ext/strscan/strscan.gemspec b/ext/strscan/strscan.gemspec index 8a61c7abe6..925edcd2d3 100644 --- a/ext/strscan/strscan.gemspec +++ b/ext/strscan/strscan.gemspec @@ -29,6 +29,11 @@ Gem::Specification.new do |s| s.require_paths = %w{lib} files << "ext/strscan/extconf.rb" files << "ext/strscan/strscan.c" + s.rdoc_options << "-idoc" + s.extra_rdoc_files = [ + ".rdoc_options", + *Dir.glob("doc/strscan/**/*") + ] s.extensions = %w{ext/strscan/extconf.rb} end s.files = files diff --git a/ext/win32ole/win32ole.gemspec b/ext/win32ole/win32ole.gemspec index 625916ae17..425942d5a0 100644 --- a/ext/win32ole/win32ole.gemspec +++ b/ext/win32ole/win32ole.gemspec @@ -30,5 +30,6 @@ Gem::Specification.new do |spec| spec.files = IO.popen(%w[git ls-files -z --] + pathspecs, chdir: __dir__, err: IO::NULL, exception: false, &:read).split("\x0") spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.extensions = "ext/win32ole/extconf.rb" spec.require_paths = ["lib"] end diff --git a/ext/zlib/zlib.c b/ext/zlib/zlib.c index fe03072576..aad9f8d28a 100644 --- a/ext/zlib/zlib.c +++ b/ext/zlib/zlib.c @@ -90,7 +90,7 @@ static void zstream_expand_buffer_into(struct zstream*, unsigned long); static int zstream_expand_buffer_non_stream(struct zstream *z); static void zstream_append_buffer(struct zstream*, const Bytef*, long); static VALUE zstream_detach_buffer(struct zstream*); -static VALUE zstream_shift_buffer(struct zstream*, long); +static VALUE zstream_shift_buffer(struct zstream*, long, VALUE); static void zstream_buffer_ungets(struct zstream*, const Bytef*, unsigned long); static void zstream_buffer_ungetbyte(struct zstream*, int); static void zstream_append_input(struct zstream*, const Bytef*, long); @@ -170,8 +170,8 @@ static void gzfile_check_footer(struct gzfile*, VALUE outbuf); static void gzfile_write(struct gzfile*, Bytef*, long); static long gzfile_read_more(struct gzfile*, VALUE outbuf); static void gzfile_calc_crc(struct gzfile*, VALUE); -static VALUE gzfile_read(struct gzfile*, long); -static VALUE gzfile_read_all(struct gzfile*); +static VALUE gzfile_read(struct gzfile*, long, VALUE); +static VALUE gzfile_read_all(struct gzfile*, VALUE); static void gzfile_ungets(struct gzfile*, const Bytef*, long); static void gzfile_ungetbyte(struct gzfile*, int); static VALUE gzfile_writer_end_run(VALUE); @@ -820,19 +820,31 @@ zstream_detach_buffer(struct zstream *z) } static VALUE -zstream_shift_buffer(struct zstream *z, long len) +zstream_shift_buffer(struct zstream *z, long len, VALUE dst) { - VALUE dst; char *bufptr; long buflen = ZSTREAM_BUF_FILLED(z); if (buflen <= len) { - return zstream_detach_buffer(z); + if (NIL_P(dst) || (!ZSTREAM_IS_FINISHED(z) && !ZSTREAM_IS_GZFILE(z) && + rb_block_given_p())) { + return zstream_detach_buffer(z); + } else { + bufptr = RSTRING_PTR(z->buf); + rb_str_resize(dst, buflen); + memcpy(RSTRING_PTR(dst), bufptr, buflen); + } + buflen = 0; + } else { + bufptr = RSTRING_PTR(z->buf); + if (NIL_P(dst)) { + dst = rb_str_new(bufptr, len); + } else { + rb_str_resize(dst, len); + memcpy(RSTRING_PTR(dst), bufptr, len); + } + buflen -= len; } - - bufptr = RSTRING_PTR(z->buf); - dst = rb_str_new(bufptr, len); - buflen -= len; memmove(bufptr, bufptr + len, buflen); rb_str_set_len(z->buf, buflen); z->stream.next_out = (Bytef*)RSTRING_END(z->buf); @@ -2874,18 +2886,18 @@ gzfile_newstr(struct gzfile *gz, VALUE str) } static long -gzfile_fill(struct gzfile *gz, long len) +gzfile_fill(struct gzfile *gz, long len, VALUE outbuf) { if (len < 0) rb_raise(rb_eArgError, "negative length %ld given", len); if (len == 0) return 0; while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) < len) { - gzfile_read_more(gz, Qnil); + gzfile_read_more(gz, outbuf); } if (GZFILE_IS_FINISHED(gz)) { if (!(gz->z.flags & GZFILE_FLAG_FOOTER_FINISHED)) { - gzfile_check_footer(gz, Qnil); + gzfile_check_footer(gz, outbuf); } return -1; } @@ -2893,14 +2905,27 @@ gzfile_fill(struct gzfile *gz, long len) } static VALUE -gzfile_read(struct gzfile *gz, long len) +gzfile_read(struct gzfile *gz, long len, VALUE outbuf) { VALUE dst; - len = gzfile_fill(gz, len); - if (len == 0) return rb_str_new(0, 0); - if (len < 0) return Qnil; - dst = zstream_shift_buffer(&gz->z, len); + len = gzfile_fill(gz, len, outbuf); + + if (len < 0) { + if (!NIL_P(outbuf)) + rb_str_resize(outbuf, 0); + return Qnil; + } + if (len == 0) { + if (NIL_P(outbuf)) + return rb_str_new(0, 0); + else { + rb_str_resize(outbuf, 0); + return outbuf; + } + } + + dst = zstream_shift_buffer(&gz->z, len, outbuf); if (!NIL_P(dst)) gzfile_calc_crc(gz, dst); return dst; } @@ -2933,29 +2958,26 @@ gzfile_readpartial(struct gzfile *gz, long len, VALUE outbuf) rb_raise(rb_eEOFError, "end of file reached"); } - dst = zstream_shift_buffer(&gz->z, len); + dst = zstream_shift_buffer(&gz->z, len, outbuf); gzfile_calc_crc(gz, dst); - if (!NIL_P(outbuf)) { - rb_str_resize(outbuf, RSTRING_LEN(dst)); - memcpy(RSTRING_PTR(outbuf), RSTRING_PTR(dst), RSTRING_LEN(dst)); - dst = outbuf; - } return dst; } static VALUE -gzfile_read_all(struct gzfile *gz) +gzfile_read_all(struct gzfile *gz, VALUE dst) { - VALUE dst; - while (!ZSTREAM_IS_FINISHED(&gz->z)) { - gzfile_read_more(gz, Qnil); + gzfile_read_more(gz, dst); } if (GZFILE_IS_FINISHED(gz)) { if (!(gz->z.flags & GZFILE_FLAG_FOOTER_FINISHED)) { - gzfile_check_footer(gz, Qnil); + gzfile_check_footer(gz, dst); } + if (!NIL_P(dst)) { + rb_str_resize(dst, 0); + return dst; + } return rb_str_new(0, 0); } @@ -2993,7 +3015,7 @@ gzfile_getc(struct gzfile *gz) de = (unsigned char *)ds + GZFILE_CBUF_CAPA; (void)rb_econv_convert(gz->ec, &sp, se, &dp, de, ECONV_PARTIAL_INPUT|ECONV_AFTER_OUTPUT); rb_econv_check_error(gz->ec); - dst = zstream_shift_buffer(&gz->z, sp - ss); + dst = zstream_shift_buffer(&gz->z, sp - ss, Qnil); gzfile_calc_crc(gz, dst); rb_str_resize(cbuf, dp - ds); return cbuf; @@ -3001,7 +3023,7 @@ gzfile_getc(struct gzfile *gz) else { buf = gz->z.buf; len = rb_enc_mbclen(RSTRING_PTR(buf), RSTRING_END(buf), gz->enc); - dst = gzfile_read(gz, len); + dst = gzfile_read(gz, len, Qnil); if (NIL_P(dst)) return dst; return gzfile_newstr(gz, dst); } @@ -3909,7 +3931,7 @@ rb_gzreader_s_zcat(int argc, VALUE *argv, VALUE klass) if (!buf) { buf = rb_str_new(0, 0); } - tmpbuf = gzfile_read_all(get_gzfile(obj)); + tmpbuf = gzfile_read_all(get_gzfile(obj), Qnil); rb_str_cat(buf, RSTRING_PTR(tmpbuf), RSTRING_LEN(tmpbuf)); } @@ -4011,19 +4033,19 @@ static VALUE rb_gzreader_read(int argc, VALUE *argv, VALUE obj) { struct gzfile *gz = get_gzfile(obj); - VALUE vlen; + VALUE vlen, outbuf; long len; - rb_scan_args(argc, argv, "01", &vlen); + rb_scan_args(argc, argv, "02", &vlen, &outbuf); if (NIL_P(vlen)) { - return gzfile_read_all(gz); + return gzfile_read_all(gz, outbuf); } len = NUM2INT(vlen); if (len < 0) { rb_raise(rb_eArgError, "negative length %ld given", len); } - return gzfile_read(gz, len); + return gzfile_read(gz, len, outbuf); } /* @@ -4096,7 +4118,7 @@ rb_gzreader_getbyte(VALUE obj) struct gzfile *gz = get_gzfile(obj); VALUE dst; - dst = gzfile_read(gz, 1); + dst = gzfile_read(gz, 1, Qnil); if (!NIL_P(dst)) { dst = INT2FIX((unsigned int)(RSTRING_PTR(dst)[0]) & 0xff); } @@ -4217,7 +4239,7 @@ gzreader_skip_linebreaks(struct gzfile *gz) } } - str = zstream_shift_buffer(&gz->z, n - 1); + str = zstream_shift_buffer(&gz->z, n - 1, Qnil); gzfile_calc_crc(gz, str); } @@ -4238,7 +4260,7 @@ gzreader_charboundary(struct gzfile *gz, long n) if (l < n) { int n_bytes = rb_enc_precise_mbclen(p, e, gz->enc); if (MBCLEN_NEEDMORE_P(n_bytes)) { - if ((l = gzfile_fill(gz, n + MBCLEN_NEEDMORE_LEN(n_bytes))) > 0) { + if ((l = gzfile_fill(gz, n + MBCLEN_NEEDMORE_LEN(n_bytes), Qnil)) > 0) { return l; } } @@ -4290,10 +4312,10 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) if (NIL_P(rs)) { if (limit < 0) { - dst = gzfile_read_all(gz); + dst = gzfile_read_all(gz, Qnil); if (RSTRING_LEN(dst) == 0) return Qnil; } - else if ((n = gzfile_fill(gz, limit)) <= 0) { + else if ((n = gzfile_fill(gz, limit, Qnil)) <= 0) { return Qnil; } else { @@ -4303,7 +4325,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) else { n = limit; } - dst = zstream_shift_buffer(&gz->z, n); + dst = zstream_shift_buffer(&gz->z, n, Qnil); if (NIL_P(dst)) return dst; gzfile_calc_crc(gz, dst); dst = gzfile_newstr(gz, dst); @@ -4330,7 +4352,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) while (ZSTREAM_BUF_FILLED(&gz->z) < rslen) { if (ZSTREAM_IS_FINISHED(&gz->z)) { if (ZSTREAM_BUF_FILLED(&gz->z) > 0) gz->lineno++; - return gzfile_read(gz, rslen); + return gzfile_read(gz, rslen, Qnil); } gzfile_read_more(gz, Qnil); } @@ -4367,7 +4389,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) } gz->lineno++; - dst = gzfile_read(gz, n); + dst = gzfile_read(gz, n, Qnil); if (NIL_P(dst)) return dst; if (rspara) { gzreader_skip_linebreaks(gz); diff --git a/ext/zlib/zlib.gemspec b/ext/zlib/zlib.gemspec index bb67ea156c..345dc5f225 100644 --- a/ext/zlib/zlib.gemspec +++ b/ext/zlib/zlib.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |spec| spec.homepage = "https://github.com/ruby/zlib" spec.licenses = ["Ruby", "BSD-2-Clause"] - spec.files = ["LICENSE.txt", "README.md", "ext/zlib/extconf.rb", "ext/zlib/zlib.c", "zlib.gemspec"] + spec.files = ["COPYING", "BSDL", "README.md", "ext/zlib/extconf.rb", "ext/zlib/zlib.c", "zlib.gemspec"] spec.bindir = "exe" spec.executables = [] spec.require_paths = ["lib"] @@ -2840,14 +2840,16 @@ newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t si // Retry allocation after moving to new page obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); - - GC_ASSERT(obj != Qfalse); } } if (unlock_vm) { RB_VM_LOCK_LEAVE_CR_LEV(GET_RACTOR(), &lev); } + + if (UNLIKELY(obj == Qfalse)) { + rb_memerror(); + } } size_pool->total_allocated_objects++; diff --git a/gems/bundled_gems b/gems/bundled_gems index ad979c1c5c..38ddd97c89 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -6,26 +6,26 @@ # - revision: revision in repository-url to test # if `revision` is not given, "v"+`version` or `version` will be used. -minitest 5.22.3 https://github.com/minitest/minitest ea9caafc0754b1d6236a490d59e624b53209734a +minitest 5.23.1 https://github.com/minitest/minitest power_assert 2.0.3 https://github.com/ruby/power_assert 84e85124c5014a139af39161d484156cfe87a9ed rake 13.2.1 https://github.com/ruby/rake test-unit 3.6.2 https://github.com/test-unit/test-unit -rexml 3.2.6 https://github.com/ruby/rexml +rexml 3.2.8 https://github.com/ruby/rexml rss 0.3.0 https://github.com/ruby/rss -net-ftp 0.3.4 https://github.com/ruby/net-ftp -net-imap 0.4.10 https://github.com/ruby/net-imap +net-ftp 0.3.5 https://github.com/ruby/net-ftp +net-imap 0.4.12 https://github.com/ruby/net-imap net-pop 0.1.2 https://github.com/ruby/net-pop net-smtp 0.5.0 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime -rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd +rbs 3.5.1 https://github.com/ruby/rbs typeprof 0.21.11 https://github.com/ruby/typeprof b19a6416da3a05d57fadd6ffdadb382b6d236ca5 debug 1.9.2 https://github.com/ruby/debug -racc 1.7.3 https://github.com/ruby/racc +racc 1.8.0 https://github.com/ruby/racc mutex_m 0.2.0 https://github.com/ruby/mutex_m getoptlong 0.2.1 https://github.com/ruby/getoptlong base64 0.2.0 https://github.com/ruby/base64 -bigdecimal 3.1.7 https://github.com/ruby/bigdecimal +bigdecimal 3.1.8 https://github.com/ruby/bigdecimal observer 0.1.2 https://github.com/ruby/observer abbrev 0.1.2 https://github.com/ruby/abbrev resolv-replace 0.1.1 https://github.com/ruby/resolv-replace @@ -357,7 +357,9 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) ((VALUE *)env->ep)[VM_ENV_DATA_INDEX_ENV] = rb_gc_location(env->ep[VM_ENV_DATA_INDEX_ENV]); } else { - VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED); + if (!VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_WB_REQUIRED)) { + VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED); + } rb_gc_mark_movable( (VALUE)rb_vm_env_prev_env(env)); } } diff --git a/include/ruby/atomic.h b/include/ruby/atomic.h index 043a6a9945..d786df20c9 100644 --- a/include/ruby/atomic.h +++ b/include/ruby/atomic.h @@ -311,7 +311,7 @@ typedef unsigned int rb_atomic_t; * @retval otherwise Something else is at `var`; not updated. */ #define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \ - RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (oldval), (newval))) + RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (void *)(oldval), (void *)(newval))) /** * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are diff --git a/include/ruby/internal/arithmetic/long.h b/include/ruby/internal/arithmetic/long.h index 6b8fd8ffc3..6c00dbceb7 100644 --- a/include/ruby/internal/arithmetic/long.h +++ b/include/ruby/internal/arithmetic/long.h @@ -114,11 +114,11 @@ RB_INT2FIX(long i) /* :NOTE: VALUE can be wider than long. As j being unsigned, 2j+1 is fully * defined. Also it can be compiled into a single LEA instruction. */ - const unsigned long j = i; + const unsigned long j = RBIMPL_CAST((unsigned long)i); const unsigned long k = (j << 1) + RUBY_FIXNUM_FLAG; - const long l = k; + const long l = RBIMPL_CAST((long)k); const SIGNED_VALUE m = l; /* Sign extend */ - const VALUE n = m; + const VALUE n = RBIMPL_CAST((VALUE)m); RBIMPL_ASSERT_OR_ASSUME(RB_FIXNUM_P(n)); return n; @@ -166,7 +166,7 @@ rbimpl_fix2long_by_idiv(VALUE x) /* :NOTE: VALUE can be wider than long. (x-1)/2 never overflows because * RB_FIXNUM_P(x) holds. Also it has no portability issue like y>>1 * below. */ - const SIGNED_VALUE y = x - RUBY_FIXNUM_FLAG; + const SIGNED_VALUE y = RBIMPL_CAST((SIGNED_VALUE)(x - RUBY_FIXNUM_FLAG)); const SIGNED_VALUE z = y / 2; const long w = RBIMPL_CAST((long)z); @@ -193,7 +193,7 @@ rbimpl_fix2long_by_shift(VALUE x) /* :NOTE: VALUE can be wider than long. If right shift is arithmetic, this * is noticeably faster than above. */ - const SIGNED_VALUE y = x; + const SIGNED_VALUE y = RBIMPL_CAST((SIGNED_VALUE)x); const SIGNED_VALUE z = y >> 1; const long w = RBIMPL_CAST((long)z); @@ -252,7 +252,7 @@ static inline unsigned long rb_fix2ulong(VALUE x) { RBIMPL_ASSERT_OR_ASSUME(RB_FIXNUM_P(x)); - return rb_fix2long(x); + return RBIMPL_CAST((unsigned long)rb_fix2long(x)); } /** @@ -323,7 +323,7 @@ static inline VALUE rb_ulong2num_inline(unsigned long v) { if (RB_POSFIXABLE(v)) - return RB_LONG2FIX(v); + return RB_LONG2FIX(RBIMPL_CAST((long)v)); else return rb_uint2big(v); } diff --git a/include/ruby/internal/arithmetic/long_long.h b/include/ruby/internal/arithmetic/long_long.h index 65dec8729d..aab455c830 100644 --- a/include/ruby/internal/arithmetic/long_long.h +++ b/include/ruby/internal/arithmetic/long_long.h @@ -127,7 +127,7 @@ static inline unsigned LONG_LONG rb_num2ull_inline(VALUE x) { if (RB_FIXNUM_P(x)) - return RB_FIX2LONG(x); + return RBIMPL_CAST((unsigned LONG_LONG)RB_FIX2LONG(x)); else return rb_num2ull(x); } diff --git a/include/ruby/internal/arithmetic/st_data_t.h b/include/ruby/internal/arithmetic/st_data_t.h index 3bff4ffc0b..91776b3fce 100644 --- a/include/ruby/internal/arithmetic/st_data_t.h +++ b/include/ruby/internal/arithmetic/st_data_t.h @@ -58,7 +58,7 @@ RBIMPL_ATTR_ARTIFICIAL() static inline VALUE RB_ST2FIX(st_data_t i) { - SIGNED_VALUE x = i; + SIGNED_VALUE x = RBIMPL_CAST((SIGNED_VALUE)i); if (x >= 0) { x &= RUBY_FIXNUM_MAX; @@ -69,7 +69,7 @@ RB_ST2FIX(st_data_t i) RBIMPL_ASSERT_OR_ASSUME(RB_FIXABLE(x)); unsigned long y = RBIMPL_CAST((unsigned long)x); - return RB_LONG2FIX(y); + return RB_LONG2FIX(RBIMPL_CAST((long)y)); } #endif /* RBIMPL_ARITHMETIC_ST_DATA_T_H */ diff --git a/include/ruby/internal/core/rdata.h b/include/ruby/internal/core/rdata.h index 43ab3c01e7..e4c146a716 100644 --- a/include/ruby/internal/core/rdata.h +++ b/include/ruby/internal/core/rdata.h @@ -37,12 +37,8 @@ #include "ruby/defines.h" /** @cond INTERNAL_MACRO */ -#ifdef RUBY_UNTYPED_DATA_WARNING -# /* Take that. */ -#elif defined(RUBY_EXPORT) -# define RUBY_UNTYPED_DATA_WARNING 1 -#else -# define RUBY_UNTYPED_DATA_WARNING 0 +#ifndef RUBY_UNTYPED_DATA_WARNING +#define RUBY_UNTYPED_DATA_WARNING 1 #endif #define RBIMPL_DATA_FUNC(f) RBIMPL_CAST((void (*)(void *))(f)) @@ -331,15 +327,6 @@ rb_data_object_get_warning(VALUE obj) return rb_data_object_get(obj); } -#if defined(HAVE_BUILTIN___BUILTIN_CHOOSE_EXPR_CONSTANT_P) -# define rb_data_object_wrap_warning(klass, ptr, mark, free) \ - RB_GNUC_EXTENSION( \ - __builtin_choose_expr( \ - __builtin_constant_p(klass) && !(klass), \ - rb_data_object_wrap(klass, ptr, mark, free), \ - (rb_data_object_wrap_warning)(klass, ptr, mark, free))) -#endif - /** * This is an implementation detail of #Data_Make_Struct. People don't use it * directly. diff --git a/include/ruby/internal/encoding/encoding.h b/include/ruby/internal/encoding/encoding.h index a680651a81..a58f9f2b15 100644 --- a/include/ruby/internal/encoding/encoding.h +++ b/include/ruby/internal/encoding/encoding.h @@ -80,7 +80,7 @@ enum ruby_encoding_consts { static inline void RB_ENCODING_SET_INLINED(VALUE obj, int encindex) { - VALUE f = /* upcast */ encindex; + VALUE f = /* upcast */ RBIMPL_CAST((VALUE)encindex); f <<= RUBY_ENCODING_SHIFT; RB_FL_UNSET_RAW(obj, RUBY_ENCODING_MASK); diff --git a/include/ruby/internal/intern/error.h b/include/ruby/internal/intern/error.h index 11e147a121..2ca51d0111 100644 --- a/include/ruby/internal/intern/error.h +++ b/include/ruby/internal/intern/error.h @@ -190,6 +190,7 @@ RBIMPL_ATTR_NONNULL(()) */ void rb_error_frozen(const char *what); +RBIMPL_ATTR_NORETURN() /** * Identical to rb_error_frozen(), except it takes arbitrary Ruby object * instead of C's string. @@ -243,24 +244,10 @@ RBIMPL_SYMBOL_EXPORT_END() * * Does anyone use this? Remain not deleted for compatibility. */ -#define rb_check_frozen_internal(obj) do { \ - VALUE frozen_obj = (obj); \ - if (RB_UNLIKELY(RB_OBJ_FROZEN(frozen_obj))) { \ - rb_error_frozen_object(frozen_obj); \ - } \ - } while (0) - -/** @alias{rb_check_frozen} */ -static inline void -rb_check_frozen_inline(VALUE obj) -{ - if (RB_UNLIKELY(RB_OBJ_FROZEN(obj))) { - rb_error_frozen_object(obj); - } -} +#define rb_check_frozen_internal rb_check_frozen /** @alias{rb_check_frozen} */ -#define rb_check_frozen rb_check_frozen_inline +#define rb_check_frozen_inline rb_check_frozen /** * Ensures that the passed integer is in the passed range. When you can use diff --git a/include/ruby/internal/intern/string.h b/include/ruby/internal/intern/string.h index 6827563e8d..37dee45527 100644 --- a/include/ruby/internal/intern/string.h +++ b/include/ruby/internal/intern/string.h @@ -602,21 +602,6 @@ VALUE rb_str_dup(VALUE str); VALUE rb_str_resurrect(VALUE str); /** - * Returns whether a string is chilled or not. - * - * This function is temporary and users must check for its presence using - * #ifdef HAVE_RB_STR_CHILLED_P. If HAVE_RB_STR_CHILLED_P is not defined, then - * strings can't be chilled. - * - * @param[in] str A string. - * @retval 1 The string is chilled. - * @retval 0 Otherwise. - */ -bool rb_str_chilled_p(VALUE str); - -#define HAVE_RB_STR_CHILLED_P 1 - -/** * Obtains a "temporary lock" of the string. This advisory locking mechanism * prevents other cooperating threads from tampering the receiver. The same * thing could be done via freeze mechanism, but this one can also be unlocked diff --git a/include/ruby/internal/memory.h b/include/ruby/internal/memory.h index 270cc1ac8b..64e850c65e 100644 --- a/include/ruby/internal/memory.h +++ b/include/ruby/internal/memory.h @@ -643,7 +643,7 @@ rbimpl_size_mul_or_raise(size_t x, size_t y) static inline void * rb_alloc_tmp_buffer2(volatile VALUE *store, long count, size_t elsize) { - const size_t total_size = rbimpl_size_mul_or_raise(count, elsize); + const size_t total_size = rbimpl_size_mul_or_raise(RBIMPL_CAST((size_t)count), elsize); const size_t cnt = (total_size + sizeof(VALUE) - 1) / sizeof(VALUE); return rb_alloc_tmp_buffer_with_count(store, total_size, cnt); } diff --git a/include/ruby/internal/special_consts.h b/include/ruby/internal/special_consts.h index dc0a6b41d6..85579e33f0 100644 --- a/include/ruby/internal/special_consts.h +++ b/include/ruby/internal/special_consts.h @@ -156,7 +156,7 @@ RB_TEST(VALUE obj) * * RTEST(v) can be 0 if and only if (v == Qfalse || v == Qnil). */ - return obj & ~RUBY_Qnil; + return obj & RBIMPL_CAST((VALUE)~RUBY_Qnil); } RBIMPL_ATTR_CONST() @@ -226,7 +226,7 @@ RB_NIL_OR_UNDEF_P(VALUE obj) * * NIL_OR_UNDEF_P(v) can be true only when v is Qundef or Qnil. */ - const VALUE mask = ~(RUBY_Qundef ^ RUBY_Qnil); + const VALUE mask = RBIMPL_CAST((VALUE)~(RUBY_Qundef ^ RUBY_Qnil)); const VALUE common_bits = RUBY_Qundef & RUBY_Qnil; return (obj & mask) == common_bits; } diff --git a/include/ruby/internal/value_type.h b/include/ruby/internal/value_type.h index 977f60a009..557f18813b 100644 --- a/include/ruby/internal/value_type.h +++ b/include/ruby/internal/value_type.h @@ -443,7 +443,7 @@ Check_Type(VALUE v, enum ruby_value_type t) } unexpected_type: - rb_unexpected_type(v, t); + rb_unexpected_type(v, RBIMPL_CAST((int)t)); } #endif /* RBIMPL_VALUE_TYPE_H */ @@ -222,7 +222,7 @@ setinstancevariable (ID id, IVC ic) (VALUE val) () -// attr bool leaf = false; /* has rb_check_frozen_internal() */ +// attr bool leaf = false; /* has rb_check_frozen() */ { vm_setinstancevariable(GET_ISEQ(), GET_SELF(), id, val, ic); } @@ -977,6 +977,9 @@ opt_newarray_send case idMax: val = vm_opt_newarray_max(ec, num, STACK_ADDR_FROM_TOP(num)); break; + case idPack: + val = rb_vm_opt_newarray_pack(ec, (long)num-1, STACK_ADDR_FROM_TOP(num), TOPN(0)); + break; default: rb_bug("unreachable"); } diff --git a/internal/basic_operators.h b/internal/basic_operators.h index a59403631e..4732a65403 100644 --- a/internal/basic_operators.h +++ b/internal/basic_operators.h @@ -37,6 +37,7 @@ enum ruby_basic_operators { BOP_OR, BOP_CMP, BOP_DEFAULT, + BOP_PACK, BOP_LAST_ }; diff --git a/internal/error.h b/internal/error.h index 7e41f134d7..7a4daca6b3 100644 --- a/internal/error.h +++ b/internal/error.h @@ -142,6 +142,8 @@ VALUE rb_syntax_error_append(VALUE, VALUE, int, int, rb_encoding*, const char*, PRINTF_ARGS(void rb_enc_warn(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_sys_enc_warning(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_syserr_enc_warning(int err, rb_encoding *enc, const char *fmt, ...), 3, 4); +PRINTF_ARGS(void rb_enc_compile_warning(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); +PRINTF_ARGS(void rb_enc_compile_warn(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); rb_warning_category_t rb_warning_category_from_name(VALUE category); bool rb_warning_category_enabled_p(rb_warning_category_t category); VALUE rb_name_err_new(VALUE mesg, VALUE recv, VALUE method); diff --git a/internal/parse.h b/internal/parse.h index a7e8e08912..4a9c4acf8a 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -13,7 +13,7 @@ #include "internal/static_assert.h" #ifdef UNIVERSAL_PARSER -#define rb_encoding void +#define rb_encoding const void #endif struct rb_iseq_struct; /* in vm_core.h */ @@ -55,7 +55,7 @@ void rb_ruby_parser_set_script_lines(rb_parser_t *p); void rb_ruby_parser_error_tolerant(rb_parser_t *p); void rb_ruby_parser_keep_tokens(rb_parser_t *p); typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int); -rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line); +rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line); RUBY_SYMBOL_EXPORT_BEGIN diff --git a/internal/string.h b/internal/string.h index fb37f73114..3533766ffb 100644 --- a/internal/string.h +++ b/internal/string.h @@ -19,6 +19,10 @@ #define STR_SHARED FL_USER2 /* = ELTS_SHARED */ #define STR_CHILLED FL_USER3 +enum ruby_rstring_private_flags { + RSTRING_CHILLED = STR_CHILLED, +}; + #ifdef rb_fstring_cstr # undef rb_fstring_cstr #endif @@ -76,6 +80,7 @@ VALUE rb_str_concat_literals(size_t num, const VALUE *strary); VALUE rb_str_eql(VALUE str1, VALUE str2); VALUE rb_id_quote_unprintable(ID); VALUE rb_sym_proc_call(ID mid, int argc, const VALUE *argv, int kw_splat, VALUE passed_proc); +VALUE rb_enc_literal_str(const char *ptr, long len, rb_encoding *enc); struct rb_execution_context_struct; VALUE rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chilled); @@ -118,15 +123,14 @@ CHILLED_STRING_P(VALUE obj) static inline void CHILLED_STRING_MUTATED(VALUE str) { + FL_UNSET_RAW(str, STR_CHILLED); rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "literal string will be frozen in the future"); - FL_UNSET_RAW(str, STR_CHILLED | FL_FREEZE); } static inline void STR_CHILL_RAW(VALUE str) { - // Chilled strings are always also frozen - FL_SET_RAW(str, STR_CHILLED | RUBY_FL_FREEZE); + FL_SET_RAW(str, STR_CHILLED); } static inline bool @@ -8026,37 +8026,18 @@ ruby_popen_writer(char *const *argv, rb_pid_t *pid) return NULL; } -static void -rb_scan_open_args(int argc, const VALUE *argv, - VALUE *fname_p, int *oflags_p, int *fmode_p, - struct rb_io_encoding *convconfig_p, mode_t *perm_p) +static VALUE +rb_open_file(VALUE io, VALUE fname, VALUE vmode, VALUE vperm, VALUE opt) { - VALUE opt, fname, vmode, vperm; + struct rb_io_encoding convconfig; int oflags, fmode; mode_t perm; - argc = rb_scan_args(argc, argv, "12:", &fname, &vmode, &vperm, &opt); FilePathValue(fname); - rb_io_extract_modeenc(&vmode, &vperm, opt, &oflags, &fmode, convconfig_p); - - perm = NIL_P(vperm) ? 0666 : NUM2MODET(vperm); - - *fname_p = fname; - *oflags_p = oflags; - *fmode_p = fmode; - *perm_p = perm; -} - -static VALUE -rb_open_file(int argc, const VALUE *argv, VALUE io) -{ - VALUE fname; - int oflags, fmode; - struct rb_io_encoding convconfig; - mode_t perm; + rb_io_extract_modeenc(&vmode, &vperm, opt, &oflags, &fmode, &convconfig); + perm = NIL_P(vperm) ? 0666 : NUM2MODET(vperm); - rb_scan_open_args(argc, argv, &fname, &oflags, &fmode, &convconfig, &perm); rb_file_open_generic(io, fname, oflags, fmode, &convconfig, perm); return io; @@ -9379,6 +9360,8 @@ rb_io_make_open_file(VALUE obj) return fp; } +static VALUE io_initialize(VALUE io, VALUE fnum, VALUE vmode, VALUE opt); + /* * call-seq: * IO.new(fd, mode = 'r', **opts) -> io @@ -9424,18 +9407,24 @@ static VALUE rb_io_initialize(int argc, VALUE *argv, VALUE io) { VALUE fnum, vmode; + VALUE opt; + + rb_scan_args(argc, argv, "11:", &fnum, &vmode, &opt); + return io_initialize(io, fnum, vmode, opt); +} + +static VALUE +io_initialize(VALUE io, VALUE fnum, VALUE vmode, VALUE opt) +{ rb_io_t *fp; int fd, fmode, oflags = O_RDONLY; struct rb_io_encoding convconfig; - VALUE opt; #if defined(HAVE_FCNTL) && defined(F_GETFL) int ofmode; #else struct stat st; #endif - - argc = rb_scan_args(argc, argv, "11:", &fnum, &vmode, &opt); rb_io_extract_modeenc(&vmode, 0, opt, &oflags, &fmode, &convconfig); fd = NUM2INT(fnum); @@ -9584,17 +9573,16 @@ rb_file_initialize(int argc, VALUE *argv, VALUE io) if (RFILE(io)->fptr) { rb_raise(rb_eRuntimeError, "reinitializing File"); } - if (0 < argc && argc < 3) { - VALUE fd = rb_check_to_int(argv[0]); + VALUE fname, vmode, vperm, opt; + int posargc = rb_scan_args(argc, argv, "12:", &fname, &vmode, &vperm, &opt); + if (posargc < 3) { /* perm is File only */ + VALUE fd = rb_check_to_int(fname); if (!NIL_P(fd)) { - argv[0] = fd; - return rb_io_initialize(argc, argv, io); + return io_initialize(io, fd, vmode, opt); } } - rb_open_file(argc, argv, io); - - return io; + return rb_open_file(io, fname, vmode, vperm, opt); } /* :nodoc: */ @@ -10902,7 +10890,8 @@ rb_io_advise(int argc, VALUE *argv, VALUE io) * Each of the arguments +read_ios+, +write_ios+, and +error_ios+ * is an array of IO objects. * - * Argument +timeout+ is an integer timeout interval in seconds. + * Argument +timeout+ is a numeric value (such as integer or float) timeout + * interval in seconds. * * The method monitors the \IO objects given in all three arrays, * waiting for some to be ready; diff --git a/io_buffer.c b/io_buffer.c index 7715aa0d37..e6ca61bdae 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -260,7 +260,7 @@ io_buffer_free(struct rb_io_buffer *buffer) if (buffer->mapping) { if (RB_IO_BUFFER_DEBUG) fprintf(stderr, "io_buffer_free:CloseHandle -> %p\n", buffer->mapping); if (!CloseHandle(buffer->mapping)) { - fprintf(stderr, "io_buffer_free:GetLastError -> %d\n", GetLastError()); + fprintf(stderr, "io_buffer_free:GetLastError -> %lu\n", GetLastError()); } buffer->mapping = NULL; } @@ -2441,13 +2441,13 @@ rb_io_buffer_initialize_copy(VALUE self, VALUE source) * * #copy can be used to put buffer into strings associated with buffer: * - * string= "buffer: " - * # => "buffer: " + * string= "data: " + * # => "data: " * buffer = IO::Buffer.for(string) * buffer.copy(IO::Buffer.for("test"), 5) * # => 4 * string - * # => "buffer:test" + * # => "data:test" * * Attempt to copy into a read-only buffer will fail: * @@ -3571,30 +3571,28 @@ io_buffer_not_inplace(VALUE self) * * \Buffer from string: * - * string = 'buffer' - * buffer = IO::Buffer.for(string) - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # ... - * buffer - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # 0x00000000 64 61 74 61 buffer - * - * buffer.get_string(2) # read content starting from offset 2 - * # => "ta" - * buffer.set_string('---', 1) # write content, starting from offset 1 - * # => 3 - * buffer - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # 0x00000000 64 2d 2d 2d d--- - * string # original string changed, too - * # => "d---" + * string = 'data' + * IO::Buffer.for(string) do |buffer| + * buffer + * # => + * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> + * # 0x00000000 64 61 74 61 data + * + * buffer.get_string(2) # read content starting from offset 2 + * # => "ta" + * buffer.set_string('---', 1) # write content, starting from offset 1 + * # => 3 + * buffer + * # => + * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> + * # 0x00000000 64 2d 2d 2d d--- + * string # original string changed, too + * # => "d---" + * end * * \Buffer from file: * - * File.write('test.txt', 'test buffer') + * File.write('test.txt', 'test data') * # => 9 * buffer = IO::Buffer.map(File.open('test.txt')) * # => @@ -3611,7 +3609,7 @@ io_buffer_not_inplace(VALUE self) * buffer.set_string('---', 1) * # => 3 -- bytes written * File.read('test.txt') - * # => "t--- buffer" + * # => "t--- data" * * <b>The class is experimental and the interface is subject to change, this * is especially true of file mappings which may be removed entirely in @@ -346,7 +346,7 @@ rb_iseq_mark_and_move(rb_iseq_t *iseq, bool reference_updating) if (cc_is_active(cds[i].cc, reference_updating)) { rb_gc_mark_and_move_ptr(&cds[i].cc); } - else { + else if (cds[i].cc != rb_vm_empty_cc()) { cds[i].cc = rb_vm_empty_cc(); } } @@ -1029,8 +1029,13 @@ pm_iseq_new_with_opt(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpa ISEQ_BODY(iseq)->prism = true; ISEQ_BODY(iseq)->param.flags.use_block = true; // unused block warning is not supported yet + rb_compile_option_t next_option; if (!option) option = &COMPILE_OPTION_DEFAULT; + next_option = *option; + next_option.coverage_enabled = node->coverage_enabled < 0 ? 0 : node->coverage_enabled > 0; + option = &next_option; + pm_location_t *location = &node->base.location; int32_t start_line = node->parser->start_line; @@ -1273,6 +1278,7 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, NUM2INT(line)); + result.node.coverage_enabled = 1; switch (option.frozen_string_literal) { case ISEQ_FROZEN_STRING_LITERAL_UNSET: @@ -1708,6 +1714,7 @@ iseqw_s_compile_file_prism(int argc, VALUE *argv, VALUE self) pm_parse_result_t result = { 0 }; result.options.line = 1; + result.node.coverage_enabled = 1; VALUE error = pm_load_parse_file(&result, file); diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index ed5f940b57..932949576b 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -28,23 +28,14 @@ module Gem::BUNDLED_GEMS "syslog" => "3.4.0", "ostruct" => "3.5.0", "pstore" => "3.5.0", + "rdoc" => "3.5.0", + "win32ole" => "3.5.0", + "fiddle" => "3.5.0", + "logger" => "3.5.0", }.freeze EXACT = { - "abbrev" => true, - "base64" => true, - "bigdecimal" => true, - "csv" => true, - "drb" => true, - "getoptlong" => true, - "mutex_m" => true, - "nkf" => true, "kconv" => "nkf", - "observer" => true, - "resolv-replace" => true, - "rinda" => true, - "syslog" => true, - "ostruct" => true, - "pstore" => true, + "kconv" => "nkf", }.freeze PREFIXED = { @@ -95,7 +86,7 @@ module Gem::BUNDLED_GEMS else return end - EXACT[n] or PREFIXED[n = n[%r[\A[^/]+(?=/)]]] && n + (EXACT[n] || !!SINCE[n]) or PREFIXED[n = n[%r[\A[^/]+(?=/)]]] && n end def self.warning?(name, specs: nil) diff --git a/lib/bundler.rb b/lib/bundler.rb index 5033109db6..0081b9554f 100644 --- a/lib/bundler.rb +++ b/lib/bundler.rb @@ -42,6 +42,7 @@ module Bundler autoload :Checksum, File.expand_path("bundler/checksum", __dir__) autoload :CLI, File.expand_path("bundler/cli", __dir__) autoload :CIDetector, File.expand_path("bundler/ci_detector", __dir__) + autoload :CompactIndexClient, File.expand_path("bundler/compact_index_client", __dir__) autoload :Definition, File.expand_path("bundler/definition", __dir__) autoload :Dependency, File.expand_path("bundler/dependency", __dir__) autoload :Deprecate, File.expand_path("bundler/deprecate", __dir__) @@ -357,7 +358,7 @@ module Bundler def settings @settings ||= Settings.new(app_config_path) rescue GemfileNotFound - @settings = Settings.new(Pathname.new(".bundle").expand_path) + @settings = Settings.new end # @return [Hash] Environment present before Bundler was activated diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 40f19c7fed..eb67668cd2 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -767,13 +767,10 @@ module Bundler return unless SharedHelpers.md5_available? - latest = Fetcher::CompactIndex. - new(nil, Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")), nil, nil). - send(:compact_index_client). - instance_variable_get(:@cache). - dependencies("bundler"). - map {|d| Gem::Version.new(d.first) }. - max + require_relative "vendored_uri" + remote = Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")) + cache_path = Bundler.user_cache.join("compact_index", remote.cache_slug) + latest = Bundler::CompactIndexClient.new(cache_path).latest_version("bundler") return unless latest current = Gem::Version.new(VERSION) diff --git a/lib/bundler/cli/install.rb b/lib/bundler/cli/install.rb index 6c102d537d..a233d5d2e5 100644 --- a/lib/bundler/cli/install.rb +++ b/lib/bundler/cli/install.rb @@ -14,7 +14,7 @@ module Bundler Bundler.self_manager.install_locked_bundler_and_restart_with_it_if_needed - Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Bundler::FREEBSD + Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Gem.freebsd_platform? # Disable color in deployment mode Bundler.ui.shell = Thor::Shell::Basic.new if options[:deployment] diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb index 68e0d7e0d5..692d68e579 100644 --- a/lib/bundler/compact_index_client.rb +++ b/lib/bundler/compact_index_client.rb @@ -4,6 +4,29 @@ require "pathname" require "set" module Bundler + # The CompactIndexClient is responsible for fetching and parsing the compact index. + # + # The compact index is a set of caching optimized files that are used to fetch gem information. + # The files are: + # - names: a list of all gem names + # - versions: a list of all gem versions + # - info/[gem]: a list of all versions of a gem + # + # The client is instantiated with: + # - `directory`: the root directory where the cache files are stored. + # - `fetcher`: (optional) an object that responds to #call(uri_path, headers) and returns an http response. + # If the `fetcher` is not provided, the client will only read cached files from disk. + # + # The client is organized into: + # - `Updater`: updates the cached files on disk using the fetcher. + # - `Cache`: calls the updater, caches files, read and return them from disk + # - `Parser`: parses the compact index file data + # - `CacheFile`: a concurrency safe file reader/writer that verifies checksums + # + # The client is intended to optimize memory usage and performance. + # It is called 100s or 1000s of times, parsing files with hundreds of thousands of lines. + # It may be called concurrently without global interpreter lock in some Rubies. + # As a result, some methods may look more complex than necessary to save memory or time. class CompactIndexClient # NOTE: MD5 is here not because we expect a server to respond with it, but # because we use it to generate the etag on first request during the upgrade @@ -12,6 +35,13 @@ module Bundler SUPPORTED_DIGESTS = { "sha-256" => :SHA256, "md5" => :MD5 }.freeze DEBUG_MUTEX = Thread::Mutex.new + # info returns an Array of INFO Arrays. Each INFO Array has the following indices: + INFO_NAME = 0 + INFO_VERSION = 1 + INFO_PLATFORM = 2 + INFO_DEPS = 3 + INFO_REQS = 4 + def self.debug return unless ENV["DEBUG_COMPACT_INDEX"] DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } @@ -21,106 +51,47 @@ module Bundler require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" - attr_reader :directory - - def initialize(directory, fetcher) - @directory = Pathname.new(directory) - @updater = Updater.new(fetcher) - @cache = Cache.new(@directory) - @endpoints = Set.new - @info_checksums_by_name = {} - @parsed_checksums = false - @mutex = Thread::Mutex.new - end - - def execution_mode=(block) - Bundler::CompactIndexClient.debug { "execution_mode=" } - @endpoints = Set.new - - @execution_mode = block - end - - # @return [Lambda] A lambda that takes an array of inputs and a block, and - # maps the inputs with the block in parallel. - # - def execution_mode - @execution_mode || sequentially - end - - def sequential_execution_mode! - self.execution_mode = sequentially - end - - def sequentially - @sequentially ||= lambda do |inputs, &blk| - inputs.map(&blk) - end + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) end def names - Bundler::CompactIndexClient.debug { "/names" } - update("names", @cache.names_path, @cache.names_etag_path) - @cache.names + Bundler::CompactIndexClient.debug { "names" } + @parser.names end def versions - Bundler::CompactIndexClient.debug { "/versions" } - update("versions", @cache.versions_path, @cache.versions_etag_path) - versions, @info_checksums_by_name = @cache.versions - versions + Bundler::CompactIndexClient.debug { "versions" } + @parser.versions end def dependencies(names) Bundler::CompactIndexClient.debug { "dependencies(#{names})" } - execution_mode.call(names) do |name| - update_info(name) - @cache.dependencies(name).map {|d| d.unshift(name) } - end.flatten(1) + names.map {|name| info(name) } end - def update_and_parse_checksums! - Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } - return @info_checksums_by_name if @parsed_checksums - update("versions", @cache.versions_path, @cache.versions_etag_path) - @info_checksums_by_name = @cache.checksums - @parsed_checksums = true - end - - private - - def update(remote_path, local_path, local_etag_path) - Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } - unless synchronize { @endpoints.add?(remote_path) } - Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } - return - end - @updater.update(url(remote_path), local_path, local_etag_path) + def info(name) + Bundler::CompactIndexClient.debug { "info(#{names})" } + @parser.info(name) end - def update_info(name) - Bundler::CompactIndexClient.debug { "update_info(#{name})" } - path = @cache.info_path(name) - unless existing = @info_checksums_by_name[name] - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } - return - end - checksum = SharedHelpers.checksum_for_file(path, :MD5) - if checksum == existing - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } - return - end - Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update("info/#{name}", path, @cache.info_etag_path(name)) + def latest_version(name) + Bundler::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max end - def url(path) - path + def available? + Bundler::CompactIndexClient.debug { "available?" } + @parser.available? end - def synchronize - @mutex.synchronize { yield } + def reset! + Bundler::CompactIndexClient.debug { "reset!" } + @cache.reset! end end end diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index 55911fdecf..bedd7f8028 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -7,114 +7,89 @@ module Bundler class Cache attr_reader :directory - def initialize(directory) + def initialize(directory, fetcher = nil) @directory = Pathname.new(directory).expand_path - info_roots.each {|dir| mkdir(dir) } - mkdir(info_etag_root) + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new + + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") end def names - lines(names_path) + fetch("names", names_path, names_etag_path) end - def names_path - directory.join("names") + def versions + fetch("versions", versions_path, versions_etag_path) end - def names_etag_path - directory.join("names.etag") - end + def info(name, remote_checksum = nil) + path = info_path(name) - def versions - versions_by_name = Hash.new {|hash, key| hash[key] = [] } - info_checksums_by_name = {} - - lines(versions_path).each do |line| - name, versions_string, info_checksum = line.split(" ", 3) - info_checksums_by_name[name] = info_checksum || "" - versions_string.split(",") do |version| - delete = version.delete_prefix!("-") - version = version.split("-", 2).unshift(name) - if delete - versions_by_name[name].delete(version) - else - versions_by_name[name] << version - end - end + if remote_checksum && remote_checksum != SharedHelpers.checksum_for_file(path, :MD5) + fetch("info/#{name}", path, info_etag_path(name)) + else + Bundler::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum is nil" : "versions index checksum matches local"})" } + read(path) end - - [versions_by_name, info_checksums_by_name] - end - - def versions_path - directory.join("versions") end - def versions_etag_path - directory.join("versions.etag") + def reset! + @mutex.synchronize { @endpoints.clear } end - def checksums - checksums = {} - - lines(versions_path).each do |line| - name, _, checksum = line.split(" ", 3) - checksums[name] = checksum - end - - checksums - end + private - def dependencies(name) - lines(info_path(name)).map do |line| - parse_gem(line) - end - end + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") def info_path(name) name = name.to_s + # TODO: converge this into the info_root by hashing all filenames like info_etag_path if /[^a-z0-9_-]/.match?(name) name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}" - info_roots.last.join(name) + @special_characters_info_root.join(name) else - info_roots.first.join(name) + @info_root.join(name) end end def info_etag_path(name) name = name.to_s - info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") + @info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") end - private - - def mkdir(dir) - SharedHelpers.filesystem_access(dir) do - FileUtils.mkdir_p(dir) + def mkdir(name) + directory.join(name).tap do |dir| + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end end end - def lines(path) - return [] unless path.file? - lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines - end + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Bundler::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end - def parse_gem(line) - @dependency_parser ||= GemParser.new - @dependency_parser.parse(line) + read(path) end - def info_roots - [ - directory.join("info"), - directory.join("info-special-characters"), - ] + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } end - def info_etag_root - directory.join("info-etags") + def read(path) + return unless path.file? + SharedHelpers.filesystem_access(path, :read, &:read) end end end diff --git a/lib/bundler/compact_index_client/cache_file.rb b/lib/bundler/compact_index_client/cache_file.rb index 5988bc91b3..299d683438 100644 --- a/lib/bundler/compact_index_client/cache_file.rb +++ b/lib/bundler/compact_index_client/cache_file.rb @@ -86,11 +86,6 @@ module Bundler end end - # remove this method when we stop generating md5 digests for legacy etags - def md5 - @digests && @digests["md5"] - end - def digests? @digests&.any? end diff --git a/lib/bundler/compact_index_client/parser.rb b/lib/bundler/compact_index_client/parser.rb new file mode 100644 index 0000000000..3a0dec4907 --- /dev/null +++ b/lib/bundler/compact_index_client/parser.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Bundler + class CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + @gem_parser = nil + @versions_data = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + data = @compact_index.info(name, info_checksum(name)) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + # parse the last, most recently updated line of the versions file to determine availability + def available? + return @available unless @available.nil? + return @available = false unless versions_data&.size&.nonzero? + + line_end = versions_data.size - 1 + return @available = false if versions_data[line_end] != "\n" + + line_start = versions_data.rindex("\n", line_end - 1) + line_start ||= -1 # allow a single line versions file + + @available = !split_last_word(versions_data, line_start + 1, line_end).nil? + end + + private + + # Search for a line starting with gem name, then return last space-separated word (the checksum) + def info_checksum(name) + return unless versions_data + return unless (line_start = rindex_of_gem(name)) + return unless (line_end = versions_data.index("\n", line_start)) + split_last_word(versions_data, line_start, line_end) + end + + def gem_parser + @gem_parser ||= GemParser.new + end + + def versions_data + @versions_data ||= begin + data = @compact_index.versions + strip_header!(data) if data + data.freeze + end + end + + def rindex_of_gem(name) + if (pos = versions_data.rindex("\n#{name} ")) + pos + 1 + elsif versions_data.start_with?("#{name} ") + 0 + end + end + + # This is similar to `string.split(" ").last` but it avoids allocating extra objects. + def split_last_word(string, line_start, line_end) + return unless line_start < line_end && line_start >= 0 + word_start = string.rindex(" ", line_end).to_i + 1 + return if word_start < line_start + string[word_start, line_end - word_start] + end + + def lines(string) + return [] if string.nil? || string.empty? + strip_header!(string) + string.split("\n") + end + + def strip_header!(string) + header_end = string.index("---\n") + string.slice!(0, header_end + 4) if header_end + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 36f6b81db8..88c7146900 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -28,7 +28,6 @@ module Bundler CacheFile.copy(local_path) do |file| etag = etag_path.read.tap(&:chomp!) if etag_path.file? - etag ||= generate_etag(etag_path, file) # Remove this after 2.5.0 has been out for a while. # Subtract a byte to ensure the range won't be empty. # Avoids 416 (Range Not Satisfiable) responses. @@ -67,16 +66,6 @@ module Bundler etag_path.read.tap(&:chomp!) if etag_path.file? end - # When first releasing this opaque etag feature, we want to generate the old MD5 etag - # based on the content of the file. After that it will always use the saved opaque etag. - # This transparently saves existing users with good caches from updating a bunch of files. - # Remove this behavior after 2.5.0 has been out for a while. - def generate_etag(etag_path, file) - etag = file.md5.hexdigest - CacheFile.write(etag_path, etag) - etag - end - def etag_from_response(response) return unless response["ETag"] etag = response["ETag"].delete_prefix("W/") diff --git a/lib/bundler/constants.rb b/lib/bundler/constants.rb index de9698b577..9564771e78 100644 --- a/lib/bundler/constants.rb +++ b/lib/bundler/constants.rb @@ -1,7 +1,14 @@ # frozen_string_literal: true +require "rbconfig" + module Bundler WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/ + deprecate_constant :WINDOWS + FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd") - NULL = File::NULL + deprecate_constant :FREEBSD + + NULL = File::NULL + deprecate_constant :NULL end diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 22070b6b17..6cf1f9a255 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -69,7 +69,6 @@ module Bundler @sources = sources @unlock = unlock @optional_groups = optional_groups - @remote = false @prefer_local = false @specs = nil @ruby_version = ruby_version @@ -164,37 +163,24 @@ module Bundler end def resolve_only_locally! - @remote = false sources.local_only! resolve end def resolve_with_cache! + sources.local! sources.cached! resolve end def resolve_remotely! - @remote = true + sources.cached! sources.remote! resolve end - def resolution_mode=(options) - if options["local"] - @remote = false - else - @remote = true - @prefer_local = options["prefer-local"] - end - end - - def setup_sources_for_resolve - if @remote == false - sources.cached! - else - sources.remote! - end + def prefer_local! + @prefer_local = true end # For given dependency list returns a SpecSet with Gemspec of all the required @@ -310,7 +296,12 @@ module Bundler end end else - Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + if lockfile_exists? + Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + else + Bundler.ui.debug "Resolving dependencies because there's no lockfile" + end + start_resolution end end @@ -483,6 +474,8 @@ module Bundler private :sources def nothing_changed? + return false unless lockfile_exists? + !@source_changes && !@dependency_changes && !@new_platform && @@ -587,7 +580,7 @@ module Bundler if missing_specs.any? missing_specs.each do |s| locked_gem = @locked_specs[s.name].last - next if locked_gem.nil? || locked_gem.version != s.version || !@remote + next if locked_gem.nil? || locked_gem.version != s.version || sources.local_mode? raise GemNotFound, "Your bundle is locked to #{locked_gem} from #{locked_gem.source}, but that version can " \ "no longer be found in that source. That means the author of #{locked_gem} has removed it. " \ "You'll need to update your bundle to a version other than #{locked_gem} that hasn't been " \ @@ -606,7 +599,7 @@ module Bundler break if incomplete_specs.empty? Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies") - setup_sources_for_resolve + sources.remote! resolution_packages.delete(incomplete_specs) @resolve = start_resolution specs = resolve.materialize(dependencies) @@ -967,7 +960,7 @@ module Bundler else { default: Source::RubygemsAggregate.new(sources, source_map) }.merge(source_map.direct_requirements) end - source_requirements.merge!(source_map.locked_requirements) unless @remote + source_requirements.merge!(source_map.locked_requirements) if nothing_changed? metadata_dependencies.each do |dep| source_requirements[dep.name] = sources.metadata_source end @@ -1038,8 +1031,6 @@ module Bundler def dup_for_full_unlock unlocked_definition = self.class.new(@lockfile, @dependencies, @sources, true, @ruby_version, @optional_groups, @gemfiles) - unlocked_definition.resolution_mode = { "local" => !@remote } - unlocked_definition.setup_sources_for_resolve unlocked_definition.gem_version_promoter.tap do |gvp| gvp.level = gem_version_promoter.level gvp.strict = gem_version_promoter.strict diff --git a/lib/bundler/errors.rb b/lib/bundler/errors.rb index b6a11cc721..c29b1bfed8 100644 --- a/lib/bundler/errors.rb +++ b/lib/bundler/errors.rb @@ -230,4 +230,18 @@ module Bundler status_code(38) end + + class CorruptBundlerInstallError < BundlerError + def initialize(loaded_spec) + @loaded_spec = loaded_spec + end + + def message + "The running version of Bundler (#{Bundler::VERSION}) does not match the version of the specification installed for it (#{@loaded_spec.version}). " \ + "This can be caused by reinstalling Ruby without removing previous installation, leaving around an upgraded default version of Bundler. " \ + "Reinstalling Ruby from scratch should fix the problem." + end + + status_code(39) + end end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index db914839b1..6e5656d26a 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -4,8 +4,6 @@ require_relative "base" require_relative "../worker" module Bundler - autoload :CompactIndexClient, File.expand_path("../compact_index_client", __dir__) - class Fetcher class CompactIndex < Base def self.compact_index_request(method_name) @@ -36,15 +34,8 @@ module Bundler until remaining_gems.empty? log_specs { "Looking up gems #{remaining_gems.inspect}" } - - deps = begin - parallel_compact_index_client.dependencies(remaining_gems) - rescue TooManyRequestsError - @bundle_worker&.stop - @bundle_worker = nil # reset it. Not sure if necessary - serial_compact_index_client.dependencies(remaining_gems) - end - next_gems = deps.flat_map {|d| d[3].flat_map(&:first) }.uniq + deps = fetch_gem_infos(remaining_gems).flatten(1) + next_gems = deps.flat_map {|d| d[CompactIndexClient::INFO_DEPS].flat_map(&:first) }.uniq deps.each {|dep| gem_info << dep } complete_gems.concat(deps.map(&:first)).uniq! remaining_gems = next_gems - complete_gems @@ -61,7 +52,7 @@ module Bundler return nil end # Read info file checksums out of /versions, so we can know if gems are up to date - compact_index_client.update_and_parse_checksums! + compact_index_client.available? rescue CompactIndexClient::Updater::MismatchedChecksumError => e Bundler.ui.debug(e.message) nil @@ -81,20 +72,20 @@ module Bundler end end - def parallel_compact_index_client - compact_index_client.execution_mode = lambda do |inputs, &blk| - func = lambda {|object, _index| blk.call(object) } - worker = bundle_worker(func) - inputs.each {|input| worker.enq(input) } - inputs.map { worker.deq } - end - - compact_index_client + def fetch_gem_infos(names) + in_parallel(names) {|name| compact_index_client.info(name) } + rescue TooManyRequestsError # rubygems.org is rate limiting us, slow down. + @bundle_worker&.stop + @bundle_worker = nil # reset it. Not sure if necessary + compact_index_client.reset! + names.map {|name| compact_index_client.info(name) } end - def serial_compact_index_client - compact_index_client.sequential_execution_mode! - compact_index_client + def in_parallel(inputs, &blk) + func = lambda {|object, _index| blk.call(object) } + worker = bundle_worker(func) + inputs.each {|input| worker.enq(input) } + inputs.map { worker.deq } end def bundle_worker(func = nil) diff --git a/lib/bundler/gem_helper.rb b/lib/bundler/gem_helper.rb index d535d54f9b..5ce0ef6280 100644 --- a/lib/bundler/gem_helper.rb +++ b/lib/bundler/gem_helper.rb @@ -47,7 +47,7 @@ module Bundler built_gem_path = build_gem end - desc "Generate SHA512 checksum if #{name}-#{version}.gem into the checksums directory." + desc "Generate SHA512 checksum of #{name}-#{version}.gem into the checksums directory." task "build:checksum" => "build" do build_checksum(built_gem_path) end diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb index 72e5602cc3..256f0be348 100644 --- a/lib/bundler/installer.rb +++ b/lib/bundler/installer.rb @@ -235,15 +235,15 @@ module Bundler # returns whether or not a re-resolve was needed def resolve_if_needed(options) - @definition.resolution_mode = options - - if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file? - return false if @definition.nothing_changed? && !@definition.missing_specs? + @definition.prefer_local! if options["prefer-local"] + + if options["local"] || (@definition.no_resolve_needed? && !@definition.missing_specs?) + @definition.resolve_with_cache! + false + else + @definition.resolve_remotely! + true end - - @definition.setup_sources_for_resolve - - true end def lock diff --git a/lib/bundler/installer/gem_installer.rb b/lib/bundler/installer/gem_installer.rb index d3bbcc90f5..a7770eb7e0 100644 --- a/lib/bundler/installer/gem_installer.rb +++ b/lib/bundler/installer/gem_installer.rb @@ -54,7 +54,6 @@ module Bundler spec.source.install( spec, force: force, - ensure_builtin_gems_cached: standalone, build_args: Array(spec_settings), previous_spec: previous_spec, ) diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index 0b09480f88..56a3b6f85c 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-ADD" "1" "April 2024" "" +.TH "BUNDLE\-ADD" "1" "May 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index fdb86bfd29..4ec301951f 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-BINSTUBS" "1" "April 2024" "" +.TH "BUNDLE\-BINSTUBS" "1" "May 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index c5899ace1a..e2da1269e6 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CACHE" "1" "April 2024" "" +.TH "BUNDLE\-CACHE" "1" "May 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index e1a971edae..dee1af1326 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CHECK" "1" "April 2024" "" +.TH "BUNDLE\-CHECK" "1" "May 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index ca2fb65f59..7c7f9b5c77 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CLEAN" "1" "April 2024" "" +.TH "BUNDLE\-CLEAN" "1" "May 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index a78c6b0a18..2de52ee375 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONFIG" "1" "April 2024" "" +.TH "BUNDLE\-CONFIG" "1" "May 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options .SH "SYNOPSIS" @@ -307,7 +307,7 @@ Any \fB\.\fR characters in a host name are mapped to a double underscore (\fB__\ .P This means that if you have a gem server named \fBmy\.gem\-host\.com\fR, you'll need to use the \fBBUNDLE_MY__GEM___HOST__COM\fR variable to configure credentials for it through ENV\. .SH "CONFIGURE BUNDLER DIRECTORIES" -Bundler's home, config, cache and plugin directories are able to be configured through environment variables\. The default location for Bundler's home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values +Bundler's home, cache and plugin directories and config file can be configured through environment variables\. The default location for Bundler's home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values .IP "" 4 .nf BUNDLE_USER_HOME : $HOME/\.bundle diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index 7e5f458fb2..1a0ec2a5dc 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -397,7 +397,7 @@ through ENV. ## CONFIGURE BUNDLER DIRECTORIES -Bundler's home, config, cache and plugin directories are able to be configured +Bundler's home, cache and plugin directories and config file can be configured through environment variables. The default location for Bundler's home directory is `~/.bundle`, which all directories inherit from by default. The following outlines the available environment variables and their default values diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index 86be5e4185..dca18ec43d 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONSOLE" "1" "April 2024" "" +.TH "BUNDLE\-CONSOLE" "1" "May 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index 94cbff4cbd..6489cc07f7 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-DOCTOR" "1" "April 2024" "" +.TH "BUNDLE\-DOCTOR" "1" "May 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 7aa49e0096..1548d29670 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-EXEC" "1" "April 2024" "" +.TH "BUNDLE\-EXEC" "1" "May 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 89eb9f1980..5df7b0ef2f 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-GEM" "1" "April 2024" "" +.TH "BUNDLE\-GEM" "1" "May 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 441ec12735..a3e7c7770d 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-HELP" "1" "April 2024" "" +.TH "BUNDLE\-HELP" "1" "May 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 287965c06a..a3d7ff0988 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INFO" "1" "April 2024" "" +.TH "BUNDLE\-INFO" "1" "May 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index d13f8ddc3b..a0edaaa18f 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INIT" "1" "April 2024" "" +.TH "BUNDLE\-INIT" "1" "May 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index 086fc88156..7a1038206e 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INJECT" "1" "April 2024" "" +.TH "BUNDLE\-INJECT" "1" "May 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 762c99e7c0..cc46a03b7f 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INSTALL" "1" "April 2024" "" +.TH "BUNDLE\-INSTALL" "1" "May 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index 93db700091..21723608cc 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LIST" "1" "April 2024" "" +.TH "BUNDLE\-LIST" "1" "May 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index c0190f91de..8b81b7732f 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LOCK" "1" "April 2024" "" +.TH "BUNDLE\-LOCK" "1" "May 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 38bacb67dc..41a8804a09 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OPEN" "1" "April 2024" "" +.TH "BUNDLE\-OPEN" "1" "May 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index cee9d63155..838fce45cd 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OUTDATED" "1" "April 2024" "" +.TH "BUNDLE\-OUTDATED" "1" "May 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 069966f1b2..0dbce7a7a4 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLATFORM" "1" "April 2024" "" +.TH "BUNDLE\-PLATFORM" "1" "May 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index f4e043c363..1290abb3ba 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLUGIN" "1" "April 2024" "" +.TH "BUNDLE\-PLUGIN" "1" "May 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 76a0479bc6..bf4a7cd323 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PRISTINE" "1" "April 2024" "" +.TH "BUNDLE\-PRISTINE" "1" "May 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index 90a664e331..c3c96b416d 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-REMOVE" "1" "April 2024" "" +.TH "BUNDLE\-REMOVE" "1" "May 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index f9f1fd1fa3..c054875efd 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-SHOW" "1" "April 2024" "" +.TH "BUNDLE\-SHOW" "1" "May 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 340ebac687..acd80ec75c 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-UPDATE" "1" "April 2024" "" +.TH "BUNDLE\-UPDATE" "1" "May 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 00ff0153cb..44eaf92224 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VERSION" "1" "April 2024" "" +.TH "BUNDLE\-VERSION" "1" "May 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index eba3b7df25..77b902214c 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VIZ" "1" "April 2024" "" +.TH "BUNDLE\-VIZ" "1" "May 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 4bd2bcaf67..199d1ce9fd 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE" "1" "April 2024" "" +.TH "BUNDLE" "1" "May 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management .SH "SYNOPSIS" diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index c9478a953e..fa9e31f615 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "GEMFILE" "5" "April 2024" "" +.TH "GEMFILE" "5" "May 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs .SH "SYNOPSIS" diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index a7539f4adb..18180a81a1 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -require "pathname" - require "rubygems" unless defined?(Gem) -require "rubygems/specification" - # We can't let `Gem::Source` be autoloaded in the `Gem::Specification#source` # redefinition below, so we need to load it upfront. The reason is that if # Bundler monkeypatches are loaded before RubyGems activates an executable (for @@ -17,10 +13,6 @@ require "rubygems/specification" # `Gem::Source` from the redefined `Gem::Specification#source`. require "rubygems/source" -require_relative "match_metadata" -require_relative "force_platform" -require_relative "match_platform" - # Cherry-pick fixes to `Gem.ruby_version` to be useful for modern Bundler # versions and ignore patchlevels # (https://github.com/rubygems/rubygems/pull/5472, @@ -31,7 +23,19 @@ unless Gem.ruby_version.to_s == RUBY_VERSION || RUBY_PATCHLEVEL == -1 end module Gem + # Can be removed once RubyGems 3.5.11 support is dropped + unless Gem.respond_to?(:freebsd_platform?) + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + end + + require "rubygems/specification" + class Specification + require_relative "match_metadata" + require_relative "match_platform" + include ::Bundler::MatchMetadata include ::Bundler::MatchPlatform @@ -48,7 +52,7 @@ module Gem def full_gem_path if source.respond_to?(:root) - Pathname.new(loaded_from).dirname.expand_path(source.root).to_s + File.expand_path(File.dirname(loaded_from), source.root) else rg_full_gem_path end @@ -148,17 +152,21 @@ module Gem module BetterPermissionError def data - Bundler::SharedHelpers.filesystem_access(loaded_from, :read) do - super - end + super + rescue Errno::EACCES + raise Bundler::PermissionError.new(loaded_from, :read) end end + require "rubygems/stub_specification" + class StubSpecification prepend BetterPermissionError end class Dependency + require_relative "force_platform" + include ::Bundler::ForcePlatform attr_accessor :source, :groups diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 494030eab2..b841462263 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -481,11 +481,25 @@ module Bundler end def all_specs + SharedHelpers.major_deprecation 2, "Bundler.rubygems.all_specs has been removed in favor of Bundler.rubygems.installed_specs" + Gem::Specification.stubs.map do |stub| StubSpecification.from_stub(stub) end end + def installed_specs + Gem::Specification.stubs.reject(&:default_gem?).map do |stub| + StubSpecification.from_stub(stub) + end + end + + def default_specs + Gem::Specification.default_stubs.map do |stub| + StubSpecification.from_stub(stub) + end + end + def find_bundler(version) find_name("bundler").find {|s| s.version.to_s == version } end diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index bfd000b1a0..5accda4bcb 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -113,7 +113,7 @@ module Bundler end def local_specs - @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true, "allow_cached" => true).specs.select {|spec| spec.name == "bundler" } + @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true).specs.select {|spec| spec.name == "bundler" } end def remote_specs diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index abbaec9783..878747a53b 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -103,6 +103,7 @@ module Bundler def initialize(root = nil) @root = root @local_config = load_config(local_config_file) + @local_root = root || Pathname.new(".bundle").expand_path @env_config = ENV.to_h @env_config.select! {|key, _value| key.start_with?("BUNDLE_") } @@ -142,7 +143,7 @@ module Bundler end def set_local(key, value) - local_config_file || raise(GemfileNotFound, "Could not locate Gemfile") + local_config_file = @local_root.join("config") set_key(key, value, @local_config, local_config_file) end @@ -491,6 +492,10 @@ module Bundler valid_file = file.exist? && !file.size.zero? return {} unless valid_file serializer_class.load(file.read).inject({}) do |config, (k, v)| + k = k.dup + k << "/" if /https?:/i.match?(k) && !k.end_with?("/", "__#{FALLBACK_TIMEOUT_URI_OPTION.upcase}") + k.gsub!(".", "__") + unless k.start_with?("#") if k.include?("-") Bundler.ui.warn "Your #{file} config includes `#{k}`, which contains the dash character (`-`).\n" \ @@ -518,26 +523,25 @@ module Bundler YAMLSerializer end - PER_URI_OPTIONS = %w[ - fallback_timeout - ].freeze + FALLBACK_TIMEOUT_URI_OPTION = "fallback_timeout" NORMALIZE_URI_OPTIONS_PATTERN = / \A (\w+\.)? # optional prefix key (https?.*?) # URI - (\.#{Regexp.union(PER_URI_OPTIONS)})? # optional suffix key + (\.#{FALLBACK_TIMEOUT_URI_OPTION})? # optional suffix key \z /ix def self.key_for(key) - key = normalize_uri(key).to_s if key.is_a?(String) && key.start_with?("http", "mirror.http") - key = key_to_s(key).gsub(".", "__") + key = key_to_s(key) + key = normalize_uri(key) if key.start_with?("http", "mirror.http") + key = key.gsub(".", "__") key.gsub!("-", "___") key.upcase! - key.prepend("BUNDLE_") + key.gsub(/\A([ #]*)/, '\1BUNDLE_') end # TODO: duplicates Rubygems#normalize_uri diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb index 78760e6fa4..e55632b89f 100644 --- a/lib/bundler/shared_helpers.rb +++ b/lib/bundler/shared_helpers.rb @@ -1,14 +1,16 @@ # frozen_string_literal: true -require "pathname" -require "rbconfig" - require_relative "version" -require_relative "constants" require_relative "rubygems_integration" require_relative "current_ruby" +autoload :Pathname, "pathname" + module Bundler + autoload :WINDOWS, File.expand_path("constants", __dir__) + autoload :FREEBSD, File.expand_path("constants", __dir__) + autoload :NULL, File.expand_path("constants", __dir__) + module SharedHelpers def root gemfile = find_gemfile diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 645851286c..2fc9c6535f 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -182,6 +182,14 @@ module Bundler if err.include?("Could not find remote branch") raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else + idx = command.index("--depth") + if idx + command.delete_at(idx) + command.delete_at(idx) + command_with_no_credentials = check_allowed(command) + + err += "Retrying without --depth argument." + end raise GitCommandError.new(command_with_no_credentials, path, err) end end diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb index 4d27761365..6b05e17727 100644 --- a/lib/bundler/source/metadata.rb +++ b/lib/bundler/source/metadata.rb @@ -11,6 +11,8 @@ module Bundler end if local_spec = Gem.loaded_specs["bundler"] + raise CorruptBundlerInstallError.new(local_spec) if local_spec.version.to_s != Bundler::VERSION + idx << local_spec else idx << Gem::Specification.new do |s| diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 2e76becb84..dafc674f9d 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -17,7 +17,7 @@ module Bundler @remotes = [] @dependency_names = [] @allow_remote = false - @allow_cached = options["allow_cached"] || false + @allow_cached = false @allow_local = options["allow_local"] || false @checksum_store = Checksum::Store.new @@ -50,10 +50,11 @@ module Bundler end def cached! + return unless File.exist?(cache_path) + return if @allow_cached @specs = nil - @allow_local = true @allow_cached = true end @@ -135,20 +136,17 @@ module Bundler index = @allow_remote ? remote_specs.dup : Index.new index.merge!(cached_specs) if @allow_cached index.merge!(installed_specs) if @allow_local + + # complete with default specs, only if not already available in the + # index through remote, cached, or installed specs + index.use(default_specs) if @allow_local + index end end def install(spec, options = {}) - force = options[:force] - ensure_builtin_gems_cached = options[:ensure_builtin_gems_cached] - - if ensure_builtin_gems_cached && spec.default_gem? && !cached_path(spec) - cached_built_in_gem(spec) unless spec.remote - force = true - end - - if installed?(spec) && !force + if (spec.default_gem? && !cached_built_in_gem(spec)) || (installed?(spec) && !options[:force]) print_using_message "Using #{version_message(spec, options[:previous_spec])}" return nil # no post-install message end @@ -361,7 +359,7 @@ module Bundler def installed_specs @installed_specs ||= Index.build do |idx| - Bundler.rubygems.all_specs.reverse_each do |spec| + Bundler.rubygems.installed_specs.reverse_each do |spec| spec.source = self if Bundler.rubygems.spec_missing_extensions?(spec, false) Bundler.ui.debug "Source #{self} is ignoring #{spec} because it is missing extensions" @@ -372,6 +370,15 @@ module Bundler end end + def default_specs + @default_specs ||= Index.build do |idx| + Bundler.rubygems.default_specs.each do |spec| + spec.source = self + idx << spec + end + end + end + def cached_specs @cached_specs ||= begin idx = Index.new diff --git a/lib/bundler/source_list.rb b/lib/bundler/source_list.rb index bbaac33a95..5f9dd68f17 100644 --- a/lib/bundler/source_list.rb +++ b/lib/bundler/source_list.rb @@ -9,7 +9,7 @@ module Bundler :metadata_source def global_rubygems_source - @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true, "allow_cached" => true) + @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true) end def initialize @@ -22,6 +22,7 @@ module Bundler @metadata_source = Source::Metadata.new @merged_gem_lockfile_sections = false + @local_mode = true end def merged_gem_lockfile_sections? @@ -73,6 +74,10 @@ module Bundler global_rubygems_source end + def local_mode? + @local_mode + end + def default_source global_path_source || global_rubygems_source end @@ -140,11 +145,17 @@ module Bundler all_sources.each(&:local_only!) end + def local! + all_sources.each(&:local!) + end + def cached! all_sources.each(&:cached!) end def remote! + @local_mode = false + all_sources.each(&:remote!) end @@ -178,7 +189,7 @@ module Bundler replacement_source = replace_rubygems_source(replacement_sources, global_rubygems_source) return global_rubygems_source unless replacement_source - replacement_source.cached! + replacement_source.local! replacement_source end diff --git a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt index 175b821a62..67fe8cee79 100644 --- a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt +++ b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt @@ -2,83 +2,131 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to a positive environment for our community include: +Examples of behavior that contributes to a positive environment for our +community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall community +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexualized language or imagery, and sexual attention or advances of + any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at <%= config[:email] %>. All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[INSERT CONTACT METHOD]. +All complaints will be reviewed and investigated promptly and fairly. -All community leaders are obligated to respect the privacy and security of the reporter of any incident. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. -**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series of actions. +**Community Impact**: A violation through a single incident or series of +actions. -**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within the community. +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, -available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/lib/bundler/yaml_serializer.rb b/lib/bundler/yaml_serializer.rb index 42e6aaf89d..93d08a05aa 100644 --- a/lib/bundler/yaml_serializer.rb +++ b/lib/bundler/yaml_serializer.rb @@ -60,7 +60,6 @@ module Bundler indent, key, quote, val = match.captures val = strip_comment(val) - convert_to_backward_compatible_key!(key) depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} @@ -92,14 +91,8 @@ module Bundler end end - # for settings' keys - def convert_to_backward_compatible_key!(key) - key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) - key.gsub!(".", "__") - end - class << self - private :dump_hash, :convert_to_backward_compatible_key! + private :dump_hash end end end diff --git a/lib/error_highlight/base.rb b/lib/error_highlight/base.rb index b9c68b8eb8..81b1b574de 100644 --- a/lib/error_highlight/base.rb +++ b/lib/error_highlight/base.rb @@ -60,14 +60,14 @@ module ErrorHighlight rescue RuntimeError => error # RubyVM::AbstractSyntaxTree.of raises an error with a message that # includes "prism" when the ISEQ was compiled with the prism compiler. - # In this case, we'll set the node to `nil`. In the future, we will - # reparse with the prism parser and pass the parsed node to Spotter. + # In this case, we'll try to parse again with prism instead. raise unless error.message.include?("prism") + prism_find(loc, **opts) end Spotter.new(node, **opts).spot - when RubyVM::AbstractSyntaxTree::Node + when RubyVM::AbstractSyntaxTree::Node, Prism::Node Spotter.new(obj, **opts).spot else @@ -81,6 +81,71 @@ module ErrorHighlight return nil end + # Accepts a Thread::Backtrace::Location object and returns a Prism::Node + # corresponding to the location in the source code. + def self.prism_find(loc, point_type: :name, name: nil) + require "prism" + return nil if Prism::VERSION < "0.29.0" + + path = loc.absolute_path + return unless path + + lineno = loc.lineno + column = RubyVM::AbstractSyntaxTree.node_id_for_backtrace_location(loc) + tunnel = Prism.parse_file(path).value.tunnel(lineno, column) + + # Prism provides the Prism::Node#tunnel API to find all of the nodes that + # correspond to the given line and column in the source code, with the first + # node in the list being the top-most node and the last node in the list + # being the bottom-most node. + tunnel.each_with_index.reverse_each.find do |part, index| + case part + when Prism::CallNode, Prism::CallOperatorWriteNode, Prism::IndexOperatorWriteNode, Prism::LocalVariableOperatorWriteNode + # If we find any of these nodes, we can stop searching as these are the + # nodes that triggered the exceptions. + break part + when Prism::ConstantReadNode, Prism::ConstantPathNode + if index != 0 && tunnel[index - 1].is_a?(Prism::ConstantPathOperatorWriteNode) + # If we're inside of a constant path operator write node, then this + # constant path may be highlighting a couple of different kinds of + # parts. + if part.name == name + # Explicitly turn off Foo::Bar += 1 where Foo and Bar are on + # different lines because error highlight expects this to not work. + break nil if part.delimiter_loc.end_line != part.name_loc.start_line + + # Otherwise, because we have matched the name we can return this + # part. + break part + end + + # If we haven't matched the name, it's the operator that we're looking + # for, and we can return the parent node here. + break tunnel[index - 1] + elsif part.name == name + # If we have matched the name of the constant, then we can return this + # inner node as the node that triggered the exception. + break part + else + # If we are at the beginning of the tunnel or we are at the beginning + # of a constant lookup chain, then we will return this node. + break part if index == 0 || !tunnel[index - 1].is_a?(Prism::ConstantPathNode) + end + when Prism::LocalVariableReadNode, Prism::ParenthesesNode + # If we find any of these nodes, we want to continue searching up the + # tree because these nodes cannot trigger the exceptions. + false + else + # If we find a different kind of node that we haven't already handled, + # we don't know how to handle it so we'll stop searching and assume this + # is not an exception we can decorate. + break nil + end + end + end + + private_class_method :prism_find + class Spotter class NonAscii < Exception; end private_constant :NonAscii @@ -205,6 +270,48 @@ module ErrorHighlight when :OP_CDECL spot_op_cdecl + + when :call_node + case @point_type + when :name + prism_spot_call_for_name + when :args + prism_spot_call_for_args + end + + when :local_variable_operator_write_node + case @point_type + when :name + prism_spot_local_variable_operator_write_for_name + when :args + prism_spot_local_variable_operator_write_for_args + end + + when :call_operator_write_node + case @point_type + when :name + prism_spot_call_operator_write_for_name + when :args + prism_spot_call_operator_write_for_args + end + + when :index_operator_write_node + case @point_type + when :name + prism_spot_index_operator_write_for_name + when :args + prism_spot_index_operator_write_for_args + end + + when :constant_read_node + prism_spot_constant_read + + when :constant_path_node + prism_spot_constant_path + + when :constant_path_operator_write_node + prism_spot_constant_path_operator_write + end if @snippet && @beg_column && @end_column && @beg_column < @end_column @@ -548,6 +655,200 @@ module ErrorHighlight @beg_lineno = @end_lineno = lineno @snippet = @fetch[lineno] end + + # Take a location from the prism parser and set the necessary instance + # variables. + def prism_location(location) + @beg_lineno = location.start_line + @beg_column = location.start_column + @end_lineno = location.end_line + @end_column = location.end_column + @snippet = @fetch[@beg_lineno, @end_lineno] + end + + # Example: + # x.foo + # ^^^^ + # x.foo(42) + # ^^^^ + # x&.foo + # ^^^^^ + # x[42] + # ^^^^ + # x.foo = 1 + # ^^^^^^ + # x[42] = 1 + # ^^^^^^ + # x + 1 + # ^ + # +x + # ^ + # foo(42) + # ^^^ + # foo 42 + # ^^^ + # foo + # ^^^ + def prism_spot_call_for_name + # Explicitly turn off foo.() syntax because error_highlight expects this + # to not work. + return nil if @node.name == :call && @node.message_loc.nil? + + location = @node.message_loc || @node.call_operator_loc || @node.location + location = @node.call_operator_loc.join(location) if @node.call_operator_loc&.start_line == location.start_line + + # If the method name ends with "=" but the message does not, then this is + # a method call using the "attribute assignment" syntax + # (e.g., foo.bar = 1). In this case we need to go retrieve the = sign and + # add it to the location. + if (name = @node.name).end_with?("=") && !@node.message.end_with?("=") + location = location.adjoin("=") + end + + prism_location(location) + + if !name.end_with?("=") && !name.match?(/[[:alpha:]_\[]/) + # If the method name is an operator, then error_highlight only + # highlights the first line. + fetch_line(location.start_line) + end + end + + # Example: + # x.foo(42) + # ^^ + # x[42] + # ^^ + # x.foo = 1 + # ^ + # x[42] = 1 + # ^^^^^^^ + # x[] = 1 + # ^^^^^ + # x + 1 + # ^ + # foo(42) + # ^^ + # foo 42 + # ^^ + def prism_spot_call_for_args + # Explicitly turn off foo.() syntax because error_highlight expects this + # to not work. + return nil if @node.name == :call && @node.message_loc.nil? + + if @node.name == :[]= && @node.opening == "[" && (@node.arguments&.arguments || []).length == 1 + prism_location(@node.opening_loc.copy(start_offset: @node.opening_loc.start_offset + 1).join(@node.arguments.location)) + else + prism_location(@node.arguments.location) + end + end + + # Example: + # x += 1 + # ^ + def prism_spot_local_variable_operator_write_for_name + prism_location(@node.binary_operator_loc.chop) + end + + # Example: + # x += 1 + # ^ + def prism_spot_local_variable_operator_write_for_args + prism_location(@node.value.location) + end + + # Example: + # x.foo += 42 + # ^^^ (for foo) + # x.foo += 42 + # ^ (for +) + # x.foo += 42 + # ^^^^^^^ (for foo=) + def prism_spot_call_operator_write_for_name + if !@name.start_with?(/[[:alpha:]_]/) + prism_location(@node.binary_operator_loc.chop) + else + location = @node.message_loc + if @node.call_operator_loc.start_line == location.start_line + location = @node.call_operator_loc.join(location) + end + + location = location.adjoin("=") if @name.end_with?("=") + prism_location(location) + end + end + + # Example: + # x.foo += 42 + # ^^ + def prism_spot_call_operator_write_for_args + prism_location(@node.value.location) + end + + # Example: + # x[1] += 42 + # ^^^ (for []) + # x[1] += 42 + # ^ (for +) + # x[1] += 42 + # ^^^^^^ (for []=) + def prism_spot_index_operator_write_for_name + case @name + when :[] + prism_location(@node.opening_loc.join(@node.closing_loc)) + when :[]= + prism_location(@node.opening_loc.join(@node.closing_loc).adjoin("=")) + else + # Explicitly turn off foo[] += 1 syntax when the operator is not on + # the same line because error_highlight expects this to not work. + return nil if @node.binary_operator_loc.start_line != @node.opening_loc.start_line + + prism_location(@node.binary_operator_loc.chop) + end + end + + # Example: + # x[1] += 42 + # ^^^^^^^^ + def prism_spot_index_operator_write_for_args + opening_loc = + if @node.arguments.nil? + @node.opening_loc.copy(start_offset: @node.opening_loc.start_offset + 1) + else + @node.arguments.location + end + + prism_location(opening_loc.join(@node.value.location)) + end + + # Example: + # Foo + # ^^^ + def prism_spot_constant_read + prism_location(@node.location) + end + + # Example: + # Foo::Bar + # ^^^^^ + def prism_spot_constant_path + if @node.parent && @node.parent.location.end_line == @node.location.end_line + fetch_line(@node.parent.location.end_line) + prism_location(@node.delimiter_loc.join(@node.name_loc)) + else + fetch_line(@node.location.end_line) + location = @node.name_loc + location = @node.delimiter_loc.join(location) if @node.delimiter_loc.end_line == location.start_line + prism_location(location) + end + end + + # Example: + # Foo::Bar += 1 + # ^^^^^^^^ + def prism_spot_constant_path_operator_write + prism_location(@node.binary_operator_loc.chop) + end end private_constant :Spotter diff --git a/lib/find.gemspec b/lib/find.gemspec index cb845e9409..aef24a5028 100644 --- a/lib/find.gemspec +++ b/lib/find.gemspec @@ -25,7 +25,5 @@ Gem::Specification.new do |spec| spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 5d2ff97328..aafce7aade 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -73,11 +73,12 @@ module IRB self.prompt_mode = IRB.conf[:PROMPT_MODE] - if IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) - @irb_name = IRB.conf[:IRB_NAME] - else - @irb_name = IRB.conf[:IRB_NAME]+"#"+IRB.JobManager.n_jobs.to_s + @irb_name = IRB.conf[:IRB_NAME] + + unless IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) + @irb_name = @irb_name + "#" + IRB.JobManager.n_jobs.to_s end + self.irb_path = "(" + @irb_name + ")" case input_method diff --git a/lib/irb/default_commands.rb b/lib/irb/default_commands.rb index 1bbc68efa7..91c6b2d041 100644 --- a/lib/irb/default_commands.rb +++ b/lib/irb/default_commands.rb @@ -2,32 +2,33 @@ require_relative "command" require_relative "command/internal_helpers" -require_relative "command/context" -require_relative "command/exit" -require_relative "command/force_exit" -require_relative "command/chws" -require_relative "command/pushws" -require_relative "command/subirb" -require_relative "command/load" -require_relative "command/debug" -require_relative "command/edit" +require_relative "command/backtrace" require_relative "command/break" require_relative "command/catch" -require_relative "command/next" -require_relative "command/delete" -require_relative "command/step" +require_relative "command/chws" +require_relative "command/context" require_relative "command/continue" +require_relative "command/debug" +require_relative "command/delete" +require_relative "command/disable_irb" +require_relative "command/edit" +require_relative "command/exit" require_relative "command/finish" -require_relative "command/backtrace" -require_relative "command/info" +require_relative "command/force_exit" require_relative "command/help" -require_relative "command/show_doc" +require_relative "command/history" +require_relative "command/info" require_relative "command/irb_info" +require_relative "command/load" require_relative "command/ls" require_relative "command/measure" +require_relative "command/next" +require_relative "command/pushws" +require_relative "command/show_doc" require_relative "command/show_source" +require_relative "command/step" +require_relative "command/subirb" require_relative "command/whereami" -require_relative "command/history" module IRB module Command @@ -235,6 +236,10 @@ module IRB [:history, NO_OVERRIDE], [:hist, NO_OVERRIDE] ) + + _register_with_aliases(:irb_disable_irb, Command::DisableIrb, + [:disable_irb, NO_OVERRIDE] + ) end ExtendCommand = Command diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 355047519c..7dc08912ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -52,6 +52,7 @@ module IRB # :nodoc: IRB.init_error IRB.parse_opts(argv: argv) IRB.run_config + IRB.validate_config IRB.load_modules unless @CONF[:PROMPT][@CONF[:PROMPT_MODE]] @@ -427,6 +428,40 @@ module IRB # :nodoc: @irbrc_files end + def IRB.validate_config + conf[:IRB_NAME] = conf[:IRB_NAME].to_s + + irb_rc = conf[:IRB_RC] + unless irb_rc.nil? || irb_rc.respond_to?(:call) + raise_validation_error "IRB.conf[:IRB_RC] should be a callable object. Got #{irb_rc.inspect}." + end + + back_trace_limit = conf[:BACK_TRACE_LIMIT] + unless back_trace_limit.is_a?(Integer) + raise_validation_error "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got #{back_trace_limit.inspect}." + end + + prompt = conf[:PROMPT] + unless prompt.is_a?(Hash) + msg = "IRB.conf[:PROMPT] should be a Hash. Got #{prompt.inspect}." + + if prompt.is_a?(Symbol) + msg += " Did you mean to set `IRB.conf[:PROMPT_MODE]`?" + end + + raise_validation_error msg + end + + eval_history = conf[:EVAL_HISTORY] + unless eval_history.nil? || eval_history.is_a?(Integer) + raise_validation_error "IRB.conf[:EVAL_HISTORY] should be an integer. Got #{eval_history.inspect}." + end + end + + def IRB.raise_validation_error(msg) + raise TypeError, msg, @irbrc_files + end + # loading modules def IRB.load_modules for m in @CONF[:LOAD_MODULES] diff --git a/lib/irb/input-method.rb b/lib/irb/input-method.rb index 684527edc4..ced35a2c5a 100644 --- a/lib/irb/input-method.rb +++ b/lib/irb/input-method.rb @@ -328,10 +328,11 @@ module IRB ->() { dialog.trap_key = nil alt_d = [ - [Reline::Key.new(nil, 0xE4, true)], # Normal Alt+d. [27, 100], # Normal Alt+d when convert-meta isn't used. - [195, 164], # The "ä" that appears when Alt+d is pressed on xterm. - [226, 136, 130] # The "∂" that appears when Alt+d in pressed on iTerm2. + # When option/alt is not configured as a meta key in terminal emulator, + # option/alt + d will send a unicode character depend on OS keyboard setting. + [195, 164], # "ä" in somewhere (FIXME: environment information is unknown). + [226, 136, 130] # "∂" Alt+d on Mac keyboard. ] if just_cursor_moving and completion_journey_data.nil? diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index cfe36be83f..f6ac7f0f5f 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -219,28 +219,7 @@ module IRB :unrecoverable_error rescue SyntaxError => e case e.message - when /unterminated (?:string|regexp) meets end of file/ - # "unterminated regexp meets end of file" - # - # example: - # / - # - # "unterminated string meets end of file" - # - # example: - # ' - return :recoverable_error - when /syntax error, unexpected end-of-input/ - # "syntax error, unexpected end-of-input, expecting keyword_end" - # - # example: - # if true - # hoge - # if false - # fuga - # end - return :recoverable_error - when /syntax error, unexpected keyword_end/ + when /unexpected keyword_end/ # "syntax error, unexpected keyword_end" # # example: @@ -250,7 +229,7 @@ module IRB # example: # end return :unrecoverable_error - when /syntax error, unexpected '\.'/ + when /unexpected '\.'/ # "syntax error, unexpected '.'" # # example: @@ -262,6 +241,27 @@ module IRB # example: # method / f / return :unrecoverable_error + when /unterminated (?:string|regexp) meets end of file/ + # "unterminated regexp meets end of file" + # + # example: + # / + # + # "unterminated string meets end of file" + # + # example: + # ' + return :recoverable_error + when /unexpected end-of-input/ + # "syntax error, unexpected end-of-input, expecting keyword_end" + # + # example: + # if true + # hoge + # if false + # fuga + # end + return :recoverable_error else return :other_error end diff --git a/lib/logger/period.rb b/lib/logger/period.rb index 0a291dbbbe..a0359defe3 100644 --- a/lib/logger/period.rb +++ b/lib/logger/period.rb @@ -8,14 +8,14 @@ class Logger def next_rotate_time(now, shift_age) case shift_age - when 'daily' + when 'daily', :daily t = Time.mktime(now.year, now.month, now.mday) + SiD - when 'weekly' + when 'weekly', :weekly t = Time.mktime(now.year, now.month, now.mday) + SiD * (7 - now.wday) - when 'monthly' + when 'monthly', :monthly t = Time.mktime(now.year, now.month, 1) + SiD * 32 return Time.mktime(t.year, t.month, 1) - when 'now', 'everytime' + when 'now', 'everytime', :now, :everytime return now else raise ArgumentError, "invalid :shift_age #{shift_age.inspect}, should be daily, weekly, monthly, or everytime" @@ -30,13 +30,13 @@ class Logger def previous_period_end(now, shift_age) case shift_age - when 'daily' + when 'daily', :daily t = Time.mktime(now.year, now.month, now.mday) - SiD / 2 - when 'weekly' + when 'weekly', :weekly t = Time.mktime(now.year, now.month, now.mday) - (SiD * now.wday + SiD / 2) - when 'monthly' + when 'monthly', :monthly t = Time.mktime(now.year, now.month, 1) - SiD / 2 - when 'now', 'everytime' + when 'now', 'everytime', :now, :everytime return now else raise ArgumentError, "invalid :shift_age #{shift_age.inspect}, should be daily, weekly, monthly, or everytime" diff --git a/lib/net/http.rb b/lib/net/http.rb index 6b78c264af..958ff09f0e 100644 --- a/lib/net/http.rb +++ b/lib/net/http.rb @@ -67,6 +67,8 @@ module Net #:nodoc: # Net::HTTP.post(uri, data) # params = {title: 'foo', body: 'bar', userId: 1} # Net::HTTP.post_form(uri, params) + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # Net::HTTP.put(uri, data) # # - If performance is important, consider using sessions, which lower request overhead. # This {session}[rdoc-ref:Net::HTTP@Sessions] has multiple requests for @@ -524,6 +526,8 @@ module Net #:nodoc: # Sends a POST request with form data and returns a response object. # - {::post}[rdoc-ref:Net::HTTP.post]: # Sends a POST request with data and returns a response object. + # - {::put}[rdoc-ref:Net::HTTP.put]: + # Sends a PUT request with data and returns a response object. # - {#copy}[rdoc-ref:Net::HTTP#copy]: # Sends a COPY request and returns a response object. # - {#delete}[rdoc-ref:Net::HTTP#delete]: @@ -893,6 +897,39 @@ module Net #:nodoc: } end + # Sends a PUT request to the server; returns a Net::HTTPResponse object. + # + # Argument +url+ must be a URL; + # argument +data+ must be a string: + # + # _uri = uri.dup + # _uri.path = '/posts' + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # headers = {'content-type': 'application/json'} + # res = Net::HTTP.put(_uri, data, headers) # => #<Net::HTTPCreated 201 Created readbody=true> + # puts res.body + # + # Output: + # + # { + # "title": "foo", + # "body": "bar", + # "userId": 1, + # "id": 101 + # } + # + # Related: + # + # - Net::HTTP::Put: request class for \HTTP method +PUT+. + # - Net::HTTP#put: convenience method for \HTTP method +PUT+. + # + def HTTP.put(url, data, header = nil) + start(url.hostname, url.port, + :use_ssl => url.scheme == 'https' ) {|http| + http.put(url, data, header) + } + end + # # \HTTP session management # @@ -2016,6 +2053,11 @@ module Net #:nodoc: # http = Net::HTTP.new(hostname) # http.put('/todos/1', data) # => #<Net::HTTPOK 200 OK readbody=true> # + # Related: + # + # - Net::HTTP::Put: request class for \HTTP method PUT. + # - Net::HTTP.put: sends PUT request, returns response body. + # def put(path, data, initheader = nil) request(Put.new(path, initheader), data) end diff --git a/lib/net/http/header.rb b/lib/net/http/header.rb index 6660c8075a..f6c36f1b5e 100644 --- a/lib/net/http/header.rb +++ b/lib/net/http/header.rb @@ -491,7 +491,7 @@ module Net::HTTPHeader alias canonical_each each_capitalized def capitalize(name) - name.to_s.split(/-/).map {|s| s.capitalize }.join('-') + name.to_s.split('-'.freeze).map {|s| s.capitalize }.join('-'.freeze) end private :capitalize diff --git a/lib/net/http/requests.rb b/lib/net/http/requests.rb index 5724164205..e58057adf1 100644 --- a/lib/net/http/requests.rb +++ b/lib/net/http/requests.rb @@ -124,6 +124,11 @@ end # - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. # - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. # +# Related: +# +# - Net::HTTP.put: sends +PUT+ request, returns response object. +# - Net::HTTP#put: sends +PUT+ request, returns response object. +# class Net::HTTP::Put < Net::HTTPRequest METHOD = 'PUT' REQUEST_HAS_BODY = true diff --git a/lib/prism.rb b/lib/prism.rb index 19774538e7..66a64e7fd0 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -13,7 +13,6 @@ module Prism autoload :BasicVisitor, "prism/visitor" autoload :Compiler, "prism/compiler" - autoload :Debug, "prism/debug" autoload :DesugarCompiler, "prism/desugar_compiler" autoload :Dispatcher, "prism/dispatcher" autoload :DotVisitor, "prism/dot_visitor" @@ -32,7 +31,6 @@ module Prism # Some of these constants are not meant to be exposed, so marking them as # private here. - private_constant :Debug private_constant :LexCompat private_constant :LexRipper @@ -71,8 +69,6 @@ require_relative "prism/polyfill/byteindex" require_relative "prism/node" require_relative "prism/node_ext" require_relative "prism/parse_result" -require_relative "prism/parse_result/comments" -require_relative "prism/parse_result/newlines" # This is a Ruby implementation of the prism parser. If we're running on CRuby # and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then diff --git a/lib/prism/debug.rb b/lib/prism/debug.rb deleted file mode 100644 index 74f824faa7..0000000000 --- a/lib/prism/debug.rb +++ /dev/null @@ -1,249 +0,0 @@ -# frozen_string_literal: true - -module Prism - # This module is used for testing and debugging and is not meant to be used by - # consumers of this library. - module Debug - # A wrapper around a RubyVM::InstructionSequence that provides a more - # convenient interface for accessing parts of the iseq. - class ISeq # :nodoc: - attr_reader :parts - - def initialize(parts) - @parts = parts - end - - def type - parts[0] - end - - def local_table - parts[10] - end - - def instructions - parts[13] - end - - def each_child - instructions.each do |instruction| - # Only look at arrays. Other instructions are line numbers or - # tracepoint events. - next unless instruction.is_a?(Array) - - instruction.each do |opnd| - # Only look at arrays. Other operands are literals. - next unless opnd.is_a?(Array) - - # Only look at instruction sequences. Other operands are literals. - next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat" - - yield ISeq.new(opnd) - end - end - end - end - - private_constant :ISeq - - # :call-seq: - # Debug::cruby_locals(source) -> Array - # - # For the given source, compiles with CRuby and returns a list of all of the - # sets of local variables that were encountered. - def self.cruby_locals(source) - verbose, $VERBOSE = $VERBOSE, nil - - begin - locals = [] #: Array[Array[Symbol | Integer]] - stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)] - - while (iseq = stack.pop) - names = [*iseq.local_table] - names.map!.with_index do |name, index| - # When an anonymous local variable is present in the iseq's local - # table, it is represented as the stack offset from the top. - # However, when these are dumped to binary and read back in, they - # are replaced with the symbol :#arg_rest. To consistently handle - # this, we replace them here with their index. - if name == :"#arg_rest" - names.length - index + 1 - else - name - end - end - - locals << names - iseq.each_child { |child| stack << child } - end - - locals - ensure - $VERBOSE = verbose - end - end - - # Used to hold the place of a local that will be in the local table but - # cannot be accessed directly from the source code. For example, the - # iteration variable in a for loop or the positional parameter on a method - # definition that is destructured. - AnonymousLocal = Object.new - private_constant :AnonymousLocal - - # :call-seq: - # Debug::prism_locals(source) -> Array - # - # For the given source, parses with prism and returns a list of all of the - # sets of local variables that were encountered. - def self.prism_locals(source) - locals = [] #: Array[Array[Symbol | Integer]] - stack = [Prism.parse(source).value] #: Array[Prism::node] - - while (node = stack.pop) - case node - when BlockNode, DefNode, LambdaNode - names = node.locals - params = - if node.is_a?(DefNode) - node.parameters - elsif node.parameters.is_a?(NumberedParametersNode) - nil - else - node.parameters&.parameters - end - - # prism places parameters in the same order that they appear in the - # source. CRuby places them in the order that they need to appear - # according to their own internal calling convention. We mimic that - # order here so that we can compare properly. - if params - sorted = [ - *params.requireds.map do |required| - if required.is_a?(RequiredParameterNode) - required.name - else - AnonymousLocal - end - end, - *params.optionals.map(&:name), - *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)), - *params.posts.map do |post| - if post.is_a?(RequiredParameterNode) - post.name - else - AnonymousLocal - end - end, - *params.keywords.grep(RequiredKeywordParameterNode).map(&:name), - *params.keywords.grep(OptionalKeywordParameterNode).map(&:name), - ] - - sorted << AnonymousLocal if params.keywords.any? - - if params.keyword_rest.is_a?(ForwardingParameterNode) - sorted.push(:*, :**, :&, :"...") - elsif params.keyword_rest.is_a?(KeywordRestParameterNode) - sorted << (params.keyword_rest.name || :**) - end - - # Recurse down the parameter tree to find any destructured - # parameters and add them after the other parameters. - param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse - while (param = param_stack.pop) - case param - when MultiTargetNode - param_stack.concat(param.rights.reverse) - param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name) - param_stack.concat(param.lefts.reverse) - when RequiredParameterNode - sorted << param.name - when SplatNode - sorted << param.expression.name - end - end - - if params.block - sorted << (params.block.name || :&) - end - - names = sorted.concat(names - sorted) - end - - names.map!.with_index do |name, index| - if name == AnonymousLocal - names.length - index + 1 - else - name - end - end - - locals << names - when ClassNode, ModuleNode, ProgramNode, SingletonClassNode - locals << node.locals - when ForNode - locals << [2] - when PostExecutionNode - locals.push([], []) - when InterpolatedRegularExpressionNode - locals << [] if node.once? - end - - stack.concat(node.compact_child_nodes) - end - - locals - end - - # :call-seq: - # Debug::newlines(source) -> Array - # - # For the given source string, return the byte offsets of every newline in - # the source. - def self.newlines(source) - Prism.parse(source).source.offsets - end - - # A wrapping around prism's internal encoding data structures. This is used - # for reflection and debugging purposes. - class Encoding - # The name of the encoding, that can be passed to Encoding.find. - attr_reader :name - - # Initialize a new encoding with the given name and whether or not it is - # a multibyte encoding. - def initialize(name, multibyte) - @name = name - @multibyte = multibyte - end - - # Whether or not the encoding is a multibyte encoding. - def multibyte? - @multibyte - end - - # Returns the number of bytes of the first character in the source string, - # if it is valid for the encoding. Otherwise, returns 0. - def width(source) - Encoding._width(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphanumeric character for the encoding. - def alnum?(source) - Encoding._alnum?(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphabetic character for the encoding. - def alpha?(source) - Encoding._alpha?(name, source) - end - - # Returns true if the first character in the source string is a valid - # uppercase character for the encoding. - def upper?(source) - Encoding._upper?(name, source) - end - end - end -end diff --git a/lib/prism/desugar_compiler.rb b/lib/prism/desugar_compiler.rb index 9b62c00df3..de02445149 100644 --- a/lib/prism/desugar_compiler.rb +++ b/lib/prism/desugar_compiler.rb @@ -73,7 +73,7 @@ module Prism # Desugar `x += y` to `x = x + y` def compile - operator_loc = node.operator_loc.chop + binary_operator_loc = node.binary_operator_loc.chop write_class.new( source, @@ -84,15 +84,15 @@ module Prism 0, read_class.new(source, *arguments, node.name_loc), nil, - operator_loc.slice.to_sym, - operator_loc, + binary_operator_loc.slice.to_sym, + binary_operator_loc, nil, ArgumentsNode.new(source, 0, [node.value], node.value.location), nil, nil, node.location ), - node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.binary_operator_loc.copy(start_offset: node.binary_operator_loc.end_offset - 1, length: 1), node.location ) end diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index b62a59d037..6b48af43cc 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -200,8 +200,8 @@ module Prism class << self # Mirror the Prism.dump API by using the serialization API. - def dump(code, **options) - LibRubyParser::PrismString.with_string(code) { |string| dump_common(string, options) } + def dump(source, **options) + LibRubyParser::PrismString.with_string(source) { |string| dump_common(string, options) } end # Mirror the Prism.dump_file API by using the serialization API. @@ -302,6 +302,27 @@ module Prism !parse_file_success?(filepath, **options) end + # Mirror the Prism.profile API by using the serialization API. + def profile(source, **options) + LibRubyParser::PrismString.with_string(source) do |string| + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options)) + nil + end + end + end + + # Mirror the Prism.profile_file API by using the serialization API. + def profile_file(filepath, **options) + LibRubyParser::PrismString.with_file(filepath) do |string| + LibRubyParser::PrismBuffer.with do |buffer| + options[:filepath] = filepath + LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options)) + nil + end + end + end + private def dump_common(string, options) # :nodoc: @@ -394,7 +415,7 @@ module Prism template << "L" if (encoding = options[:encoding]) - name = encoding.name + name = encoding.is_a?(Encoding) ? encoding.name : encoding values.push(name.bytesize, name.b) template << "A*" else diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb index d4c9b21079..aa6a18cf29 100644 --- a/lib/prism/node_ext.rb +++ b/lib/prism/node_ext.rb @@ -3,6 +3,20 @@ # Here we are reopening the prism module to provide methods on nodes that aren't # templated and are meant as convenience methods. module Prism + class Node + def deprecated(*replacements) # :nodoc: + location = caller_locations(1, 1) + location = location[0].label if location + suggest = replacements.map { |replacement| "#{self.class}##{replacement}" } + + warn(<<~MSG, category: :deprecated) + [deprecation]: #{self.class}##{location} is deprecated and will be \ + removed in the next major version. Use #{suggest.join("/")} instead. + #{(caller(1, 3) || []).join("\n")} + MSG + end + end + module RegularExpressionOptions # :nodoc: # Returns a numeric value that represents the flags that were used to create # the regular expression. @@ -92,7 +106,19 @@ module Prism class RationalNode < Node # Returns the value of the node as a Ruby Rational. def value - Rational(numeric.is_a?(IntegerNode) ? numeric.value : slice.chomp("r")) + Rational(numerator, denominator) + end + + # Returns the value of the node as an IntegerNode or a FloatNode. This + # method is deprecated in favor of #value or #numerator/#denominator. + def numeric + deprecated("value", "numerator", "denominator") + + if denominator == 1 + IntegerNode.new(source, flags, numerator, location.chop) + else + FloatNode.new(source, numerator.to_f / denominator, location.chop) + end end end @@ -168,13 +194,7 @@ module Prism # constant read or a missing node. To not cause a breaking change, we # continue to supply that API. def child - warn(<<~MSG, category: :deprecated) - DEPRECATED: ConstantPathNode#child is deprecated and will be removed \ - in the next major version. Use \ - ConstantPathNode#name/ConstantPathNode#name_loc instead. Called from \ - #{caller(1, 1)&.first}. - MSG - + deprecated("name", "name_loc") name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) end end @@ -210,13 +230,7 @@ module Prism # constant read or a missing node. To not cause a breaking change, we # continue to supply that API. def child - warn(<<~MSG, category: :deprecated) - DEPRECATED: ConstantPathTargetNode#child is deprecated and will be \ - removed in the next major version. Use \ - ConstantPathTargetNode#name/ConstantPathTargetNode#name_loc instead. \ - Called from #{caller(1, 1)&.first}. - MSG - + deprecated("name", "name_loc") name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) end end @@ -250,9 +264,10 @@ module Prism end posts.each do |param| - if param.is_a?(MultiTargetNode) + case param + when MultiTargetNode names << [:req] - elsif param.is_a?(NoKeywordsParameterNode) + when NoKeywordsParameterNode, KeywordRestParameterNode, ForwardingParameterNode # Invalid syntax, e.g. "def f(**nil, ...)" moves the NoKeywordsParameterNode to posts raise "Invalid syntax" else @@ -286,4 +301,147 @@ module Prism names end end + + class CallNode < Node + # When a call node has the attribute_write flag set, it means that the call + # is using the attribute write syntax. This is either a method call to []= + # or a method call to a method that ends with =. Either way, the = sign is + # present in the source. + # + # Prism returns the message_loc _without_ the = sign attached, because there + # can be any amount of space between the message and the = sign. However, + # sometimes you want the location of the full message including the inner + # space and the = sign. This method provides that. + def full_message_loc + attribute_write? ? message_loc&.adjoin("=") : message_loc + end + end + + class CallOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ClassVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantPathOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class GlobalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class IndexOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class InstanceVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class LocalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end end diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb index 63cc72a966..798fde09e5 100644 --- a/lib/prism/parse_result.rb +++ b/lib/prism/parse_result.rb @@ -574,6 +574,12 @@ module Prism # This is a result specific to the `parse` and `parse_file` methods. class ParseResult < Result + autoload :Comments, "prism/parse_result/comments" + autoload :Newlines, "prism/parse_result/newlines" + + private_constant :Comments + private_constant :Newlines + # The syntax tree that was parsed from the source code. attr_reader :value @@ -587,6 +593,17 @@ module Prism def deconstruct_keys(keys) super.merge!(value: value) end + + # Attach the list of comments to their respective locations in the tree. + def attach_comments! + Comments.new(self).attach! # steep:ignore + end + + # Walk the tree and mark nodes that are on a new line, loosely emulating + # the behavior of CRuby's `:line` tracepoint event. + def mark_newlines! + value.accept(Newlines.new(source.offsets.size)) # steep:ignore + end end # This is a result specific to the `lex` and `lex_file` methods. diff --git a/lib/prism/parse_result/comments.rb b/lib/prism/parse_result/comments.rb index f8f74d2503..22c4148b2c 100644 --- a/lib/prism/parse_result/comments.rb +++ b/lib/prism/parse_result/comments.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # When we've parsed the source, we have both the syntax tree and the list of # comments that we found in the source. This class is responsible for # walking the tree and finding the nearest location to attach each comment. @@ -183,12 +183,5 @@ module Prism [preceding, NodeTarget.new(node), following] end end - - private_constant :Comments - - # Attach the list of comments to their respective locations in the tree. - def attach_comments! - Comments.new(self).attach! # steep:ignore - end end end diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index 927c17fe2f..808a129a6b 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # The :line tracepoint event gets fired whenever the Ruby VM encounters an # expression on a new line. The types of expressions that can trigger this # event are: @@ -17,21 +17,27 @@ module Prism # Note that the logic in this file should be kept in sync with the Java # MarkNewlinesVisitor, since that visitor is responsible for marking the # newlines for JRuby/TruffleRuby. + # + # This file is autoloaded only when `mark_newlines!` is called, so the + # re-opening of the various nodes in this file will only be performed in + # that case. We do that to avoid storing the extra `@newline` instance + # variable on every node if we don't need it. class Newlines < Visitor # Create a new Newlines visitor with the given newline offsets. - def initialize(newline_marked) - @newline_marked = newline_marked + def initialize(lines) + # @type var lines: Integer + @lines = Array.new(1 + lines, false) end # Permit block/lambda nodes to mark newlines within themselves. def visit_block_node(node) - old_newline_marked = @newline_marked - @newline_marked = Array.new(old_newline_marked.size, false) + old_lines = @lines + @lines = Array.new(old_lines.size, false) begin super(node) ensure - @newline_marked = old_newline_marked + @lines = old_lines end end @@ -39,7 +45,7 @@ module Prism # Mark if/unless nodes as newlines. def visit_if_node(node) - node.set_newline_flag(@newline_marked) + node.newline!(@lines) super(node) end @@ -48,17 +54,101 @@ module Prism # Permit statements lists to mark newlines within themselves. def visit_statements_node(node) node.body.each do |child| - child.set_newline_flag(@newline_marked) + child.newline!(@lines) end super(node) end end + end + + class Node + def newline? # :nodoc: + @newline ? true : false + end + + def newline!(lines) # :nodoc: + line = location.start_line + unless lines[line] + lines[line] = true + @newline = true + end + end + end + + class BeginNode < Node + def newline!(lines) # :nodoc: + # Never mark BeginNode with a newline flag, mark children instead. + end + end + + class ParenthesesNode < Node + def newline!(lines) # :nodoc: + # Never mark ParenthesesNode with a newline flag, mark children instead. + end + end + + class IfNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UnlessNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UntilNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class WhileNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class RescueModifierNode < Node + def newline!(lines) # :nodoc: + expression.newline!(lines) + end + end + + class InterpolatedMatchLastLineNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedRegularExpressionNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - private_constant :Newlines + class InterpolatedSymbolNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - # Walk the tree and mark nodes that are on a new line. - def mark_newlines! - value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore + class InterpolatedXStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first end end end diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 8a3efe826d..b39ba706dc 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "0.27.0" + spec.version = "0.30.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -65,12 +65,10 @@ Gem::Specification.new do |spec| "include/prism/util/pm_newline_list.h", "include/prism/util/pm_strncasecmp.h", "include/prism/util/pm_string.h", - "include/prism/util/pm_string_list.h", "include/prism/util/pm_strpbrk.h", "include/prism/version.h", "lib/prism.rb", "lib/prism/compiler.rb", - "lib/prism/debug.rb", "lib/prism/desugar_compiler.rb", "lib/prism/dispatcher.rb", "lib/prism/dot_visitor.rb", @@ -121,6 +119,7 @@ Gem::Specification.new do |spec| "sig/prism/dot_visitor.rbs", "sig/prism/dsl.rbs", "sig/prism/inspect_visitor.rbs", + "sig/prism/lex_compat.rbs", "sig/prism/mutation_compiler.rbs", "sig/prism/node_ext.rbs", "sig/prism/node.rbs", @@ -148,7 +147,6 @@ Gem::Specification.new do |spec| "src/util/pm_list.c", "src/util/pm_memchr.c", "src/util/pm_newline_list.c", - "src/util/pm_string_list.c", "src/util/pm_string.c", "src/util/pm_strncasecmp.c", "src/util/pm_strpbrk.c" diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index 193bbae406..3748fc896e 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "parser" +begin + require "parser" +rescue LoadError + warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index c311613388..48f3d4db5c 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -90,7 +90,11 @@ module Prism end if node.constant - builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc)) + if visited.empty? + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)), token(node.closing_loc)) + else + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc)) + end else builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)) end @@ -105,38 +109,46 @@ module Prism # { a: 1 } # ^^^^ def visit_assoc_node(node) + key = node.key + if in_pattern if node.value.is_a?(ImplicitNode) - if node.key.is_a?(SymbolNode) - builder.match_hash_var([node.key.unescaped, srange(node.key.location)]) + if key.is_a?(SymbolNode) + if key.opening.nil? + builder.match_hash_var([key.unescaped, srange(key.location)]) + else + builder.match_hash_var_from_str(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc)) + end else - builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc)) + builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc)) end + elsif key.opening.nil? + builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value)) else - builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + builder.pair_quoted(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc), visit(node.value)) end elsif node.value.is_a?(ImplicitNode) if (value = node.value.value).is_a?(LocalVariableReadNode) builder.pair_keyword( - [node.key.unescaped, srange(node.key)], - builder.ident([value.name, srange(node.key.value_loc)]).updated(:lvar) + [key.unescaped, srange(key)], + builder.ident([value.name, srange(key.value_loc)]).updated(:lvar) ) else - builder.pair_label([node.key.unescaped, srange(node.key.location)]) + builder.pair_label([key.unescaped, srange(key.location)]) end elsif node.operator_loc - builder.pair(visit(node.key), token(node.operator_loc), visit(node.value)) - elsif node.key.is_a?(SymbolNode) && node.key.opening_loc.nil? - builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + builder.pair(visit(key), token(node.operator_loc), visit(node.value)) + elsif key.is_a?(SymbolNode) && key.opening_loc.nil? + builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value)) else parts = - if node.key.is_a?(SymbolNode) - [builder.string_internal([node.key.unescaped, srange(node.key.value_loc)])] + if key.is_a?(SymbolNode) + [builder.string_internal([key.unescaped, srange(key.value_loc)])] else - visit_all(node.key.parts) + visit_all(key.parts) end - builder.pair_quoted(token(node.key.opening_loc), parts, token(node.key.closing_loc), visit(node.value)) + builder.pair_quoted(token(key.opening_loc), parts, token(key.closing_loc), visit(node.value)) end end @@ -146,7 +158,9 @@ module Prism # { **foo } # ^^^^^ def visit_assoc_splat_node(node) - if node.value.nil? && forwarding.include?(:**) + if in_pattern + builder.match_rest(token(node.operator_loc), token(node.value&.location)) + elsif node.value.nil? && forwarding.include?(:**) builder.forwarded_kwrestarg(token(node.operator_loc)) else builder.kwsplat(token(node.operator_loc), visit(node.value)) @@ -328,18 +342,48 @@ module Prism [], nil ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo.bar &&= baz # ^^^^^^^^^^^^^^^ - alias visit_call_and_write_node visit_call_operator_write_node + def visit_call_and_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar ||= baz # ^^^^^^^^^^^^^^^ - alias visit_call_or_write_node visit_call_operator_write_node + def visit_call_or_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar, = 1 # ^^^^^^^ @@ -419,18 +463,30 @@ module Prism def visit_class_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.cvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @@foo &&= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_and_write_node visit_class_variable_operator_write_node + def visit_class_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo ||= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_or_write_node visit_class_variable_operator_write_node + def visit_class_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo, = bar # ^^^^^ @@ -458,18 +514,30 @@ module Prism def visit_constant_operator_write_node(node) builder.op_assign( builder.assignable(builder.const([node.name, srange(node.name_loc)])), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo &&= bar # ^^^^^^^^^^^^ - alias visit_constant_and_write_node visit_constant_operator_write_node + def visit_constant_and_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo ||= bar # ^^^^^^^^^^^^ - alias visit_constant_or_write_node visit_constant_operator_write_node + def visit_constant_or_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo, = bar # ^^^ @@ -512,18 +580,30 @@ module Prism def visit_constant_path_operator_write_node(node) builder.op_assign( builder.assignable(visit(node.target)), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo::Bar &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_and_write_node visit_constant_path_operator_write_node + def visit_constant_path_and_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_or_write_node visit_constant_path_operator_write_node + def visit_constant_path_or_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar, = baz # ^^^^^^^^ @@ -711,18 +791,30 @@ module Prism def visit_global_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.gvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # $foo &&= bar # ^^^^^^^^^^^^ - alias visit_global_variable_and_write_node visit_global_variable_operator_write_node + def visit_global_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo ||= bar # ^^^^^^^^^^^^ - alias visit_global_variable_or_write_node visit_global_variable_operator_write_node + def visit_global_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo, = bar # ^^^^ @@ -803,7 +895,7 @@ module Prism # 1i # ^^ def visit_imaginary_node(node) - visit_numeric(node, builder.complex([imaginary_value(node), srange(node.location)])) + visit_numeric(node, builder.complex([Complex(0, node.numeric.value), srange(node.location)])) end # { foo: } @@ -857,18 +949,46 @@ module Prism visit_all(arguments), token(node.closing_loc) ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo[bar] &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_and_write_node visit_index_operator_write_node + def visit_index_and_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar] ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_or_write_node visit_index_operator_write_node + def visit_index_or_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar], = 1 # ^^^^^^^^ @@ -902,18 +1022,30 @@ module Prism def visit_instance_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ivar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @foo &&= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_and_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo ||= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_or_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo, = bar # ^^^^ @@ -946,36 +1078,7 @@ module Prism # ^^^^^^^^^^^^ def visit_interpolated_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node) - opening = token(node.opening_loc) - - start_offset = node.opening_loc.end_offset + 1 - end_offset = node.parts.first.location.start_offset - - # In the below case, the offsets should be the same: - # - # <<~HEREDOC - # a #{b} - # HEREDOC - # - # But in this case, the end_offset would be greater than the start_offset: - # - # <<~HEREDOC - # #{b} - # HEREDOC - # - # So we need to make sure the result node's heredoc range is correct, without updating the children - result = if start_offset < end_offset - # We need to add a padding string to ensure that the heredoc has correct range for its body - padding_string_node = builder.string_internal(["", srange_offsets(start_offset, end_offset)]) - node_with_correct_location = builder.string_compose(opening, [padding_string_node, *children], closing) - # But the padding string should not be included in the final AST, so we need to update the result's children - node_with_correct_location.updated(:dstr, children) - else - builder.string_compose(opening, children, closing) - end - - return result + return visit_heredoc(node) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) } end parts = if node.parts.one? { |part| part.type == :string_node } @@ -1019,8 +1122,7 @@ module Prism # ^^^^^^^^^^^^ def visit_interpolated_x_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node) - builder.xstring_compose(token(node.opening_loc), children, closing) + visit_heredoc(node) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) } else builder.xstring_compose( token(node.opening_loc), @@ -1031,6 +1133,12 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + builder.ident([:it, srange(node.location)]).updated(:lvar) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) builder.args(nil, [], nil, false) @@ -1083,14 +1191,7 @@ module Prism # foo # ^^^ def visit_local_variable_read_node(node) - name = node.name - - # This is just a guess. parser doesn't have support for the implicit - # `it` variable yet, so we'll probably have to visit this once it - # does. - name = :it if name == :"0it" - - builder.ident([name, srange(node.location)]).updated(:lvar) + builder.ident([node.name, srange(node.location)]).updated(:lvar) end # foo = 1 @@ -1108,18 +1209,30 @@ module Prism def visit_local_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ident(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo &&= bar # ^^^^^^^^^^^ - alias visit_local_variable_and_write_node visit_local_variable_operator_write_node + def visit_local_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo ||= bar # ^^^^^^^^^^^ - alias visit_local_variable_or_write_node visit_local_variable_operator_write_node + def visit_local_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo, = bar # ^^^ @@ -1182,13 +1295,9 @@ module Prism # foo, bar = baz # ^^^^^^^^ def visit_multi_target_node(node) - elements = [*node.lefts] - elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) - elements.concat(node.rights) - builder.multi_lhs( token(node.lparen_loc), - visit_all(elements), + visit_all(multi_target_elements(node)), token(node.rparen_loc) ) end @@ -1196,9 +1305,11 @@ module Prism # foo, bar = baz # ^^^^^^^^^^^^^^ def visit_multi_write_node(node) - elements = [*node.lefts] - elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) - elements.concat(node.rights) + elements = multi_target_elements(node) + + if elements.length == 1 && elements.first.is_a?(MultiTargetNode) + elements = multi_target_elements(elements.first) + end builder.multi_assign( builder.multi_lhs( @@ -1279,12 +1390,12 @@ module Prism if node.requireds.any? node.requireds.each do |required| - if required.is_a?(RequiredParameterNode) - params << visit(required) - else - compiler = copy_compiler(in_destructure: true) - params << required.accept(compiler) - end + params << + if required.is_a?(RequiredParameterNode) + visit(required) + else + required.accept(copy_compiler(in_destructure: true)) + end end end @@ -1293,12 +1404,12 @@ module Prism if node.posts.any? node.posts.each do |post| - if post.is_a?(RequiredParameterNode) - params << visit(post) - else - compiler = copy_compiler(in_destructure: true) - params << post.accept(compiler) - end + params << + if post.is_a?(RequiredParameterNode) + visit(post) + else + post.accept(copy_compiler(in_destructure: true)) + end end end @@ -1384,7 +1495,7 @@ module Prism # 1r # ^^ def visit_rational_node(node) - visit_numeric(node, builder.rational([rational_value(node), srange(node.location)])) + visit_numeric(node, builder.rational([node.value, srange(node.location)])) end # redo @@ -1396,9 +1507,20 @@ module Prism # /foo/ # ^^^^^ def visit_regular_expression_node(node) + content = node.content + parts = + if content.include?("\n") + offset = node.content_loc.start_offset + content.lines.map do |line| + builder.string_internal([line, srange_offsets(offset, offset += line.bytesize)]) + end + else + [builder.string_internal(token(node.content_loc))] + end + builder.regexp_compose( token(node.opening_loc), - [builder.string_internal(token(node.content_loc))], + parts, [node.closing[0], srange_offsets(node.closing_loc.start_offset, node.closing_loc.start_offset + 1)], builder.regexp_options([node.closing[1..], srange_offsets(node.closing_loc.start_offset + 1, node.closing_loc.end_offset)]) ) @@ -1544,10 +1666,11 @@ module Prism # ^^^^^ def visit_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node.to_interpolated) - builder.string_compose(token(node.opening_loc), children, closing) + visit_heredoc(node.to_interpolated) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) } elsif node.opening == "?" builder.character([node.unescaped, srange(node.location)]) + elsif node.opening&.start_with?("%") && node.unescaped.empty? + builder.string_compose(token(node.opening_loc), [], token(node.closing_loc)) else content_lines = node.content.lines unescaped_lines = node.unescaped.lines @@ -1747,8 +1870,7 @@ module Prism # ^^^^^ def visit_x_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node.to_interpolated) - builder.xstring_compose(token(node.opening_loc), children, closing) + visit_heredoc(node.to_interpolated) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) } else parts = if node.unescaped.lines.one? [builder.string_internal([node.unescaped, srange(node.content_loc)])] @@ -1810,10 +1932,12 @@ module Prism forwarding end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def imaginary_value(node) - Complex(0, node.numeric.is_a?(RationalNode) ? rational_value(node.numeric) : node.numeric.value) + # Returns the set of targets for a MultiTargetNode or a MultiWriteNode. + def multi_target_elements(node) + elements = [*node.lefts] + elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + elements.concat(node.rights) + elements end # Negate the value of a numeric node. This is a special case where you @@ -1825,7 +1949,9 @@ module Prism case receiver.type when :integer_node, :float_node receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location)) - when :rational_node, :imaginary_node + when :rational_node + receiver.copy(numerator: -receiver.numerator, location: message_loc.join(receiver.location)) + when :imaginary_node receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location)) end end @@ -1844,16 +1970,6 @@ module Prism parameters.block.nil? end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def rational_value(node) - if node.numeric.is_a?(IntegerNode) - Rational(node.numeric.value) - else - Rational(node.slice.gsub(/\s/, "").chomp("r")) - end - end - # Locations in the parser gem AST are generated using this class. We # store a reference to its constant to make it slightly faster to look # up. @@ -1876,7 +1992,7 @@ module Prism # Note that end_offset is allowed to be nil, in which case this will # search until the end of the string. def srange_find(start_offset, end_offset, tokens) - if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/(\s*)(#{tokens.join("|")})/)) + if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/\A(\s*)(#{tokens.join("|")})/)) _, whitespace, token = *match token_offset = start_offset + whitespace.bytesize @@ -1907,7 +2023,8 @@ module Prism token(parameters.opening_loc), if procarg0?(parameters.parameters) parameter = parameters.parameters.requireds.first - [builder.procarg0(visit(parameter))].concat(visit_all(parameters.locals)) + visited = parameter.is_a?(RequiredParameterNode) ? visit(parameter) : parameter.accept(copy_compiler(in_destructure: true)) + [builder.procarg0(visited)].concat(visit_all(parameters.locals)) else visit(parameters) end, @@ -1923,29 +2040,55 @@ module Prism end end + # The parser gem automatically converts \r\n to \n, meaning our offsets + # need to be adjusted to always subtract 1 from the length. + def chomped_bytesize(line) + chomped = line.chomp + chomped.bytesize + (chomped == line ? 0 : 1) + end + # Visit a heredoc that can be either a string or an xstring. def visit_heredoc(node) children = Array.new + indented = false + + # If this is a dedenting heredoc, then we need to insert the opening + # content into the children as well. + if node.opening.start_with?("<<~") && node.parts.length > 0 && !node.parts.first.is_a?(StringNode) + location = node.parts.first.location + location = location.copy(start_offset: location.start_offset - location.start_line_slice.bytesize) + children << builder.string_internal(token(location)) + indented = true + end + node.parts.each do |part| pushing = if part.is_a?(StringNode) && part.unescaped.include?("\n") - unescaped = part.unescaped.lines(chomp: true) - escaped = part.content.lines(chomp: true) + unescaped = part.unescaped.lines + escaped = part.content.lines - escaped_lengths = - if node.opening.end_with?("'") - escaped.map { |line| line.bytesize + 1 } - else - escaped.chunk_while { |before, after| before.match?(/(?<!\\)\\$/) }.map { |line| line.join.bytesize + line.length } + escaped_lengths = [] + normalized_lengths = [] + + if node.opening.end_with?("'") + escaped.each do |line| + escaped_lengths << line.bytesize + normalized_lengths << chomped_bytesize(line) end + else + escaped + .chunk_while { |before, after| before.match?(/(?<!\\)\\\r?\n$/) } + .each do |lines| + escaped_lengths << lines.sum(&:bytesize) + normalized_lengths << lines.sum { |line| chomped_bytesize(line) } + end + end start_offset = part.location.start_offset - end_offset = nil - unescaped.zip(escaped_lengths).map do |unescaped_line, escaped_length| - end_offset = start_offset + (escaped_length || 0) - inner_part = builder.string_internal(["#{unescaped_line}\n", srange_offsets(start_offset, end_offset)]) - start_offset = end_offset + unescaped.map.with_index do |unescaped_line, index| + inner_part = builder.string_internal([unescaped_line, srange_offsets(start_offset, start_offset + normalized_lengths.fetch(index, 0))]) + start_offset += escaped_lengths.fetch(index, 0) inner_part end else @@ -1956,7 +2099,12 @@ module Prism if child.type == :str && child.children.last == "" # nothing elsif child.type == :str && children.last && children.last.type == :str && !children.last.children.first.end_with?("\n") - children.last.children.first << child.children.first + appendee = children[-1] + + location = appendee.loc + location = location.with_expression(location.expression.join(child.loc.expression)) + + children[-1] = appendee.updated(:str, [appendee.children.first << child.children.first], location: location) else children << child end @@ -1965,8 +2113,10 @@ module Prism closing = node.closing closing_t = [closing.chomp, srange_offsets(node.closing_loc.start_offset, node.closing_loc.end_offset - (closing[/\s+$/]&.length || 0))] + composed = yield children, closing_t - [children, closing_t] + composed = composed.updated(nil, children[1..-1]) if indented + composed end # Visit a numeric node and account for the optional sign. diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index d226bb58df..79ba0e7ab3 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -1181,8 +1181,8 @@ module Prism bounds(node.location) target = on_field(receiver, call_operator, message) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1339,8 +1339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_cvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1409,8 +1409,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_const(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1510,8 +1510,8 @@ module Prism target = visit_constant_path_write_node_target(node.target) value = visit(node.value) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1802,8 +1802,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_gvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1983,8 +1983,8 @@ module Prism bounds(node.location) target = on_aref_field(receiver, arguments) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2059,8 +2059,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ivar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2218,6 +2218,13 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + bounds(node.location) + on_vcall(on_ident(node.slice)) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) end @@ -2312,12 +2319,7 @@ module Prism # ^^^ def visit_local_variable_read_node(node) bounds(node.location) - - if node.name == :"0it" - on_vcall(on_ident(node.slice)) - else - on_var_ref(on_ident(node.slice)) - end + on_var_ref(on_ident(node.slice)) end # foo = 1 @@ -2337,8 +2339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ident(node.name_loc.slice)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index c6f741c8e5..38690c54b3 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "ruby_parser" +begin + require "ruby_parser" +rescue LoadError + warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation @@ -271,9 +276,9 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_call_operator_write_node(node) if op_asgn?(node) - s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.operator) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator) else - s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.operator, visit_write_value(node.value)) + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value)) end end @@ -372,7 +377,7 @@ module Prism # @@foo += bar # ^^^^^^^^^^^^ def visit_class_variable_operator_write_node(node) - s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.operator, visit_write_value(node.value))) + s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value))) end # @@foo &&= bar @@ -417,7 +422,7 @@ module Prism # Foo += bar # ^^^^^^^^^^^ def visit_constant_operator_write_node(node) - s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.operator, visit_write_value(node.value))) + s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value))) end # Foo &&= bar @@ -460,7 +465,7 @@ module Prism # Foo::Bar += baz # ^^^^^^^^^^^^^^^ def visit_constant_path_operator_write_node(node) - s(node, :op_asgn, visit(node.target), node.operator, visit_write_value(node.value)) + s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value)) end # Foo::Bar &&= baz @@ -480,9 +485,9 @@ module Prism def visit_constant_path_target_node(node) inner = if node.parent.nil? - s(node, :colon3, node.child.name) + s(node, :colon3, node.name) else - s(node, :colon2, visit(node.parent), node.child.name) + s(node, :colon2, visit(node.parent), node.name) end s(node, :const, inner) @@ -627,7 +632,7 @@ module Prism # $foo += bar # ^^^^^^^^^^^ def visit_global_variable_operator_write_node(node) - s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.operator, visit(node.value))) + s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value))) end # $foo &&= bar @@ -719,7 +724,7 @@ module Prism arglist << visit(node.block) if !node.block.nil? end - s(node, :op_asgn1, visit(node.receiver), arglist, node.operator, visit_write_value(node.value)) + s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value)) end # foo[bar] &&= baz @@ -775,7 +780,7 @@ module Prism # @foo += bar # ^^^^^^^^^^^ def visit_instance_variable_operator_write_node(node) - s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.operator, visit_write_value(node.value))) + s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value))) end # @foo &&= bar @@ -870,6 +875,8 @@ module Prism else visited << result end + elsif result[0] == :dstr + visited.concat(result[1..-1]) else visited << result end @@ -900,12 +907,23 @@ module Prism results << result state = :interpolated_content end - else - results << result + when :interpolated_content + if result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line) + results[-1][1] << result[1] + results[-1].line_max = result.line_max + else + results << result + end end end end + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + s(node, :call, nil, :it) + end + # foo(bar: baz) # ^^^^^^^^ def visit_keyword_hash_node(node) @@ -960,7 +978,7 @@ module Prism # foo += bar # ^^^^^^^^^^ def visit_local_variable_operator_write_node(node) - s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.operator, visit_write_value(node.value))) + s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value))) end # foo &&= bar diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb index 5c72a5f224..1416059763 100644 --- a/lib/rdoc/markdown.rb +++ b/lib/rdoc/markdown.rb @@ -6,7 +6,7 @@ # RDoc::Markdown as described by the [markdown syntax][syntax]. # # To choose Markdown as your only default format see -# RDoc::Options@Saved+Options for instructions on setting up a `.doc_options` +# RDoc::Options@Saved+Options for instructions on setting up a `.rdoc_options` # file to store your project default. # # ## Usage diff --git a/lib/rdoc/ri/driver.rb b/lib/rdoc/ri/driver.rb index 64783dc163..c6fddbac67 100644 --- a/lib/rdoc/ri/driver.rb +++ b/lib/rdoc/ri/driver.rb @@ -110,10 +110,6 @@ class RDoc::RI::Driver options = default_options opts = OptionParser.new do |opt| - opt.accept File do |file,| - File.readable?(file) and not File.directory?(file) and file - end - opt.program_name = File.basename $0 opt.version = RDoc::VERSION opt.release = nil @@ -345,9 +341,17 @@ or the PAGER environment variable. opt.separator nil - opt.on("--dump=CACHE", File, + opt.on("--dump=CACHE", "Dump data from an ri cache or data file.") do |value| - options[:dump_path] = value + unless File.readable?(value) + abort "#{value.inspect} is not readable" + end + + if File.directory?(value) + abort "#{value.inspect} is a directory" + end + + options[:dump_path] = File.new(value) end end diff --git a/lib/rdoc/version.rb b/lib/rdoc/version.rb index 87842d9847..427d4ae232 100644 --- a/lib/rdoc/version.rb +++ b/lib/rdoc/version.rb @@ -5,6 +5,6 @@ module RDoc ## # RDoc version you are using - VERSION = '6.6.3.1' + VERSION = '6.7.0' end diff --git a/lib/reline.rb b/lib/reline.rb index d5bf2e363b..720df286a1 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -7,6 +7,7 @@ require 'reline/key_stroke' require 'reline/line_editor' require 'reline/history' require 'reline/terminfo' +require 'reline/io' require 'reline/face' require 'rbconfig' @@ -18,20 +19,10 @@ module Reline class ConfigEncodingConversionError < StandardError; end Key = Struct.new(:char, :combined_char, :with_meta) do - def match?(other) - case other - when Reline::Key - (other.char.nil? or char.nil? or char == other.char) and - (other.combined_char.nil? or combined_char.nil? or combined_char == other.combined_char) and - (other.with_meta.nil? or with_meta.nil? or with_meta == other.with_meta) - when Integer, Symbol - (combined_char and combined_char == other) or - (combined_char.nil? and char and char == other) - else - false - end + # For dialog_proc `key.match?(dialog.name)` + def match?(sym) + combined_char.is_a?(Symbol) && combined_char == sym end - alias_method :==, :match? end CursorPos = Struct.new(:x, :y) DialogRenderInfo = Struct.new( @@ -263,7 +254,6 @@ module Reline raise ArgumentError.new('#readmultiline needs block to confirm multiline termination') end - Reline.update_iogate io_gate.with_raw_input do inner_readline(prompt, add_hist, true, &confirm_multiline_termination) end @@ -286,7 +276,6 @@ module Reline def readline(prompt = '', add_hist = false) @mutex.synchronize do - Reline.update_iogate io_gate.with_raw_input do inner_readline(prompt, add_hist, false) end @@ -312,6 +301,10 @@ module Reline $stderr.sync = true $stderr.puts "Reline is used by #{Process.pid}" end + unless config.test_mode or config.loaded? + config.read + io_gate.set_default_key_bindings(config) + end otio = io_gate.prep may_req_ambiguous_char_width @@ -332,17 +325,12 @@ module Reline line_editor.auto_indent_proc = auto_indent_proc line_editor.dig_perfect_match_proc = dig_perfect_match_proc pre_input_hook&.call - unless Reline::IOGate == Reline::GeneralIO + unless Reline::IOGate.dumb? @dialog_proc_list.each_pair do |name_sym, d| line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) end end - unless config.test_mode or config.loaded? - config.read - io_gate.set_default_key_bindings(config) - end - line_editor.print_nomultiline_prompt(prompt) line_editor.update_dialogs line_editor.rerender @@ -352,7 +340,15 @@ module Reline loop do read_io(config.keyseq_timeout) { |inputs| line_editor.set_pasting_state(io_gate.in_pasting?) - inputs.each { |key| line_editor.update(key) } + inputs.each do |key| + if key.char == :bracketed_paste_start + text = io_gate.read_bracketed_paste + line_editor.insert_pasted_text(text) + line_editor.scroll_into_view + else + line_editor.update(key) + end + end } if line_editor.finished? line_editor.render_finished @@ -371,92 +367,39 @@ module Reline end end - # GNU Readline waits for "keyseq-timeout" milliseconds to see if the ESC - # is followed by a character, and times out and treats it as a standalone - # ESC if the second character does not arrive. If the second character - # comes before timed out, it is treated as a modifier key with the - # meta-property of meta-key, so that it can be distinguished from - # multibyte characters with the 8th bit turned on. - # - # GNU Readline will wait for the 2nd character with "keyseq-timeout" - # milli-seconds but wait forever after 3rd characters. + # GNU Readline watis for "keyseq-timeout" milliseconds when the input is + # ambiguous whether it is matching or matched. + # If the next character does not arrive within the specified timeout, input + # is considered as matched. + # `ESC` is ambiguous because it can be a standalone ESC (matched) or part of + # `ESC char` or part of CSI sequence (matching). private def read_io(keyseq_timeout, &block) buffer = [] + status = KeyStroke::MATCHING loop do - c = io_gate.getc(Float::INFINITY) - if c == -1 - result = :unmatched - else - buffer << c - result = key_stroke.match_status(buffer) - end - case result - when :matched - expanded = key_stroke.expand(buffer).map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) - break - when :matching - if buffer.size == 1 - case read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - when :break then break - when :next then next - end - end - when :unmatched - if buffer.size == 1 and c == "\e".ord - read_escaped_key(keyseq_timeout, c, block) + timeout = status == KeyStroke::MATCHING_MATCHED ? keyseq_timeout.fdiv(1000) : Float::INFINITY + c = io_gate.getc(timeout) + if c.nil? || c == -1 + if status == KeyStroke::MATCHING_MATCHED + status = KeyStroke::MATCHED + elsif buffer.empty? + # io_gate is closed and reached EOF + block.call([Key.new(nil, nil, false)]) + return else - expanded = buffer.map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) + status = KeyStroke::UNMATCHED end - break + else + buffer << c + status = key_stroke.match_status(buffer) end - end - end - private def read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - succ_c = io_gate.getc(keyseq_timeout.fdiv(1000)) - if succ_c - case key_stroke.match_status(buffer.dup.push(succ_c)) - when :unmatched - if c == "\e".ord - block.([Reline::Key.new(succ_c, succ_c | 0b10000000, true)]) - else - block.([Reline::Key.new(c, c, false), Reline::Key.new(succ_c, succ_c, false)]) - end - return :break - when :matching - io_gate.ungetc(succ_c) - return :next - when :matched - buffer << succ_c - expanded = key_stroke.expand(buffer).map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) - return :break + if status == KeyStroke::MATCHED || status == KeyStroke::UNMATCHED + expanded, rest_bytes = key_stroke.expand(buffer) + rest_bytes.reverse_each { |c| io_gate.ungetc(c) } + block.call(expanded) + return end - else - block.([Reline::Key.new(c, c, false)]) - return :break - end - end - - private def read_escaped_key(keyseq_timeout, c, block) - escaped_c = io_gate.getc(keyseq_timeout.fdiv(1000)) - - if escaped_c.nil? - block.([Reline::Key.new(c, c, false)]) - elsif escaped_c >= 128 # maybe, first byte of multi byte - block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) - elsif escaped_c == "\e".ord # escape twice - block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) - else - block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) end end @@ -466,7 +409,7 @@ module Reline end private def may_req_ambiguous_char_width - @ambiguous_width = 2 if io_gate == Reline::GeneralIO or !STDOUT.tty? + @ambiguous_width = 2 if io_gate.dumb? || !STDIN.tty? || !STDOUT.tty? return if defined? @ambiguous_width io_gate.move_cursor_column(0) begin @@ -560,37 +503,13 @@ module Reline def self.line_editor core.line_editor end +end - def self.update_iogate - return if core.config.test_mode - # Need to change IOGate when `$stdout.tty?` change from false to true by `$stdout.reopen` - # Example: rails/spring boot the application in non-tty, then run console in tty. - if ENV['TERM'] != 'dumb' && core.io_gate == Reline::GeneralIO && $stdout.tty? - require 'reline/ansi' - remove_const(:IOGate) - const_set(:IOGate, Reline::ANSI) - end - end -end +Reline::IOGate = Reline::IO.decide_io_gate -require 'reline/general_io' -io = Reline::GeneralIO -unless ENV['TERM'] == 'dumb' - case RbConfig::CONFIG['host_os'] - when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ - require 'reline/windows' - tty = (io = Reline::Windows).msys_tty? - else - tty = $stdout.tty? - end -end -Reline::IOGate = if tty - require 'reline/ansi' - Reline::ANSI -else - io -end +# Deprecated +Reline::GeneralIO = Reline::Dumb.new Reline::Face.load_initial_configs diff --git a/lib/reline/config.rb b/lib/reline/config.rb index e06b4d16e5..774b06b6fd 100644 --- a/lib/reline/config.rb +++ b/lib/reline/config.rb @@ -29,18 +29,20 @@ class Reline::Config attr_accessor :autocompletion def initialize - @additional_key_bindings = {} # from inputrc - @additional_key_bindings[:emacs] = {} - @additional_key_bindings[:vi_insert] = {} - @additional_key_bindings[:vi_command] = {} - @oneshot_key_bindings = {} + @additional_key_bindings = { # from inputrc + emacs: Reline::KeyActor::Base.new, + vi_insert: Reline::KeyActor::Base.new, + vi_command: Reline::KeyActor::Base.new + } + @oneshot_key_bindings = Reline::KeyActor::Base.new @editing_mode_label = :emacs @keymap_label = :emacs @keymap_prefix = [] - @key_actors = {} - @key_actors[:emacs] = Reline::KeyActor::Emacs.new - @key_actors[:vi_insert] = Reline::KeyActor::ViInsert.new - @key_actors[:vi_command] = Reline::KeyActor::ViCommand.new + @default_key_bindings = { + emacs: Reline::KeyActor::Base.new(Reline::KeyActor::EMACS_MAPPING), + vi_insert: Reline::KeyActor::Base.new(Reline::KeyActor::VI_INSERT_MAPPING), + vi_command: Reline::KeyActor::Base.new(Reline::KeyActor::VI_COMMAND_MAPPING) + } @vi_cmd_mode_string = '(cmd)' @vi_ins_mode_string = '(ins)' @emacs_mode_string = '@' @@ -51,6 +53,7 @@ class Reline::Config @autocompletion = false @convert_meta = true if seven_bit_encoding?(Reline::IOGate.encoding) @loaded = false + @enable_bracketed_paste = true end def reset @@ -61,7 +64,7 @@ class Reline::Config end def editing_mode - @key_actors[@editing_mode_label] + @default_key_bindings[@editing_mode_label] end def editing_mode=(val) @@ -73,7 +76,7 @@ class Reline::Config end def keymap - @key_actors[@keymap_label] + @default_key_bindings[@keymap_label] end def loaded? @@ -132,14 +135,14 @@ class Reline::Config def key_bindings # The key bindings for each editing mode will be overwritten by the user-defined ones. - kb = @key_actors[@editing_mode_label].default_key_bindings.dup - kb.merge!(@additional_key_bindings[@editing_mode_label]) - kb.merge!(@oneshot_key_bindings) - kb + Reline::KeyActor::Composite.new([@oneshot_key_bindings, @additional_key_bindings[@editing_mode_label], @default_key_bindings[@editing_mode_label]]) end def add_oneshot_key_binding(keystroke, target) - @oneshot_key_bindings[keystroke] = target + # IRB sets invalid keystroke [Reline::Key]. We should ignore it. + return unless keystroke.all? { |c| c.is_a?(Integer) } + + @oneshot_key_bindings.add(keystroke, target) end def reset_oneshot_key_bindings @@ -147,11 +150,11 @@ class Reline::Config end def add_default_key_binding_by_keymap(keymap, keystroke, target) - @key_actors[keymap].default_key_bindings[keystroke] = target + @default_key_bindings[keymap].add(keystroke, target) end def add_default_key_binding(keystroke, target) - @key_actors[@keymap_label].default_key_bindings[keystroke] = target + add_default_key_binding_by_keymap(@keymap_label, keystroke, target) end def read_lines(lines, file = nil) @@ -181,16 +184,17 @@ class Reline::Config next if if_stack.any? { |_no, skip| skip } case line - when /^set +([^ ]+) +([^ ]+)/i - var, value = $1.downcase, $2 - bind_variable(var, value) + when /^set +([^ ]+) +(.+)/i + # value ignores everything after a space, raw_value does not. + var, value, raw_value = $1.downcase, $2.partition(' ').first, $2 + bind_variable(var, value, raw_value) next when /\s*("#{KEYSEQ_PATTERN}+")\s*:\s*(.*)\s*$/o key, func_name = $1, $2 func_name = func_name.split.first keystroke, func = bind_key(key, func_name) next unless keystroke - @additional_key_bindings[@keymap_label][@keymap_prefix + keystroke] = func + @additional_key_bindings[@keymap_label].add(@keymap_prefix + keystroke, func) end end unless if_stack.empty? @@ -233,7 +237,7 @@ class Reline::Config end end - def bind_variable(name, value) + def bind_variable(name, value, raw_value) case name when 'history-size' begin @@ -258,7 +262,7 @@ class Reline::Config when 'completion-query-items' @completion_query_items = value.to_i when 'isearch-terminators' - @isearch_terminators = retrieve_string(value) + @isearch_terminators = retrieve_string(raw_value) when 'editing-mode' case value when 'emacs' @@ -300,11 +304,11 @@ class Reline::Config @show_mode_in_prompt = false end when 'vi-cmd-mode-string' - @vi_cmd_mode_string = retrieve_string(value) + @vi_cmd_mode_string = retrieve_string(raw_value) when 'vi-ins-mode-string' - @vi_ins_mode_string = retrieve_string(value) + @vi_ins_mode_string = retrieve_string(raw_value) when 'emacs-mode-string' - @emacs_mode_string = retrieve_string(value) + @emacs_mode_string = retrieve_string(raw_value) when *VARIABLE_NAMES then variable_name = :"@#{name.tr(?-, ?_)}" instance_variable_set(variable_name, value.nil? || value == '1' || value == 'on') diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb deleted file mode 100644 index d52151ad3c..0000000000 --- a/lib/reline/general_io.rb +++ /dev/null @@ -1,111 +0,0 @@ -require 'io/wait' - -class Reline::GeneralIO - RESET_COLOR = '' # Do not send color reset sequence - - def self.reset(encoding: nil) - @@pasting = false - if encoding - @@encoding = encoding - elsif defined?(@@encoding) - remove_class_variable(:@@encoding) - end - end - - def self.encoding - if defined?(@@encoding) - @@encoding - elsif RUBY_PLATFORM =~ /mswin|mingw/ - Encoding::UTF_8 - else - Encoding::default_external - end - end - - def self.win? - false - end - - def self.set_default_key_bindings(_) - end - - @@buf = [] - @@input = STDIN - - def self.input=(val) - @@input = val - end - - def self.with_raw_input - yield - end - - def self.getc(_timeout_second) - unless @@buf.empty? - return @@buf.shift - end - c = nil - loop do - Reline.core.line_editor.handle_signal - result = @@input.wait_readable(0.1) - next if result.nil? - c = @@input.read(1) - break - end - c&.ord - end - - def self.ungetc(c) - @@buf.unshift(c) - end - - def self.get_screen_size - [24, 80] - end - - def self.cursor_pos - Reline::CursorPos.new(1, 1) - end - - def self.hide_cursor - end - - def self.show_cursor - end - - def self.move_cursor_column(val) - end - - def self.move_cursor_up(val) - end - - def self.move_cursor_down(val) - end - - def self.erase_after_cursor - end - - def self.scroll_down(val) - end - - def self.clear_screen - end - - def self.set_screen_size(rows, columns) - end - - def self.set_winch_handler(&handler) - end - - @@pasting = false - - def self.in_pasting? - @@pasting - end - - def self.prep - end - - def self.deprep(otio) - end -end diff --git a/lib/reline/io.rb b/lib/reline/io.rb new file mode 100644 index 0000000000..c1dd1a56c8 --- /dev/null +++ b/lib/reline/io.rb @@ -0,0 +1,41 @@ + +module Reline + class IO + RESET_COLOR = "\e[0m" + + def self.decide_io_gate + if ENV['TERM'] == 'dumb' + Reline::Dumb.new + else + require 'reline/io/ansi' + + case RbConfig::CONFIG['host_os'] + when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ + require 'reline/io/windows' + io = Reline::Windows.new + if io.msys_tty? + Reline::ANSI.new + else + io + end + else + Reline::ANSI.new + end + end + end + + def dumb? + false + end + + def win? + false + end + + def reset_color_sequence + self.class::RESET_COLOR + end + end +end + +require 'reline/io/dumb' diff --git a/lib/reline/ansi.rb b/lib/reline/io/ansi.rb index 5e1f7249e3..30a89bc471 100644 --- a/lib/reline/ansi.rb +++ b/lib/reline/io/ansi.rb @@ -1,10 +1,7 @@ require 'io/console' require 'io/wait' -require_relative 'terminfo' - -class Reline::ANSI - RESET_COLOR = "\e[0m" +class Reline::ANSI < Reline::IO CAPNAME_KEY_BINDINGS = { 'khome' => :ed_move_to_beg, 'kend' => :ed_move_to_end, @@ -36,15 +33,19 @@ class Reline::ANSI Reline::Terminfo.setupterm(0, 2) end - def self.encoding - Encoding.default_external + def initialize + @input = STDIN + @output = STDOUT + @buf = [] + @old_winch_handler = nil end - def self.win? - false + def encoding + Encoding.default_external end - def self.set_default_key_bindings(config, allow_terminfo: true) + def set_default_key_bindings(config, allow_terminfo: true) + set_bracketed_paste_key_bindings(config) set_default_key_bindings_ansi_cursor(config) if allow_terminfo && Reline::Terminfo.enabled? set_default_key_bindings_terminfo(config) @@ -66,7 +67,13 @@ class Reline::ANSI end end - def self.set_default_key_bindings_ansi_cursor(config) + def set_bracketed_paste_key_bindings(config) + [:emacs, :vi_insert, :vi_command].each do |keymap| + config.add_default_key_binding_by_keymap(keymap, START_BRACKETED_PASTE.bytes, :bracketed_paste_start) + end + end + + def set_default_key_bindings_ansi_cursor(config) ANSI_CURSOR_KEY_BINDINGS.each do |char, (default_func, modifiers)| bindings = [["\e[#{char}", default_func]] # CSI + char if modifiers[:ctrl] @@ -88,7 +95,7 @@ class Reline::ANSI end end - def self.set_default_key_bindings_terminfo(config) + def set_default_key_bindings_terminfo(config) key_bindings = CAPNAME_KEY_BINDINGS.map do |capname, key_binding| begin key_code = Reline::Terminfo.tigetstr(capname) @@ -105,12 +112,16 @@ class Reline::ANSI end end - def self.set_default_key_bindings_comprehensive_list(config) + def set_default_key_bindings_comprehensive_list(config) { + # xterm + [27, 91, 51, 126] => :key_delete, # kdch1 + [27, 91, 53, 126] => :ed_search_prev_history, # kpp + [27, 91, 54, 126] => :ed_search_next_history, # knp + # Console (80x25) [27, 91, 49, 126] => :ed_move_to_beg, # Home [27, 91, 52, 126] => :ed_move_to_end, # End - [27, 91, 51, 126] => :key_delete, # Del # KDE # Del is 0x08 @@ -140,132 +151,107 @@ class Reline::ANSI end end - @@input = STDIN - def self.input=(val) - @@input = val + def input=(val) + @input = val end - @@output = STDOUT - def self.output=(val) - @@output = val + def output=(val) + @output = val end - def self.with_raw_input - if @@input.tty? - @@input.raw(intr: true) { yield } + def with_raw_input + if @input.tty? + @input.raw(intr: true) { yield } else yield end end - @@buf = [] - def self.inner_getc(timeout_second) - unless @@buf.empty? - return @@buf.shift + def inner_getc(timeout_second) + unless @buf.empty? + return @buf.shift end - until @@input.wait_readable(0.01) + until @input.wait_readable(0.01) timeout_second -= 0.01 return nil if timeout_second <= 0 Reline.core.line_editor.handle_signal end - c = @@input.getbyte - (c == 0x16 && @@input.raw(min: 0, time: 0, &:getbyte)) || c + c = @input.getbyte + (c == 0x16 && @input.tty? && @input.raw(min: 0, time: 0, &:getbyte)) || c rescue Errno::EIO # Maybe the I/O has been closed. nil - rescue Errno::ENOTTY - nil end - @@in_bracketed_paste_mode = false - START_BRACKETED_PASTE = String.new("\e[200~,", encoding: Encoding::ASCII_8BIT) - END_BRACKETED_PASTE = String.new("\e[200~.", encoding: Encoding::ASCII_8BIT) - def self.getc_with_bracketed_paste(timeout_second) + START_BRACKETED_PASTE = String.new("\e[200~", encoding: Encoding::ASCII_8BIT) + END_BRACKETED_PASTE = String.new("\e[201~", encoding: Encoding::ASCII_8BIT) + def read_bracketed_paste buffer = String.new(encoding: Encoding::ASCII_8BIT) - buffer << inner_getc(timeout_second) - while START_BRACKETED_PASTE.start_with?(buffer) or END_BRACKETED_PASTE.start_with?(buffer) do - if START_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = true - return inner_getc(timeout_second) - elsif END_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = false - ungetc(-1) - return inner_getc(timeout_second) - end - succ_c = inner_getc(Reline.core.config.keyseq_timeout) - - if succ_c - buffer << succ_c - else - break - end + until buffer.end_with?(END_BRACKETED_PASTE) + c = inner_getc(Float::INFINITY) + break unless c + buffer << c end - buffer.bytes.reverse_each do |ch| - ungetc ch - end - inner_getc(timeout_second) + string = buffer.delete_suffix(END_BRACKETED_PASTE).force_encoding(encoding) + string.valid_encoding? ? string : '' end # if the usage expects to wait indefinitely, use Float::INFINITY for timeout_second - def self.getc(timeout_second) - if Reline.core.config.enable_bracketed_paste - getc_with_bracketed_paste(timeout_second) - else - inner_getc(timeout_second) - end + def getc(timeout_second) + inner_getc(timeout_second) end - def self.in_pasting? - @@in_bracketed_paste_mode or (not empty_buffer?) + def in_pasting? + not empty_buffer? end - def self.empty_buffer? - unless @@buf.empty? + def empty_buffer? + unless @buf.empty? return false end - !@@input.wait_readable(0) + !@input.wait_readable(0) end - def self.ungetc(c) - @@buf.unshift(c) + def ungetc(c) + @buf.unshift(c) end - def self.retrieve_keybuffer + def retrieve_keybuffer begin - return unless @@input.wait_readable(0.001) - str = @@input.read_nonblock(1024) + return unless @input.wait_readable(0.001) + str = @input.read_nonblock(1024) str.bytes.each do |c| - @@buf.push(c) + @buf.push(c) end rescue EOFError end end - def self.get_screen_size - s = @@input.winsize + def get_screen_size + s = @input.winsize return s if s[0] > 0 && s[1] > 0 s = [ENV["LINES"].to_i, ENV["COLUMNS"].to_i] return s if s[0] > 0 && s[1] > 0 [24, 80] - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV [24, 80] end - def self.set_screen_size(rows, columns) - @@input.winsize = [rows, columns] + def set_screen_size(rows, columns) + @input.winsize = [rows, columns] self - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV self end - def self.cursor_pos - begin + def cursor_pos + if both_tty? res = +'' m = nil - @@input.raw do |stdin| - @@output << "\e[6n" - @@output.flush + @input.raw do |stdin| + @output << "\e[6n" + @output.flush loop do c = stdin.getc next if c.nil? @@ -279,9 +265,9 @@ class Reline::ANSI end column = m[:column].to_i - 1 row = m[:row].to_i - 1 - rescue Errno::ENOTTY + else begin - buf = @@output.pread(@@output.pos, 0) + buf = @output.pread(@output.pos, 0) row = buf.count("\n") column = buf.rindex("\n") ? (buf.size - buf.rindex("\n")) - 1 : 0 rescue Errno::ESPIPE, IOError @@ -294,78 +280,85 @@ class Reline::ANSI Reline::CursorPos.new(column, row) end - def self.move_cursor_column(x) - @@output.write "\e[#{x + 1}G" + def both_tty? + @input.tty? && @output.tty? + end + + def move_cursor_column(x) + @output.write "\e[#{x + 1}G" end - def self.move_cursor_up(x) + def move_cursor_up(x) if x > 0 - @@output.write "\e[#{x}A" + @output.write "\e[#{x}A" elsif x < 0 move_cursor_down(-x) end end - def self.move_cursor_down(x) + def move_cursor_down(x) if x > 0 - @@output.write "\e[#{x}B" + @output.write "\e[#{x}B" elsif x < 0 move_cursor_up(-x) end end - def self.hide_cursor + def hide_cursor + seq = "\e[?25l" if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin - @@output.write Reline::Terminfo.tigetstr('civis') + seq = Reline::Terminfo.tigetstr('civis') rescue Reline::Terminfo::TerminfoError # civis is undefined end - else - # ignored end + @output.write seq end - def self.show_cursor + def show_cursor + seq = "\e[?25h" if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin - @@output.write Reline::Terminfo.tigetstr('cnorm') + seq = Reline::Terminfo.tigetstr('cnorm') rescue Reline::Terminfo::TerminfoError # cnorm is undefined end - else - # ignored end + @output.write seq end - def self.erase_after_cursor - @@output.write "\e[K" + def erase_after_cursor + @output.write "\e[K" end # This only works when the cursor is at the bottom of the scroll range # For more details, see https://github.com/ruby/reline/pull/577#issuecomment-1646679623 - def self.scroll_down(x) + def scroll_down(x) return if x.zero? # We use `\n` instead of CSI + S because CSI + S would cause https://github.com/ruby/reline/issues/576 - @@output.write "\n" * x + @output.write "\n" * x end - def self.clear_screen - @@output.write "\e[2J" - @@output.write "\e[1;1H" + def clear_screen + @output.write "\e[2J" + @output.write "\e[1;1H" end - @@old_winch_handler = nil - def self.set_winch_handler(&handler) - @@old_winch_handler = Signal.trap('WINCH', &handler) + def set_winch_handler(&handler) + @old_winch_handler = Signal.trap('WINCH', &handler) end - def self.prep + def prep + # Enable bracketed paste + @output.write "\e[?2004h" if Reline.core.config.enable_bracketed_paste && both_tty? retrieve_keybuffer nil end - def self.deprep(otio) - Signal.trap('WINCH', @@old_winch_handler) if @@old_winch_handler + def deprep(otio) + # Disable bracketed paste + @output.write "\e[?2004l" if Reline.core.config.enable_bracketed_paste && both_tty? + Signal.trap('WINCH', @old_winch_handler) if @old_winch_handler end end diff --git a/lib/reline/io/dumb.rb b/lib/reline/io/dumb.rb new file mode 100644 index 0000000000..6ed69ffdfa --- /dev/null +++ b/lib/reline/io/dumb.rb @@ -0,0 +1,106 @@ +require 'io/wait' + +class Reline::Dumb < Reline::IO + RESET_COLOR = '' # Do not send color reset sequence + + def initialize(encoding: nil) + @input = STDIN + @buf = [] + @pasting = false + @encoding = encoding + @screen_size = [24, 80] + end + + def dumb? + true + end + + def encoding + if @encoding + @encoding + elsif RUBY_PLATFORM =~ /mswin|mingw/ + Encoding::UTF_8 + else + Encoding::default_external + end + end + + def set_default_key_bindings(_) + end + + def input=(val) + @input = val + end + + def with_raw_input + yield + end + + def getc(_timeout_second) + unless @buf.empty? + return @buf.shift + end + c = nil + loop do + Reline.core.line_editor.handle_signal + result = @input.wait_readable(0.1) + next if result.nil? + c = @input.read(1) + break + end + c&.ord + end + + def ungetc(c) + @buf.unshift(c) + end + + def get_screen_size + @screen_size + end + + def cursor_pos + Reline::CursorPos.new(1, 1) + end + + def hide_cursor + end + + def show_cursor + end + + def move_cursor_column(val) + end + + def move_cursor_up(val) + end + + def move_cursor_down(val) + end + + def erase_after_cursor + end + + def scroll_down(val) + end + + def clear_screen + end + + def set_screen_size(rows, columns) + @screen_size = [rows, columns] + end + + def set_winch_handler(&handler) + end + + def in_pasting? + @pasting + end + + def prep + end + + def deprep(otio) + end +end diff --git a/lib/reline/windows.rb b/lib/reline/io/windows.rb index ee3f73e383..6ba4b830d6 100644 --- a/lib/reline/windows.rb +++ b/lib/reline/io/windows.rb @@ -1,21 +1,49 @@ require 'fiddle/import' -class Reline::Windows - RESET_COLOR = "\e[0m" +class Reline::Windows < Reline::IO + def initialize + @input_buf = [] + @output_buf = [] + + @output = STDOUT + @hsg = nil + @getwch = Win32API.new('msvcrt', '_getwch', [], 'I') + @kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I') + @GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L') + @GetConsoleScreenBufferInfo = Win32API.new('kernel32', 'GetConsoleScreenBufferInfo', ['L', 'P'], 'L') + @SetConsoleCursorPosition = Win32API.new('kernel32', 'SetConsoleCursorPosition', ['L', 'L'], 'L') + @GetStdHandle = Win32API.new('kernel32', 'GetStdHandle', ['L'], 'L') + @FillConsoleOutputCharacter = Win32API.new('kernel32', 'FillConsoleOutputCharacter', ['L', 'L', 'L', 'L', 'P'], 'L') + @ScrollConsoleScreenBuffer = Win32API.new('kernel32', 'ScrollConsoleScreenBuffer', ['L', 'P', 'P', 'L', 'P'], 'L') + @hConsoleHandle = @GetStdHandle.call(STD_OUTPUT_HANDLE) + @hConsoleInputHandle = @GetStdHandle.call(STD_INPUT_HANDLE) + @GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L') + @ReadConsoleInputW = Win32API.new('kernel32', 'ReadConsoleInputW', ['L', 'P', 'L', 'P'], 'L') + @GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L') + @GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I') + @FillConsoleOutputAttribute = Win32API.new('kernel32', 'FillConsoleOutputAttribute', ['L', 'L', 'L', 'L', 'P'], 'L') + @SetConsoleCursorInfo = Win32API.new('kernel32', 'SetConsoleCursorInfo', ['L', 'P'], 'L') + + @GetConsoleMode = Win32API.new('kernel32', 'GetConsoleMode', ['L', 'P'], 'L') + @SetConsoleMode = Win32API.new('kernel32', 'SetConsoleMode', ['L', 'L'], 'L') + @WaitForSingleObject = Win32API.new('kernel32', 'WaitForSingleObject', ['L', 'L'], 'L') + + @legacy_console = getconsolemode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 + end - def self.encoding + def encoding Encoding::UTF_8 end - def self.win? + def win? true end - def self.win_legacy_console? - @@legacy_console + def win_legacy_console? + @legacy_console end - def self.set_default_key_bindings(config) + def set_default_key_bindings(config) { [224, 72] => :ed_prev_history, # ↑ [224, 80] => :ed_next_history, # ↓ @@ -129,58 +157,32 @@ class Reline::Windows STD_OUTPUT_HANDLE = -11 FILE_TYPE_PIPE = 0x0003 FILE_NAME_INFO = 2 - @@getwch = Win32API.new('msvcrt', '_getwch', [], 'I') - @@kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I') - @@GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L') - @@GetConsoleScreenBufferInfo = Win32API.new('kernel32', 'GetConsoleScreenBufferInfo', ['L', 'P'], 'L') - @@SetConsoleCursorPosition = Win32API.new('kernel32', 'SetConsoleCursorPosition', ['L', 'L'], 'L') - @@GetStdHandle = Win32API.new('kernel32', 'GetStdHandle', ['L'], 'L') - @@FillConsoleOutputCharacter = Win32API.new('kernel32', 'FillConsoleOutputCharacter', ['L', 'L', 'L', 'L', 'P'], 'L') - @@ScrollConsoleScreenBuffer = Win32API.new('kernel32', 'ScrollConsoleScreenBuffer', ['L', 'P', 'P', 'L', 'P'], 'L') - @@hConsoleHandle = @@GetStdHandle.call(STD_OUTPUT_HANDLE) - @@hConsoleInputHandle = @@GetStdHandle.call(STD_INPUT_HANDLE) - @@GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L') - @@ReadConsoleInputW = Win32API.new('kernel32', 'ReadConsoleInputW', ['L', 'P', 'L', 'P'], 'L') - @@GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L') - @@GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I') - @@FillConsoleOutputAttribute = Win32API.new('kernel32', 'FillConsoleOutputAttribute', ['L', 'L', 'L', 'L', 'P'], 'L') - @@SetConsoleCursorInfo = Win32API.new('kernel32', 'SetConsoleCursorInfo', ['L', 'P'], 'L') - - @@GetConsoleMode = Win32API.new('kernel32', 'GetConsoleMode', ['L', 'P'], 'L') - @@SetConsoleMode = Win32API.new('kernel32', 'SetConsoleMode', ['L', 'L'], 'L') - @@WaitForSingleObject = Win32API.new('kernel32', 'WaitForSingleObject', ['L', 'L'], 'L') ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 - private_class_method def self.getconsolemode + private def getconsolemode mode = "\000\000\000\000" - @@GetConsoleMode.call(@@hConsoleHandle, mode) + @GetConsoleMode.call(@hConsoleHandle, mode) mode.unpack1('L') end - private_class_method def self.setconsolemode(mode) - @@SetConsoleMode.call(@@hConsoleHandle, mode) + private def setconsolemode(mode) + @SetConsoleMode.call(@hConsoleHandle, mode) end - @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) - #if @@legacy_console + #if @legacy_console # setconsolemode(getconsolemode() | ENABLE_VIRTUAL_TERMINAL_PROCESSING) - # @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) + # @legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) #end - @@input_buf = [] - @@output_buf = [] - - @@output = STDOUT - - def self.msys_tty?(io = @@hConsoleInputHandle) + def msys_tty?(io = @hConsoleInputHandle) # check if fd is a pipe - if @@GetFileType.call(io) != FILE_TYPE_PIPE + if @GetFileType.call(io) != FILE_TYPE_PIPE return false end bufsize = 1024 p_buffer = "\0" * bufsize - res = @@GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2) + res = @GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2) return false if res == 0 # get pipe name: p_buffer layout is: @@ -217,65 +219,63 @@ class Reline::Windows [ { control_keys: :SHIFT, virtual_key_code: VK_TAB }, [27, 91, 90] ], ] - @@hsg = nil - - def self.process_key_event(repeat_count, virtual_key_code, virtual_scan_code, char_code, control_key_state) + def process_key_event(repeat_count, virtual_key_code, virtual_scan_code, char_code, control_key_state) # high-surrogate if 0xD800 <= char_code and char_code <= 0xDBFF - @@hsg = char_code + @hsg = char_code return end # low-surrogate if 0xDC00 <= char_code and char_code <= 0xDFFF - if @@hsg - char_code = 0x10000 + (@@hsg - 0xD800) * 0x400 + char_code - 0xDC00 - @@hsg = nil + if @hsg + char_code = 0x10000 + (@hsg - 0xD800) * 0x400 + char_code - 0xDC00 + @hsg = nil else # no high-surrogate. ignored. return end else # ignore high-surrogate without low-surrogate if there - @@hsg = nil + @hsg = nil end key = KeyEventRecord.new(virtual_key_code, char_code, control_key_state) match = KEY_MAP.find { |args,| key.matches?(**args) } unless match.nil? - @@output_buf.concat(match.last) + @output_buf.concat(match.last) return end # no char, only control keys return if key.char_code == 0 and key.control_keys.any? - @@output_buf.push("\e".ord) if key.control_keys.include?(:ALT) and !key.control_keys.include?(:CTRL) + @output_buf.push("\e".ord) if key.control_keys.include?(:ALT) and !key.control_keys.include?(:CTRL) - @@output_buf.concat(key.char.bytes) + @output_buf.concat(key.char.bytes) end - def self.check_input_event + def check_input_event num_of_events = 0.chr * 8 - while @@output_buf.empty? + while @output_buf.empty? Reline.core.line_editor.handle_signal - if @@WaitForSingleObject.(@@hConsoleInputHandle, 100) != 0 # max 0.1 sec + if @WaitForSingleObject.(@hConsoleInputHandle, 100) != 0 # max 0.1 sec # prevent for background consolemode change - @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) + @legacy_console = getconsolemode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 next end - next if @@GetNumberOfConsoleInputEvents.(@@hConsoleInputHandle, num_of_events) == 0 or num_of_events.unpack1('L') == 0 + next if @GetNumberOfConsoleInputEvents.(@hConsoleInputHandle, num_of_events) == 0 or num_of_events.unpack1('L') == 0 input_records = 0.chr * 20 * 80 read_event = 0.chr * 4 - if @@ReadConsoleInputW.(@@hConsoleInputHandle, input_records, 80, read_event) != 0 + if @ReadConsoleInputW.(@hConsoleInputHandle, input_records, 80, read_event) != 0 read_events = read_event.unpack1('L') 0.upto(read_events) do |idx| input_record = input_records[idx * 20, 20] event = input_record[0, 2].unpack1('s*') case event when WINDOW_BUFFER_SIZE_EVENT - @@winch_handler.() + @winch_handler.() when KEY_EVENT key_down = input_record[4, 4].unpack1('l*') repeat_count = input_record[8, 2].unpack1('s*') @@ -293,34 +293,34 @@ class Reline::Windows end end - def self.with_raw_input + def with_raw_input yield end - def self.getc(_timeout_second) + def getc(_timeout_second) check_input_event - @@output_buf.shift + @output_buf.shift end - def self.ungetc(c) - @@output_buf.unshift(c) + def ungetc(c) + @output_buf.unshift(c) end - def self.in_pasting? - not self.empty_buffer? + def in_pasting? + not empty_buffer? end - def self.empty_buffer? - if not @@output_buf.empty? + def empty_buffer? + if not @output_buf.empty? false - elsif @@kbhit.call == 0 + elsif @kbhit.call == 0 true else false end end - def self.get_console_screen_buffer_info + def get_console_screen_buffer_info # CONSOLE_SCREEN_BUFFER_INFO # [ 0,2] dwSize.X # [ 2,2] dwSize.Y @@ -334,18 +334,18 @@ class Reline::Windows # [18,2] dwMaximumWindowSize.X # [20,2] dwMaximumWindowSize.Y csbi = 0.chr * 22 - return if @@GetConsoleScreenBufferInfo.call(@@hConsoleHandle, csbi) == 0 + return if @GetConsoleScreenBufferInfo.call(@hConsoleHandle, csbi) == 0 csbi end - def self.get_screen_size + def get_screen_size unless csbi = get_console_screen_buffer_info return [1, 1] end csbi[0, 4].unpack('SS').reverse end - def self.cursor_pos + def cursor_pos unless csbi = get_console_screen_buffer_info return Reline::CursorPos.new(0, 0) end @@ -354,49 +354,49 @@ class Reline::Windows Reline::CursorPos.new(x, y) end - def self.move_cursor_column(val) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, cursor_pos.y * 65536 + val) + def move_cursor_column(val) + @SetConsoleCursorPosition.call(@hConsoleHandle, cursor_pos.y * 65536 + val) end - def self.move_cursor_up(val) + def move_cursor_up(val) if val > 0 y = cursor_pos.y - val y = 0 if y < 0 - @@SetConsoleCursorPosition.call(@@hConsoleHandle, y * 65536 + cursor_pos.x) + @SetConsoleCursorPosition.call(@hConsoleHandle, y * 65536 + cursor_pos.x) elsif val < 0 move_cursor_down(-val) end end - def self.move_cursor_down(val) + def move_cursor_down(val) if val > 0 return unless csbi = get_console_screen_buffer_info screen_height = get_screen_size.first y = cursor_pos.y + val y = screen_height - 1 if y > (screen_height - 1) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, (cursor_pos.y + val) * 65536 + cursor_pos.x) + @SetConsoleCursorPosition.call(@hConsoleHandle, (cursor_pos.y + val) * 65536 + cursor_pos.x) elsif val < 0 move_cursor_up(-val) end end - def self.erase_after_cursor + def erase_after_cursor return unless csbi = get_console_screen_buffer_info attributes = csbi[8, 2].unpack1('S') cursor = csbi[4, 4].unpack1('L') written = 0.chr * 4 - @@FillConsoleOutputCharacter.call(@@hConsoleHandle, 0x20, get_screen_size.last - cursor_pos.x, cursor, written) - @@FillConsoleOutputAttribute.call(@@hConsoleHandle, attributes, get_screen_size.last - cursor_pos.x, cursor, written) + @FillConsoleOutputCharacter.call(@hConsoleHandle, 0x20, get_screen_size.last - cursor_pos.x, cursor, written) + @FillConsoleOutputAttribute.call(@hConsoleHandle, attributes, get_screen_size.last - cursor_pos.x, cursor, written) end - def self.scroll_down(val) + def scroll_down(val) return if val < 0 return unless csbi = get_console_screen_buffer_info buffer_width, buffer_lines, x, y, attributes, window_left, window_top, window_bottom = csbi.unpack('ssssSssx2s') screen_height = window_bottom - window_top + 1 val = screen_height if val > screen_height - if @@legacy_console || window_left != 0 + if @legacy_console || window_left != 0 # unless ENABLE_VIRTUAL_TERMINAL, # if srWindow.Left != 0 then it's conhost.exe hosted console # and puts "\n" causes horizontal scroll. its glitch. @@ -404,11 +404,11 @@ class Reline::Windows scroll_rectangle = [0, val, buffer_width, buffer_lines - val].pack('s4') destination_origin = 0 # y * 65536 + x fill = [' '.ord, attributes].pack('SS') - @@ScrollConsoleScreenBuffer.call(@@hConsoleHandle, scroll_rectangle, nil, destination_origin, fill) + @ScrollConsoleScreenBuffer.call(@hConsoleHandle, scroll_rectangle, nil, destination_origin, fill) else origin_x = x + 1 origin_y = y - window_top + 1 - @@output.write [ + @output.write [ (origin_y != screen_height) ? "\e[#{screen_height};H" : nil, "\n" * val, (origin_y != screen_height or !x.zero?) ? "\e[#{origin_y};#{origin_x}H" : nil @@ -416,49 +416,49 @@ class Reline::Windows end end - def self.clear_screen - if @@legacy_console + def clear_screen + if @legacy_console return unless csbi = get_console_screen_buffer_info buffer_width, _buffer_lines, attributes, window_top, window_bottom = csbi.unpack('ss@8S@12sx2s') fill_length = buffer_width * (window_bottom - window_top + 1) screen_topleft = window_top * 65536 written = 0.chr * 4 - @@FillConsoleOutputCharacter.call(@@hConsoleHandle, 0x20, fill_length, screen_topleft, written) - @@FillConsoleOutputAttribute.call(@@hConsoleHandle, attributes, fill_length, screen_topleft, written) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, screen_topleft) + @FillConsoleOutputCharacter.call(@hConsoleHandle, 0x20, fill_length, screen_topleft, written) + @FillConsoleOutputAttribute.call(@hConsoleHandle, attributes, fill_length, screen_topleft, written) + @SetConsoleCursorPosition.call(@hConsoleHandle, screen_topleft) else - @@output.write "\e[2J" "\e[H" + @output.write "\e[2J" "\e[H" end end - def self.set_screen_size(rows, columns) + def set_screen_size(rows, columns) raise NotImplementedError end - def self.hide_cursor + def hide_cursor size = 100 visible = 0 # 0 means false cursor_info = [size, visible].pack('Li') - @@SetConsoleCursorInfo.call(@@hConsoleHandle, cursor_info) + @SetConsoleCursorInfo.call(@hConsoleHandle, cursor_info) end - def self.show_cursor + def show_cursor size = 100 visible = 1 # 1 means true cursor_info = [size, visible].pack('Li') - @@SetConsoleCursorInfo.call(@@hConsoleHandle, cursor_info) + @SetConsoleCursorInfo.call(@hConsoleHandle, cursor_info) end - def self.set_winch_handler(&handler) - @@winch_handler = handler + def set_winch_handler(&handler) + @winch_handler = handler end - def self.prep + def prep # do nothing nil end - def self.deprep(otio) + def deprep(otio) # do nothing end diff --git a/lib/reline/key_actor.rb b/lib/reline/key_actor.rb index ebe09d2009..0ac7604556 100644 --- a/lib/reline/key_actor.rb +++ b/lib/reline/key_actor.rb @@ -2,6 +2,7 @@ module Reline::KeyActor end require 'reline/key_actor/base' +require 'reline/key_actor/composite' require 'reline/key_actor/emacs' require 'reline/key_actor/vi_command' require 'reline/key_actor/vi_insert' diff --git a/lib/reline/key_actor/base.rb b/lib/reline/key_actor/base.rb index 194e98938c..ee28c7681e 100644 --- a/lib/reline/key_actor/base.rb +++ b/lib/reline/key_actor/base.rb @@ -1,15 +1,31 @@ class Reline::KeyActor::Base - MAPPING = Array.new(256) + def initialize(mapping = []) + @mapping = mapping + @matching_bytes = {} + @key_bindings = {} + end def get_method(key) - self.class::MAPPING[key] + @mapping[key] + end + + def add(key, func) + (1...key.size).each do |size| + @matching_bytes[key.take(size)] = true + end + @key_bindings[key] = func + end + + def matching?(key) + @matching_bytes[key] end - def initialize - @default_key_bindings = {} + def get(key) + @key_bindings[key] end - def default_key_bindings - @default_key_bindings + def clear + @matching_bytes.clear + @key_bindings.clear end end diff --git a/lib/reline/key_actor/composite.rb b/lib/reline/key_actor/composite.rb new file mode 100644 index 0000000000..37e94ce6cf --- /dev/null +++ b/lib/reline/key_actor/composite.rb @@ -0,0 +1,17 @@ +class Reline::KeyActor::Composite + def initialize(key_actors) + @key_actors = key_actors + end + + def matching?(key) + @key_actors.any? { |key_actor| key_actor.matching?(key) } + end + + def get(key) + @key_actors.each do |key_actor| + func = key_actor.get(key) + return func if func + end + nil + end +end diff --git a/lib/reline/key_actor/emacs.rb b/lib/reline/key_actor/emacs.rb index 9c797ba43e..ad84ee1d99 100644 --- a/lib/reline/key_actor/emacs.rb +++ b/lib/reline/key_actor/emacs.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::Emacs < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + EMACS_MAPPING = [ # 0 ^@ :em_set_mark, # 1 ^A @@ -63,7 +63,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 30 ^^ :ed_unassigned, # 31 ^_ - :ed_unassigned, + :undo, # 32 SPACE :ed_insert, # 33 ! @@ -319,7 +319,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 158 M-^^ :ed_unassigned, # 159 M-^_ - :ed_unassigned, + :redo, # 160 M-SPACE :em_set_mark, # 161 M-! diff --git a/lib/reline/key_actor/vi_command.rb b/lib/reline/key_actor/vi_command.rb index 06bb0ba8e4..d972c5e67f 100644 --- a/lib/reline/key_actor/vi_command.rb +++ b/lib/reline/key_actor/vi_command.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::ViCommand < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + VI_COMMAND_MAPPING = [ # 0 ^@ :ed_unassigned, # 1 ^A diff --git a/lib/reline/key_actor/vi_insert.rb b/lib/reline/key_actor/vi_insert.rb index f8ccf468c6..312df1646b 100644 --- a/lib/reline/key_actor/vi_insert.rb +++ b/lib/reline/key_actor/vi_insert.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::ViInsert < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + VI_INSERT_MAPPING = [ # 0 ^@ :ed_unassigned, # 1 ^A diff --git a/lib/reline/key_stroke.rb b/lib/reline/key_stroke.rb index bceffbb53f..ba40899685 100644 --- a/lib/reline/key_stroke.rb +++ b/lib/reline/key_stroke.rb @@ -7,138 +7,99 @@ class Reline::KeyStroke @config = config end - def compress_meta_key(ary) - return ary unless @config.convert_meta - ary.inject([]) { |result, key| - if result.size > 0 and result.last == "\e".ord - result[result.size - 1] = Reline::Key.new(key, key | 0b10000000, true) - else - result << key - end - result - } - end + # Input exactly matches to a key sequence + MATCHING = :matching + # Input partially matches to a key sequence + MATCHED = :matched + # Input matches to a key sequence and the key sequence is a prefix of another key sequence + MATCHING_MATCHED = :matching_matched + # Input does not match to any key sequence + UNMATCHED = :unmatched - def start_with?(me, other) - compressed_me = compress_meta_key(me) - compressed_other = compress_meta_key(other) - i = 0 - loop do - my_c = compressed_me[i] - other_c = compressed_other[i] - other_is_last = (i + 1) == compressed_other.size - me_is_last = (i + 1) == compressed_me.size - if my_c != other_c - if other_c == "\e".ord and other_is_last and my_c.is_a?(Reline::Key) and my_c.with_meta - return true - else - return false - end - elsif other_is_last - return true - elsif me_is_last - return false - end - i += 1 - end - end + def match_status(input) + matching = key_mapping.matching?(input) + matched = key_mapping.get(input) - def equal?(me, other) - case me - when Array - compressed_me = compress_meta_key(me) - compressed_other = compress_meta_key(other) - compressed_me.size == compressed_other.size and [compressed_me, compressed_other].transpose.all?{ |i| equal?(i[0], i[1]) } - when Integer - if other.is_a?(Reline::Key) - if other.combined_char == "\e".ord - false - else - other.combined_char == me - end - else - me == other - end - when Reline::Key - if other.is_a?(Integer) - me.combined_char == other - else - me == other - end - end - end + # FIXME: Workaround for single byte. remove this after MAPPING is merged into KeyActor. + matched ||= input.size == 1 + matching ||= input == [ESC_BYTE] - def match_status(input) - key_mapping.keys.select { |lhs| - start_with?(lhs, input) - }.tap { |it| - return :matched if it.size == 1 && equal?(it[0], input) - return :matching if it.size == 1 && !equal?(it[0], input) - return :matched if it.max_by(&:size)&.size&.< input.size - return :matching if it.size > 1 - } - if key_mapping.keys.any? { |lhs| start_with?(input, lhs) } - :matched + if matching && matched + MATCHING_MATCHED + elsif matching + MATCHING + elsif matched + MATCHED + elsif input[0] == ESC_BYTE + match_unknown_escape_sequence(input, vi_mode: @config.editing_mode_is?(:vi_insert, :vi_command)) + elsif input.size == 1 + MATCHED else - match_unknown_escape_sequence(input).first + UNMATCHED end end def expand(input) - lhs = key_mapping.keys.select { |item| start_with?(input, item) }.sort_by(&:size).last - unless lhs - status, size = match_unknown_escape_sequence(input) - case status - when :matched - return [:ed_unassigned] + expand(input.drop(size)) - when :matching - return [:ed_unassigned] - else - return input - end + matched_bytes = nil + (1..input.size).each do |i| + bytes = input.take(i) + status = match_status(bytes) + matched_bytes = bytes if status == MATCHED || status == MATCHING_MATCHED end - rhs = key_mapping[lhs] + return [[], []] unless matched_bytes - case rhs - when String - rhs_bytes = rhs.bytes - expand(expand(rhs_bytes) + expand(input.drop(lhs.size))) - when Symbol - [rhs] + expand(input.drop(lhs.size)) - when Array - rhs + func = key_mapping.get(matched_bytes) + if func.is_a?(Array) + keys = func.map { |c| Reline::Key.new(c, c, false) } + elsif func + keys = [Reline::Key.new(func, func, false)] + elsif matched_bytes.size == 1 + keys = [Reline::Key.new(matched_bytes.first, matched_bytes.first, false)] + elsif matched_bytes.size == 2 && matched_bytes[0] == ESC_BYTE + keys = [Reline::Key.new(matched_bytes[1], matched_bytes[1] | 0b10000000, true)] + else + keys = [] end + + [keys, input.drop(matched_bytes.size)] end private # returns match status of CSI/SS3 sequence and matched length - def match_unknown_escape_sequence(input) + def match_unknown_escape_sequence(input, vi_mode: false) idx = 0 - return [:unmatched, nil] unless input[idx] == ESC_BYTE + return UNMATCHED unless input[idx] == ESC_BYTE idx += 1 idx += 1 if input[idx] == ESC_BYTE case input[idx] when nil - return [:matching, nil] + if idx == 1 # `ESC` + return MATCHING_MATCHED + else # `ESC ESC` + return MATCHING + end when 91 # == '['.ord - # CSI sequence + # CSI sequence `ESC [ ... char` idx += 1 idx += 1 while idx < input.size && CSI_PARAMETER_BYTES_RANGE.cover?(input[idx]) idx += 1 while idx < input.size && CSI_INTERMEDIATE_BYTES_RANGE.cover?(input[idx]) - input[idx] ? [:matched, idx + 1] : [:matching, nil] when 79 # == 'O'.ord - # SS3 sequence - input[idx + 1] ? [:matched, idx + 2] : [:matching, nil] + # SS3 sequence `ESC O char` + idx += 1 else - if idx == 1 - # `ESC char`, make it :unmatched so that it will be handled correctly in `read_2nd_character_of_key_sequence` - [:unmatched, nil] - else - # `ESC ESC char` - [:matched, idx + 1] - end + # `ESC char` or `ESC ESC char` + return UNMATCHED if vi_mode + end + + case input.size + when idx + MATCHING + when idx + 1 + MATCHED + else + UNMATCHED end end diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index 57a3ab2f97..d659378acd 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -4,7 +4,6 @@ require 'reline/unicode' require 'tempfile' class Reline::LineEditor - # TODO: undo # TODO: Use "private alias_method" idiom after drop Ruby 2.5. attr_reader :byte_pointer attr_accessor :confirm_multiline_termination_proc @@ -46,6 +45,7 @@ class Reline::LineEditor RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) CompletionJourneyState = Struct.new(:line_index, :pre, :target, :post, :list, :pointer) + NullActionState = [nil, nil].freeze class MenuInfo attr_reader :list @@ -75,7 +75,7 @@ class Reline::LineEditor def initialize(config, encoding) @config = config @completion_append_character = '' - @screen_size = Reline::IOGate.get_screen_size + @screen_size = [0, 0] # Should be initialized with actual winsize in LineEditor#reset reset_variables(encoding: encoding) end @@ -238,7 +238,6 @@ class Reline::LineEditor @perfect_matched = nil @menu_info = nil @searching_prompt = nil - @first_char = true @just_cursor_moving = false @eof = false @continuous_insertion_buffer = String.new(encoding: @encoding) @@ -251,6 +250,11 @@ class Reline::LineEditor @resized = false @cache = {} @rendered_screen = RenderedScreen.new(base_y: 0, lines: [], cursor_y: 0) + @input_lines = [[[""], 0, 0]] + @input_lines_position = 0 + @undoing = false + @prev_action_state = NullActionState + @next_action_state = NullActionState reset_line end @@ -283,7 +287,7 @@ class Reline::LineEditor indent1 = @auto_indent_proc.(@buffer_of_lines.take(@line_index - 1).push(''), @line_index - 1, 0, true) indent2 = @auto_indent_proc.(@buffer_of_lines.take(@line_index), @line_index - 1, @buffer_of_lines[@line_index - 1].bytesize, false) indent = indent2 || indent1 - @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A */, '') + @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A\s*/, '') ) process_auto_indent @line_index, add_newline: true else @@ -410,7 +414,7 @@ class Reline::LineEditor # do nothing elsif level == :blank Reline::IOGate.move_cursor_column base_x - @output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}" + @output.write "#{Reline::IOGate.reset_color_sequence}#{' ' * width}" else x, w, content = new_items[level] cover_begin = base_x != 0 && new_levels[base_x - 1] == level @@ -420,7 +424,7 @@ class Reline::LineEditor content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true) end Reline::IOGate.move_cursor_column x + pos - @output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}" + @output.write "#{Reline::IOGate.reset_color_sequence}#{content}#{Reline::IOGate.reset_color_sequence}" end base_x += width end @@ -682,10 +686,8 @@ class Reline::LineEditor @trap_key.each do |t| @config.add_oneshot_key_binding(t, @name) end - elsif @trap_key.is_a?(Array) + else @config.add_oneshot_key_binding(@trap_key, @name) - elsif @trap_key.is_a?(Integer) or @trap_key.is_a?(Reline::Key) - @config.add_oneshot_key_binding([@trap_key], @name) end end dialog_render_info @@ -948,7 +950,8 @@ class Reline::LineEditor unless @waiting_proc byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end else @@ -1009,7 +1012,8 @@ class Reline::LineEditor if @vi_waiting_operator byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end @kill_ring.process @@ -1076,17 +1080,7 @@ class Reline::LineEditor else # single byte return if key.char >= 128 # maybe, first byte of multi byte method_symbol = @config.editing_mode.get_method(key.combined_char) - if key.with_meta and method_symbol == :ed_unassigned - if @config.editing_mode_is?(:vi_command, :vi_insert) - # split ESC + key in vi mode - method_symbol = @config.editing_mode.get_method("\e".ord) - process_key("\e".ord, method_symbol) - method_symbol = @config.editing_mode.get_method(key.char) - process_key(key.char, method_symbol) - end - else - process_key(key.combined_char, method_symbol) - end + process_key(key.combined_char, method_symbol) @multibyte_buffer.clear end if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize @@ -1106,6 +1100,7 @@ class Reline::LineEditor end def input_key(key) + save_old_buffer @config.reset_oneshot_key_bindings @dialogs.each do |dialog| if key.char.instance_of?(Symbol) and key.char == dialog.name @@ -1114,14 +1109,10 @@ class Reline::LineEditor end if key.char.nil? process_insert(force: true) - if @first_char - @eof = true - end + @eof = buffer_empty? finish return end - old_lines = @buffer_of_lines.dup - @first_char = false @completion_occurs = false if key.char.is_a?(Symbol) @@ -1129,17 +1120,23 @@ class Reline::LineEditor else normal_char(key) end + + @prev_action_state, @next_action_state = @next_action_state, NullActionState + unless @completion_occurs @completion_state = CompletionState::NORMAL @completion_journey_state = nil end + push_input_lines unless @undoing + @undoing = false + if @in_pasting clear_dialogs return end - modified = old_lines != @buffer_of_lines + modified = @old_buffer_of_lines != @buffer_of_lines if !@completion_occurs && modified && !@config.disable_completion && @config.autocompletion # Auto complete starts only when edited process_insert(force: true) @@ -1148,6 +1145,29 @@ class Reline::LineEditor modified end + def save_old_buffer + @old_buffer_of_lines = @buffer_of_lines.dup + end + + def push_input_lines + if @old_buffer_of_lines == @buffer_of_lines + @input_lines[@input_lines_position] = [@buffer_of_lines.dup, @byte_pointer, @line_index] + else + @input_lines = @input_lines[0..@input_lines_position] + @input_lines_position += 1 + @input_lines.push([@buffer_of_lines.dup, @byte_pointer, @line_index]) + end + trim_input_lines + end + + MAX_INPUT_LINES = 100 + def trim_input_lines + if @input_lines.size > MAX_INPUT_LINES + @input_lines.shift + @input_lines_position -= 1 + end + end + def scroll_into_view _wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position if wrapped_cursor_y < screen_scroll_top @@ -1224,6 +1244,18 @@ class Reline::LineEditor process_auto_indent end + def set_current_lines(lines, byte_pointer = nil, line_index = 0) + cursor = current_byte_pointer_cursor + @buffer_of_lines = lines + @line_index = line_index + if byte_pointer + @byte_pointer = byte_pointer + else + calculate_nearest_cursor(cursor) + end + process_auto_indent + end + def retrieve_completion_block(set_completion_quote_character = false) if Reline.completer_word_break_characters.empty? word_break_regexp = nil @@ -1305,6 +1337,18 @@ class Reline::LineEditor @confirm_multiline_termination_proc.(temp_buffer.join("\n") + "\n") end + def insert_pasted_text(text) + save_old_buffer + pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) + post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) + lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines << '' if lines.empty? + @buffer_of_lines[@line_index, 1] = lines + @line_index += lines.size - 1 + @byte_pointer = @buffer_of_lines[@line_index].bytesize - post.bytesize + push_input_lines + end + def insert_text(text) if @buffer_of_lines[@line_index].bytesize == @byte_pointer @buffer_of_lines[@line_index] += text @@ -1361,6 +1405,10 @@ class Reline::LineEditor whole_lines.join("\n") end + private def buffer_empty? + current_line.empty? and @buffer_of_lines.size == 1 + end + def finished? @finished end @@ -1709,29 +1757,31 @@ class Reline::LineEditor end private def ed_search_prev_history(key, arg: 1) - substr = current_line.byteslice(0, @byte_pointer) + substr = prev_action_state_value(:search_history) == :empty ? '' : current_line.byteslice(0, @byte_pointer) return if @history_pointer == 0 return if @history_pointer.nil? && substr.empty? && !current_line.empty? history_range = 0...(@history_pointer || Reline::HISTORY.size) h_pointer, line_index = search_history(substr, history_range.reverse_each) return unless h_pointer - move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) + move_history(h_pointer, line: line_index || :start, cursor: substr.empty? ? :end : @byte_pointer) arg -= 1 + set_next_action_state(:search_history, :empty) if substr.empty? ed_search_prev_history(key, arg: arg) if arg > 0 end alias_method :history_search_backward, :ed_search_prev_history private def ed_search_next_history(key, arg: 1) - substr = current_line.byteslice(0, @byte_pointer) + substr = prev_action_state_value(:search_history) == :empty ? '' : current_line.byteslice(0, @byte_pointer) return if @history_pointer.nil? history_range = @history_pointer + 1...Reline::HISTORY.size h_pointer, line_index = search_history(substr, history_range) return if h_pointer.nil? and not substr.empty? - move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) + move_history(h_pointer, line: line_index || :start, cursor: substr.empty? ? :end : @byte_pointer) arg -= 1 + set_next_action_state(:search_history, :empty) if substr.empty? ed_search_next_history(key, arg: arg) if arg > 0 end alias_method :history_search_forward, :ed_search_next_history @@ -1887,7 +1937,7 @@ class Reline::LineEditor alias_method :kill_whole_line, :em_kill_line private def em_delete(key) - if current_line.empty? and @buffer_of_lines.size == 1 and key == "\C-d".ord + if buffer_empty? and key == "\C-d".ord @eof = true finish elsif @byte_pointer < current_line.bytesize @@ -2235,8 +2285,7 @@ class Reline::LineEditor end private def vi_list_or_eof(key) - if current_line.empty? and @buffer_of_lines.size == 1 - set_current_line('', 0) + if buffer_empty? @eof = true finish else @@ -2477,4 +2526,32 @@ class Reline::LineEditor private def vi_editing_mode(key) @config.editing_mode = :vi_insert end + + private def undo(_key) + @undoing = true + + return if @input_lines_position <= 0 + + @input_lines_position -= 1 + target_lines, target_cursor_x, target_cursor_y = @input_lines[@input_lines_position] + set_current_lines(target_lines.dup, target_cursor_x, target_cursor_y) + end + + private def redo(_key) + @undoing = true + + return if @input_lines_position >= @input_lines.size - 1 + + @input_lines_position += 1 + target_lines, target_cursor_x, target_cursor_y = @input_lines[@input_lines_position] + set_current_lines(target_lines.dup, target_cursor_x, target_cursor_y) + end + + private def prev_action_state_value(type) + @prev_action_state[0] == type ? @prev_action_state[1] : nil + end + + private def set_next_action_state(type, value) + @next_action_state = [type, value] + end end diff --git a/lib/reline/terminfo.rb b/lib/reline/terminfo.rb index 6885a0c6be..1eb371464b 100644 --- a/lib/reline/terminfo.rb +++ b/lib/reline/terminfo.rb @@ -1,4 +1,7 @@ begin + # Ignore warning `Add fiddle to your Gemfile or gemspec` in Ruby 3.4. + # terminfo.rb and ansi.rb supports fiddle unavailable environment. + verbose, $VERBOSE = $VERBOSE, nil require 'fiddle' require 'fiddle/import' rescue LoadError @@ -7,6 +10,8 @@ rescue LoadError false end end +ensure + $VERBOSE = verbose end module Reline::Terminfo diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index 82c9ec427c..d7460d6d4a 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -43,11 +43,13 @@ class Reline::Unicode def self.escape_for_print(str) str.chars.map! { |gr| - escaped = EscapedPairs[gr.ord] - if escaped && gr != -"\n" && gr != -"\t" - escaped - else + case gr + when -"\n" gr + when -"\t" + -' ' + else + EscapedPairs[gr.ord] || gr end }.join end diff --git a/lib/reline/version.rb b/lib/reline/version.rb index ab10713f45..680e5a9cca 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.5.5' + VERSION = '0.5.8' end diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index f9450241c9..9e4b28f87a 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -504,7 +504,7 @@ module RubyVM::RJIT shape = C.rb_shape_get_shape_by_id(shape_id) current_capacity = shape.capacity - dest_shape = C.rb_shape_get_next(shape, comptime_receiver, ivar_name) + dest_shape = C.rb_shape_get_next_no_warnings(shape, comptime_receiver, ivar_name) new_shape_id = C.rb_shape_id(dest_shape) if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID diff --git a/lib/rubygems.rb b/lib/rubygems.rb index ad7ab10756..ac225ca70a 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -1013,6 +1013,13 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end ## + # Is this platform FreeBSD + + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + + ## # Load +plugins+ as Ruby files def self.load_plugin_files(plugins) # :nodoc: diff --git a/lib/rubygems/basic_specification.rb b/lib/rubygems/basic_specification.rb index 0380fceece..f25756f92c 100644 --- a/lib/rubygems/basic_specification.rb +++ b/lib/rubygems/basic_specification.rb @@ -144,6 +144,19 @@ class Gem::BasicSpecification end ## + # Returns the full name of this Gem (see `Gem::BasicSpecification#full_name`). + # Information about where the gem is installed is also included if not + # installed in the default GEM_HOME. + + def full_name_with_location + if base_dir != Gem.dir + "#{full_name} in #{base_dir}" + else + full_name + end + end + + ## # Full paths in the gem to add to <code>$LOAD_PATH</code> when this gem is # activated. diff --git a/lib/rubygems/commands/pristine_command.rb b/lib/rubygems/commands/pristine_command.rb index 456d897df2..999c9fef0f 100644 --- a/lib/rubygems/commands/pristine_command.rb +++ b/lib/rubygems/commands/pristine_command.rb @@ -57,7 +57,7 @@ class Gem::Commands::PristineCommand < Gem::Command end add_option("-i", "--install-dir DIR", - "Gem repository to get binstubs and plugins installed") do |value, options| + "Gem repository to get gems restored") do |value, options| options[:install_dir] = File.expand_path(value) end @@ -103,21 +103,25 @@ extensions will be restored. end def execute + install_dir = options[:install_dir] + + specification_record = install_dir ? Gem::SpecificationRecord.from_path(install_dir) : Gem::Specification.specification_record + specs = if options[:all] - Gem::Specification.map + specification_record.map # `--extensions` must be explicitly given to pristine only gems # with extensions. elsif options[:extensions_set] && options[:extensions] && options[:args].empty? - Gem::Specification.select do |spec| + specification_record.select do |spec| spec.extensions && !spec.extensions.empty? end elsif options[:only_missing_extensions] - Gem::Specification.select(&:missing_extensions?) + specification_record.select(&:missing_extensions?) else get_all_gem_names.sort.map do |gem_name| - Gem::Specification.find_all_by_name(gem_name, options[:version]).reverse + specification_record.find_all_by_name(gem_name, options[:version]).reverse end.flatten end @@ -144,7 +148,7 @@ extensions will be restored. end unless spec.extensions.empty? || options[:extensions] || options[:only_executables] || options[:only_plugins] - say "Skipped #{spec.full_name}, it needs to compile an extension" + say "Skipped #{spec.full_name_with_location}, it needs to compile an extension" next end @@ -153,7 +157,7 @@ extensions will be restored. unless File.exist?(gem) || options[:only_executables] || options[:only_plugins] require_relative "../remote_fetcher" - say "Cached gem for #{spec.full_name} not found, attempting to fetch..." + say "Cached gem for #{spec.full_name_with_location} not found, attempting to fetch..." dep = Gem::Dependency.new spec.name, spec.version found, _ = Gem::SpecFetcher.fetcher.spec_for_dependency dep @@ -176,7 +180,6 @@ extensions will be restored. end bin_dir = options[:bin_dir] if options[:bin_dir] - install_dir = options[:install_dir] if options[:install_dir] installer_options = { wrappers: true, @@ -198,7 +201,7 @@ extensions will be restored. installer.install end - say "Restored #{spec.full_name}" + say "Restored #{spec.full_name_with_location}" end end end diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb index 3f38074280..9c633d6ef7 100644 --- a/lib/rubygems/commands/setup_command.rb +++ b/lib/rubygems/commands/setup_command.rb @@ -585,6 +585,8 @@ abort "#{deprecation_message}" args = %w[--all --only-executables --silent] args << "--bindir=#{bindir}" + args << "--install-dir=#{default_dir}" + if options[:env_shebang] args << "--env-shebang" end diff --git a/lib/rubygems/commands/uninstall_command.rb b/lib/rubygems/commands/uninstall_command.rb index 2a77ec72cf..283bc96ce3 100644 --- a/lib/rubygems/commands/uninstall_command.rb +++ b/lib/rubygems/commands/uninstall_command.rb @@ -184,7 +184,7 @@ that is a dependency of an existing gem. You can use the rescue Gem::GemNotInHomeException => e spec = e.spec alert("In order to remove #{spec.name}, please execute:\n" \ - "\tgem uninstall #{spec.name} --install-dir=#{spec.installation_path}") + "\tgem uninstall #{spec.name} --install-dir=#{spec.base_dir}") rescue Gem::UninstallError => e spec = e.spec alert_error("Error: unable to successfully uninstall '#{spec.name}' which is " \ diff --git a/lib/rubygems/dependency.rb b/lib/rubygems/dependency.rb index d1bf074441..5ce9c5e840 100644 --- a/lib/rubygems/dependency.rb +++ b/lib/rubygems/dependency.rb @@ -271,15 +271,7 @@ class Gem::Dependency end def matching_specs(platform_only = false) - env_req = Gem.env_requirement(name) - matches = Gem::Specification.stubs_for(name).find_all do |spec| - requirement.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) - end.map(&:to_spec) - - if prioritizes_bundler? - require_relative "bundler_version_finder" - Gem::BundlerVersionFinder.prioritize!(matches) - end + matches = Gem::Specification.find_all_by_name(name, requirement) if platform_only matches.reject! do |spec| @@ -297,10 +289,6 @@ class Gem::Dependency @requirement.specific? end - def prioritizes_bundler? - name == "bundler" && !specific? - end - def to_specs matches = matching_specs true diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index 8f6f9a5aa8..844f292ba2 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -344,7 +344,7 @@ class Gem::Installer say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? - Gem::Specification.add_spec(spec) + Gem::Specification.add_spec(spec) unless @install_dir load_plugin diff --git a/lib/rubygems/package/tar_header.rb b/lib/rubygems/package/tar_header.rb index 087f13f6c9..dd5e835a1e 100644 --- a/lib/rubygems/package/tar_header.rb +++ b/lib/rubygems/package/tar_header.rb @@ -95,14 +95,14 @@ class Gem::Package::TarHeader attr_reader(*FIELDS) - EMPTY_HEADER = ("\0" * 512).freeze # :nodoc: + EMPTY_HEADER = ("\0" * 512).b.freeze # :nodoc: ## # Creates a tar header from IO +stream+ def self.from(stream) header = stream.read 512 - empty = (header == EMPTY_HEADER) + return EMPTY if header == EMPTY_HEADER fields = header.unpack UNPACK_FORMAT @@ -123,7 +123,7 @@ class Gem::Package::TarHeader devminor: strict_oct(fields.shift), prefix: fields.shift, - empty: empty + empty: false end def self.strict_oct(str) @@ -172,6 +172,22 @@ class Gem::Package::TarHeader @empty = vals[:empty] end + EMPTY = new({ # :nodoc: + checksum: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + name: "", + prefix: "", + size: 0, + uname: "", + version: 0, + + empty: true, + }).freeze + private_constant :EMPTY + ## # Is the tar entry empty? @@ -241,7 +257,7 @@ class Gem::Package::TarHeader header = header.pack PACK_FORMAT - header << ("\0" * ((512 - header.size) % 512)) + header.ljust 512, "\0" end def oct(num, len) diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb index 48b7344aee..d54ad12880 100644 --- a/lib/rubygems/platform.rb +++ b/lib/rubygems/platform.rb @@ -134,6 +134,7 @@ class Gem::Platform when /netbsdelf/ then ["netbsdelf", nil] when /openbsd(\d+\.\d+)?/ then ["openbsd", $1] when /solaris(\d+\.\d+)?/ then ["solaris", $1] + when /wasi/ then ["wasi", nil] # test when /^(\w+_platform)(\d+)?/ then [$1, $2] else ["unknown", nil] diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index a1eaf1248e..da55a2e6d3 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -11,6 +11,7 @@ require_relative "deprecate" require_relative "basic_specification" require_relative "stub_specification" require_relative "platform" +require_relative "specification_record" require_relative "util/list" require "rbconfig" @@ -179,19 +180,9 @@ class Gem::Specification < Gem::BasicSpecification @@default_value[k].nil? end - def self.clear_specs # :nodoc: - @@all = nil - @@stubs = nil - @@stubs_by_name = {} - @@spec_with_requirable_file = {} - @@active_stub_with_requirable_file = {} - end - private_class_method :clear_specs - - clear_specs - # Sentinel object to represent "not found" stubs NOT_FOUND = Struct.new(:to_spec, :this).new # :nodoc: + deprecate_constant :NOT_FOUND # Tracking removed method calls to warn users during build time. REMOVED_METHODS = [:rubyforge_project=, :mark_version].freeze # :nodoc: @@ -770,7 +761,7 @@ class Gem::Specification < Gem::BasicSpecification attr_accessor :specification_version def self._all # :nodoc: - @@all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + specification_record.all end def self.clear_load_cache # :nodoc: @@ -788,26 +779,9 @@ class Gem::Specification < Gem::BasicSpecification end end - def self.gemspec_stubs_in(dir, pattern) + def self.gemspec_stubs_in(dir, pattern) # :nodoc: Gem::Util.glob_files_in_dir(pattern, dir).map {|path| yield path }.select(&:valid?) end - private_class_method :gemspec_stubs_in - - def self.installed_stubs(dirs, pattern) - map_stubs(dirs, pattern) do |path, base_dir, gems_dir| - Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) - end - end - private_class_method :installed_stubs - - def self.map_stubs(dirs, pattern) # :nodoc: - dirs.flat_map do |dir| - base_dir = File.dirname dir - gems_dir = File.join base_dir, "gems" - gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } - end - end - private_class_method :map_stubs def self.each_spec(dirs) # :nodoc: each_gemspec(dirs) do |path| @@ -820,13 +794,7 @@ class Gem::Specification < Gem::BasicSpecification # Returns a Gem::StubSpecification for every installed gem def self.stubs - @@stubs ||= begin - pattern = "*.gemspec" - stubs = stubs_for_pattern(pattern, false) - - @@stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) - stubs - end + specification_record.stubs end ## @@ -845,13 +813,7 @@ class Gem::Specification < Gem::BasicSpecification # only returns stubs that match Gem.platforms def self.stubs_for(name) - if @@stubs - @@stubs_by_name[name] || [] - else - @@stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| - s.name == name - end - end + specification_record.stubs_for(name) end ## @@ -859,12 +821,7 @@ class Gem::Specification < Gem::BasicSpecification # optionally filtering out specs not matching the current platform # def self.stubs_for_pattern(pattern, match_platform = true) # :nodoc: - installed_stubs = installed_stubs(Gem::Specification.dirs, pattern) - installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform - stubs = installed_stubs + default_stubs(pattern) - stubs = stubs.uniq(&:full_name) - _resort!(stubs) - stubs + specification_record.stubs_for_pattern(pattern, match_platform) end def self._resort!(specs) # :nodoc: @@ -873,7 +830,9 @@ class Gem::Specification < Gem::BasicSpecification next names if names.nonzero? versions = b.version <=> a.version next versions if versions.nonzero? - Gem::Platform.sort_priority(b.platform) + platforms = Gem::Platform.sort_priority(b.platform) <=> Gem::Platform.sort_priority(a.platform) + next platforms if platforms.nonzero? + b.base_dir == Gem.path.first ? 1 : -1 end end @@ -893,23 +852,14 @@ class Gem::Specification < Gem::BasicSpecification # properly sorted. def self.add_spec(spec) - return if _all.include? spec - - _all << spec - stubs << spec - (@@stubs_by_name[spec.name] ||= []) << spec - - _resort!(@@stubs_by_name[spec.name]) - _resort!(stubs) + specification_record.add_spec(spec) end ## # Removes +spec+ from the known specs. def self.remove_spec(spec) - _all.delete spec.to_spec - stubs.delete spec - (@@stubs_by_name[spec.name] || []).delete spec + specification_record.remove_spec(spec) end ## @@ -923,27 +873,17 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the known specs to +specs+. Not guaranteed to work for you in - # the future. Use at your own risk. Caveat emptor. Doomy doom doom. - # Etc etc. - # - #-- - # Makes +specs+ the known specs - # Listen, time is a river - # Winter comes, code breaks - # - # -- wilsonb + # Sets the known specs to +specs+. def self.all=(specs) - @@stubs_by_name = specs.group_by(&:name) - @@all = @@stubs = specs + specification_record.all = specs end ## # Return full names of all specs in sorted order. def self.all_names - _all.map(&:full_name) + specification_record.all_names end ## @@ -968,9 +908,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the directories that Specification uses to find specs. def self.dirs - @@dirs ||= Gem.path.collect do |dir| - File.join dir, "specifications" - end + @@dirs ||= Gem::SpecificationRecord.dirs_from(Gem.path) end ## @@ -980,7 +918,7 @@ class Gem::Specification < Gem::BasicSpecification def self.dirs=(dirs) reset - @@dirs = Array(dirs).map {|dir| File.join dir, "specifications" } + @@dirs = Gem::SpecificationRecord.dirs_from(Array(dirs)) end extend Enumerable @@ -989,21 +927,15 @@ class Gem::Specification < Gem::BasicSpecification # Enumerate every known spec. See ::dirs= and ::add_spec to set the list of # specs. - def self.each - return enum_for(:each) unless block_given? - - _all.each do |x| - yield x - end + def self.each(&block) + specification_record.each(&block) end ## # Returns every spec that matches +name+ and optional +requirements+. def self.find_all_by_name(name, *requirements) - requirements = Gem::Requirement.default if requirements.empty? - - Gem::Dependency.new(name, *requirements).matching_specs + specification_record.find_all_by_name(name, *requirements) end ## @@ -1033,12 +965,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the best specification that contains the file matching +path+. def self.find_by_path(path) - path = path.dup.freeze - spec = @@spec_with_requirable_file[path] ||= stubs.find do |s| - s.contains_requirable_file? path - end || NOT_FOUND - - spec.to_spec + specification_record.find_by_path(path) end ## @@ -1046,19 +973,15 @@ class Gem::Specification < Gem::BasicSpecification # amongst the specs that are not activated. def self.find_inactive_by_path(path) - stub = stubs.find do |s| - next if s.activated? - s.contains_requirable_file? path - end - stub&.to_spec + specification_record.find_inactive_by_path(path) end - def self.find_active_stub_by_path(path) - stub = @@active_stub_with_requirable_file[path] ||= stubs.find do |s| - s.activated? && s.contains_requirable_file?(path) - end || NOT_FOUND + ## + # Return the best specification that contains the file matching +path+, among + # those already activated. - stub.this + def self.find_active_stub_by_path(path) + specification_record.find_active_stub_by_path(path) end ## @@ -1125,14 +1048,14 @@ class Gem::Specification < Gem::BasicSpecification # +prerelease+ is true. def self.latest_specs(prerelease = false) - _latest_specs Gem::Specification.stubs, prerelease + specification_record.latest_specs(prerelease) end ## # Return the latest installed spec for gem +name+. def self.latest_spec_for(name) - latest_specs(true).find {|installed_spec| installed_spec.name == name } + specification_record.latest_spec_for(name) end def self._latest_specs(specs, prerelease = false) # :nodoc: @@ -1270,7 +1193,7 @@ class Gem::Specification < Gem::BasicSpecification def self.reset @@dirs = nil Gem.pre_reset_hooks.each(&:call) - clear_specs + @specification_record = nil clear_load_cache unresolved = unresolved_deps unless unresolved.empty? @@ -1291,6 +1214,13 @@ class Gem::Specification < Gem::BasicSpecification Gem.post_reset_hooks.each(&:call) end + ## + # Keeps track of all currently known specifications + + def self.specification_record + @specification_record ||= Gem::SpecificationRecord.new(dirs) + end + # DOC: This method needs documented or nodoc'd def self.unresolved_deps @unresolved_deps ||= Hash.new {|h, n| h[n] = Gem::Dependency.new n } diff --git a/lib/rubygems/specification_record.rb b/lib/rubygems/specification_record.rb new file mode 100644 index 0000000000..664d506265 --- /dev/null +++ b/lib/rubygems/specification_record.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true + +module Gem + class SpecificationRecord + def self.dirs_from(paths) + paths.map do |path| + File.join(path, "specifications") + end + end + + def self.from_path(path) + new(dirs_from([path])) + end + + def initialize(dirs) + @all = nil + @stubs = nil + @stubs_by_name = {} + @spec_with_requirable_file = {} + @active_stub_with_requirable_file = {} + + @dirs = dirs + end + + # Sentinel object to represent "not found" stubs + NOT_FOUND = Struct.new(:to_spec, :this).new + private_constant :NOT_FOUND + + ## + # Returns the list of all specifications in the record + + def all + @all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + + def stubs + @stubs ||= begin + pattern = "*.gemspec" + stubs = stubs_for_pattern(pattern, false) + + @stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) + stubs + end + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + # named +name+ only returns stubs that match Gem.platforms + + def stubs_for(name) + if @stubs + @stubs_by_name[name] || [] + else + @stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| + s.name == name + end + end + end + + ## + # Finds stub specifications matching a pattern in the record, optionally + # filtering out specs not matching the current platform + + def stubs_for_pattern(pattern, match_platform = true) + installed_stubs = installed_stubs(pattern) + installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform + stubs = installed_stubs + Gem::Specification.default_stubs(pattern) + Gem::Specification._resort!(stubs) + stubs + end + + ## + # Adds +spec+ to the the record, keeping the collection properly sorted. + + def add_spec(spec) + return if all.include? spec + + all << spec + stubs << spec + (@stubs_by_name[spec.name] ||= []) << spec + + Gem::Specification._resort!(@stubs_by_name[spec.name]) + Gem::Specification._resort!(stubs) + end + + ## + # Removes +spec+ from the record. + + def remove_spec(spec) + all.delete spec.to_spec + stubs.delete spec + (@stubs_by_name[spec.name] || []).delete spec + end + + ## + # Sets the specs known by the record to +specs+. + + def all=(specs) + @stubs_by_name = specs.group_by(&:name) + @all = @stubs = specs + end + + ## + # Return full names of all specs in the record in sorted order. + + def all_names + all.map(&:full_name) + end + + include Enumerable + + ## + # Enumerate every known spec. + + def each + return enum_for(:each) unless block_given? + + all.each do |x| + yield x + end + end + + ## + # Returns every spec in the record that matches +name+ and optional +requirements+. + + def find_all_by_name(name, *requirements) + req = Gem::Requirement.create(*requirements) + env_req = Gem.env_requirement(name) + + matches = stubs_for(name).find_all do |spec| + req.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) + end.map(&:to_spec) + + if name == "bundler" && !req.specific? + require_relative "bundler_version_finder" + Gem::BundlerVersionFinder.prioritize!(matches) + end + + matches + end + + ## + # Return the best specification in the record that contains the file matching +path+. + + def find_by_path(path) + path = path.dup.freeze + spec = @spec_with_requirable_file[path] ||= stubs.find do |s| + s.contains_requirable_file? path + end || NOT_FOUND + + spec.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+ amongst the specs that are not activated. + + def find_inactive_by_path(path) + stub = stubs.find do |s| + next if s.activated? + s.contains_requirable_file? path + end + stub&.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+, among those already activated. + + def find_active_stub_by_path(path) + stub = @active_stub_with_requirable_file[path] ||= stubs.find do |s| + s.activated? && s.contains_requirable_file?(path) + end || NOT_FOUND + + stub.this + end + + ## + # Return the latest specs in the record, optionally including prerelease + # specs if +prerelease+ is true. + + def latest_specs(prerelease) + Gem::Specification._latest_specs stubs, prerelease + end + + ## + # Return the latest installed spec in the record for gem +name+. + + def latest_spec_for(name) + latest_specs(true).find {|installed_spec| installed_spec.name == name } + end + + private + + def installed_stubs(pattern) + map_stubs(pattern) do |path, base_dir, gems_dir| + Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) + end + end + + def map_stubs(pattern) + @dirs.flat_map do |dir| + base_dir = File.dirname dir + gems_dir = File.join base_dir, "gems" + Gem::Specification.gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } + end + end + end +end diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index 58748df5d6..ea66fbc3f6 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -210,4 +210,25 @@ class Gem::StubSpecification < Gem::BasicSpecification def stubbed? data.is_a? StubLine end + + def ==(other) # :nodoc: + self.class === other && + name == other.name && + version == other.version && + platform == other.platform + end + + alias_method :eql?, :== # :nodoc: + + def hash # :nodoc: + name.hash ^ version.hash ^ platform.hash + end + + def <=>(other) # :nodoc: + sort_obj <=> other.sort_obj + end + + def sort_obj # :nodoc: + [name, version, Gem::Platform.sort_priority(platform)] + end end diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index c96df2a085..23791313c8 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -32,7 +32,7 @@ class Gem::Uninstaller attr_reader :bin_dir ## - # The gem repository the gem will be installed into + # The gem repository the gem will be uninstalled from attr_reader :gem_home @@ -49,8 +49,9 @@ class Gem::Uninstaller # TODO: document the valid options @gem = gem @version = options[:version] || Gem::Requirement.default - @gem_home = File.realpath(options[:install_dir] || Gem.dir) - @plugins_dir = Gem.plugindir(@gem_home) + @install_dir = options[:install_dir] + @gem_home = File.realpath(@install_dir || Gem.dir) + @user_dir = File.exist?(Gem.user_dir) ? File.realpath(Gem.user_dir) : Gem.user_dir @force_executables = options[:executables] @force_all = options[:all] @force_ignore = options[:ignore] @@ -70,7 +71,7 @@ class Gem::Uninstaller # only add user directory if install_dir is not set @user_install = false - @user_install = options[:user_install] unless options[:install_dir] + @user_install = options[:user_install] unless @install_dir # Optimization: populated during #uninstall @default_specs_matching_uninstall_params = [] @@ -85,11 +86,7 @@ class Gem::Uninstaller list = [] - dirs = - Gem::Specification.dirs + - [Gem.default_specifications_dir] - - Gem::Specification.each_spec dirs do |spec| + specification_record.stubs.each do |spec| next unless dependency.matches_spec? spec list << spec @@ -101,11 +98,11 @@ class Gem::Uninstaller default_specs, list = list.partition(&:default_gem?) warn_cannot_uninstall_default_gems(default_specs - list) - @default_specs_matching_uninstall_params = default_specs + @default_specs_matching_uninstall_params = default_specs.map(&:to_spec) list, other_repo_specs = list.partition do |spec| @gem_home == spec.base_dir || - (@user_install && spec.base_dir == Gem.user_dir) + (@user_install && spec.base_dir == @user_dir) end list.sort! @@ -125,7 +122,7 @@ class Gem::Uninstaller remove_all list elsif list.size > 1 - gem_names = list.map(&:full_name) + gem_names = list.map(&:full_name_with_location) gem_names << "All versions" say @@ -146,7 +143,9 @@ class Gem::Uninstaller ## # Uninstalls gem +spec+ - def uninstall_gem(spec) + def uninstall_gem(stub) + spec = stub.to_spec + @spec = spec unless dependencies_ok? spec @@ -164,6 +163,8 @@ class Gem::Uninstaller remove_plugins @spec remove @spec + specification_record.remove_spec(stub) + regenerate_plugins Gem.post_uninstall_hooks.each do |hook| @@ -239,7 +240,7 @@ class Gem::Uninstaller def remove(spec) unless path_ok?(@gem_home, spec) || - (@user_install && path_ok?(Gem.user_dir, spec)) + (@user_install && path_ok?(@user_dir, spec)) e = Gem::GemNotInHomeException.new \ "Gem '#{spec.full_name}' is not installed in directory #{@gem_home}" e.spec = spec @@ -274,8 +275,6 @@ class Gem::Uninstaller safe_delete { FileUtils.rm_r gemspec } announce_deletion_of(spec) - - Gem::Specification.reset end ## @@ -284,17 +283,17 @@ class Gem::Uninstaller def remove_plugins(spec) # :nodoc: return if spec.plugins.empty? - remove_plugins_for(spec, @plugins_dir) + remove_plugins_for(spec, plugin_dir_for(spec)) end ## # Regenerates plugin wrappers after removal. def regenerate_plugins - latest = Gem::Specification.latest_spec_for(@spec.name) + latest = specification_record.latest_spec_for(@spec.name) return if latest.nil? - regenerate_plugins_for(latest, @plugins_dir) + regenerate_plugins_for(latest, plugin_dir_for(@spec)) end ## @@ -379,6 +378,10 @@ class Gem::Uninstaller private + def specification_record + @specification_record ||= @install_dir ? Gem::SpecificationRecord.from_path(@install_dir) : Gem::Specification.specification_record + end + def announce_deletion_of(spec) name = spec.full_name say "Successfully uninstalled #{name}" @@ -406,4 +409,8 @@ class Gem::Uninstaller say "Gem #{spec.full_name} cannot be uninstalled because it is a default gem" end end + + def plugin_dir_for(spec) + Gem.plugindir(spec.base_dir) + end end diff --git a/lib/rubygems/util/licenses.rb b/lib/rubygems/util/licenses.rb index f3c7201639..192ae30b9b 100644 --- a/lib/rubygems/util/licenses.rb +++ b/lib/rubygems/util/licenses.rb @@ -15,6 +15,7 @@ class Gem::Licenses # license identifiers LICENSE_IDENTIFIERS = %w[ 0BSD + 3D-Slicer-1.0 AAL ADSL AFL-1.1 @@ -26,6 +27,7 @@ class Gem::Licenses AGPL-1.0-or-later AGPL-3.0-only AGPL-3.0-or-later + AMD-newlib AMDPLPA AML AML-glslang @@ -62,6 +64,7 @@ class Gem::Licenses BSD-2-Clause-Darwin BSD-2-Clause-Patent BSD-2-Clause-Views + BSD-2-Clause-first-lines BSD-3-Clause BSD-3-Clause-Attribution BSD-3-Clause-Clear @@ -191,6 +194,7 @@ class Gem::Licenses CUA-OPL-1.0 Caldera Caldera-no-preamble + Catharon ClArtistic Clips Community-Spec-1.0 @@ -270,25 +274,32 @@ class Gem::Licenses Glide Glulxe Graphics-Gems + Gutmann HP-1986 HP-1989 HPND HPND-DEC HPND-Fenneberg-Livingston HPND-INRIA-IMAG + HPND-Intel HPND-Kevlin-Henney HPND-MIT-disclaimer HPND-Markus-Kuhn HPND-Pbmplus HPND-UC + HPND-UC-export-US HPND-doc HPND-doc-sell HPND-export-US + HPND-export-US-acknowledgement HPND-export-US-modify + HPND-export2-US + HPND-merchantability-variant HPND-sell-MIT-disclaimer-xserver HPND-sell-regexpr HPND-sell-variant HPND-sell-variant-MIT-disclaimer + HPND-sell-variant-MIT-disclaimer-rev HTMLTIDY HaskellReport Hippocratic-2.1 @@ -353,6 +364,7 @@ class Gem::Licenses MIT-0 MIT-CMU MIT-Festival + MIT-Khronos-old MIT-Modern-Variant MIT-Wu MIT-advertising @@ -386,7 +398,9 @@ class Gem::Licenses NAIST-2003 NASA-1.3 NBPL-1.0 + NCBI-PD NCGL-UK-2.0 + NCL NCSA NGPL NICTA-1.0 @@ -410,6 +424,7 @@ class Gem::Licenses Nokia Noweb O-UDA-1.0 + OAR OCCT-PL OCLC-2.0 ODC-By-1.0 @@ -463,6 +478,7 @@ class Gem::Licenses PDDL-1.0 PHP-3.0 PHP-3.01 + PPL PSF-2.0 Parity-6.0.0 Parity-7.0.0 @@ -518,6 +534,7 @@ class Gem::Licenses Spencer-99 SugarCRM-1.1.3 Sun-PPP + Sun-PPP-2000 SunPro Symlinks TAPR-OHL-1.0 @@ -574,6 +591,7 @@ class Gem::Licenses Zimbra-1.3 Zimbra-1.4 Zlib + any-OSI bcrypt-Solar-Designer blessing bzip2-1.0.6 @@ -582,6 +600,7 @@ class Gem::Licenses copyleft-next-0.3.0 copyleft-next-0.3.1 curl + cve-tou diffmark dtoa dvipdfm @@ -604,6 +623,7 @@ class Gem::Licenses mpi-permissive mpich2 mplus + pkgconf pnmstitch psfrag psutils @@ -613,12 +633,14 @@ class Gem::Licenses softSurfer ssh-keyscan swrule + threeparttable ulem w3m xinetd xkeyboard-config-Zinoviev xlock xpp + xzoom zlib-acknowledgement ].freeze @@ -660,6 +682,7 @@ class Gem::Licenses EXCEPTION_IDENTIFIERS = %w[ 389-exception Asterisk-exception + Asterisk-linking-protocols-exception Autoconf-exception-2.0 Autoconf-exception-3.0 Autoconf-exception-generic @@ -697,11 +720,13 @@ class Gem::Licenses OCCT-exception-1.0 OCaml-LGPL-linking-exception OpenJDK-assembly-exception-1.0 + PCRE2-exception PS-or-PDF-font-exception-20170817 QPL-1.0-INRIA-2004-exception Qt-GPL-exception-1.0 Qt-LGPL-exception-1.1 Qwt-exception-1.0 + RRDtool-FLOSS-exception-2.0 SANE-exception SHL-2.0 SHL-2.1 diff --git a/lib/rubygems/yaml_serializer.rb b/lib/rubygems/yaml_serializer.rb index 128becc1ce..af86c63ef7 100644 --- a/lib/rubygems/yaml_serializer.rb +++ b/lib/rubygems/yaml_serializer.rb @@ -60,7 +60,6 @@ module Gem indent, key, quote, val = match.captures val = strip_comment(val) - convert_to_backward_compatible_key!(key) depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} @@ -92,14 +91,8 @@ module Gem end end - # for settings' keys - def convert_to_backward_compatible_key!(key) - key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) - key.gsub!(".", "__") - end - class << self - private :dump_hash, :convert_to_backward_compatible_key! + private :dump_hash end end end diff --git a/lib/tempfile.rb b/lib/tempfile.rb index 1d7b80a74d..bddef3bf22 100644 --- a/lib/tempfile.rb +++ b/lib/tempfile.rb @@ -8,18 +8,61 @@ require 'delegate' require 'tmpdir' -# A utility class for managing temporary files. When you create a Tempfile -# object, it will create a temporary file with a unique filename. A Tempfile -# objects behaves just like a File object, and you can perform all the usual -# file operations on it: reading data, writing data, changing its permissions, -# etc. So although this class does not explicitly document all instance methods -# supported by File, you can in fact call any File instance method on a -# Tempfile object. +# A utility class for managing temporary files. +# +# There are two kind of methods of creating a temporary file: +# +# - Tempfile.create (recommended) +# - Tempfile.new and Tempfile.open (mostly for backward compatibility, not recommended) +# +# Tempfile.create creates a usual \File object. +# The timing of file deletion is predictable. +# Also, it supports open-and-unlink technique which +# removes the temporary file immediately after creation. +# +# Tempfile.new and Tempfile.open creates a \Tempfile object. +# The created file is removed by the GC (finalizer). +# The timing of file deletion is not predictable. # # == Synopsis # # require 'tempfile' # +# # Tempfile.create with a block +# # The filename are choosen automatically. +# # (You can specify the prefix and suffix of the filename by an optional argument.) +# Tempfile.create {|f| +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } # The file is removed at block exit. +# +# # Tempfile.create without a block +# # You need to unlink the file in non-block form. +# f = Tempfile.create +# f.puts "foo" +# f.close +# File.unlink(f.path) # You need to unlink the file. +# +# # Tempfile.create(anonymous: true) without a block +# f = Tempfile.create(anonymous: true) +# # The file is already removed because anonymous. +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.close +# +# # Tempfile.create(anonymous: true) with a block +# Tempfile.create(anonymous: true) {|f| +# # The file is already removed because anonymous. +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } +# +# # Not recommended: Tempfile.new without a block # file = Tempfile.new('foo') # file.path # => A unique filename in the OS's temp directory, # # e.g.: "/tmp/foo.24722.0" @@ -30,7 +73,27 @@ require 'tmpdir' # file.close # file.unlink # deletes the temp file # -# == Good practices +# == About Tempfile.new and Tempfile.open +# +# This section does not apply to Tempfile.create because +# it returns a File object (not a Tempfile object). +# +# When you create a Tempfile object, +# it will create a temporary file with a unique filename. A Tempfile +# objects behaves just like a File object, and you can perform all the usual +# file operations on it: reading data, writing data, changing its permissions, +# etc. So although this class does not explicitly document all instance methods +# supported by File, you can in fact call any File instance method on a +# Tempfile object. +# +# A Tempfile object has a finalizer to remove the temporary file. +# This means that the temporary file is removed via GC. +# This can cause several problems: +# +# - Long GC intervals and conservative GC can accumulate temporary files that are not removed. +# - Temporary files are not removed if Ruby exits abnormally (such as SIGKILL, SEGV). +# +# There are legacy good practices for Tempfile.new and Tempfile.open as follows. # # === Explicit close # @@ -71,12 +134,17 @@ require 'tmpdir' # be able to read from or write to the Tempfile, and you do not need to # know the Tempfile's filename either. # +# Also, this guarantees the temporary file is removed even if Ruby exits abnormally. +# The OS reclaims the storage for the temporary file when the file is closed or +# the Ruby process exits (normally or abnormally). +# # For example, a practical use case for unlink-after-creation would be this: # you need a large byte buffer that's too large to comfortably fit in RAM, # e.g. when you're writing a web server and you want to buffer the client's # file upload data. # -# Please refer to #unlink for more information and a code example. +# `Tempfile.create(anonymous: true)` supports this behavior. +# It also works on Windows. # # == Minor notes # @@ -392,8 +460,9 @@ end # see {File Permissions}[rdoc-ref:File@File+Permissions]. # - Mode is <tt>'w+'</tt> (read/write mode, positioned at the end). # -# With no block, the file is not removed automatically, -# and so should be explicitly removed. +# The temporary file removal depends on the keyword argument +anonymous+ and +# whether a block is given or not. +# See the description about the +anonymous+ keyword argument later. # # Example: # @@ -401,11 +470,36 @@ end # f.class # => File # f.path # => "/tmp/20220505-9795-17ky6f6" # f.stat.mode.to_s(8) # => "100600" +# f.close # File.exist?(f.path) # => true # File.unlink(f.path) # File.exist?(f.path) # => false # -# Argument +basename+, if given, may be one of: +# Tempfile.create {|f| +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.path # => "/tmp/20240524-380207-oma0ny" +# File.exist?(f.path) # => true +# } # The file is removed at block exit. +# +# f = Tempfile.create(anonymous: true) +# # The file is already removed because anonymous +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.close +# +# Tempfile.create(anonymous: true) {|f| +# # The file is already removed because anonymous +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } +# +# The argument +basename+, if given, may be one of the following: # # - A string: the generated filename begins with +basename+: # @@ -416,27 +510,57 @@ end # # Tempfile.create(%w/foo .jpg/) # => #<File:/tmp/foo20220505-17839-tnjchh.jpg> # -# With arguments +basename+ and +tmpdir+, the file is created in directory +tmpdir+: +# With arguments +basename+ and +tmpdir+, the file is created in the directory +tmpdir+: # # Tempfile.create('foo', '.') # => #<File:./foo20220505-9795-1emu6g8> # -# Keyword arguments +mode+ and +options+ are passed directly to method +# Keyword arguments +mode+ and +options+ are passed directly to the method # {File.open}[rdoc-ref:File.open]: # -# - The value given with +mode+ must be an integer, +# - The value given for +mode+ must be an integer # and may be expressed as the logical OR of constants defined in # {File::Constants}[rdoc-ref:File::Constants]. # - For +options+, see {Open Options}[rdoc-ref:IO@Open+Options]. # -# With a block given, creates the file as above, passes it to the block, -# and returns the block's value; -# before the return, the file object is closed and the underlying file is removed: +# The keyword argument +anonymous+ specifies when the file is removed. +# +# - +anonymous=false+ (default) without a block: the file is not removed. +# - +anonymous=false+ (default) with a block: the file is removed after the block exits. +# - +anonymous=true+ without a block: the file is removed before returning. +# - +anonymous=true+ with a block: the file is removed before the block is called. +# +# In the first case (+anonymous=false+ without a block), +# the file is not removed automatically. +# It should be explicitly closed. +# It can be used to rename to the desired filename. +# If the file is not needed, it should be explicitly removed. +# +# The +File#path+ method of the created file object returns the temporary directory with a trailing slash +# when +anonymous+ is true. +# +# When a block is given, it creates the file as described above, passes it to the block, +# and returns the block's value. +# Before the returning, the file object is closed and the underlying file is removed: # # Tempfile.create {|file| file.path } # => "/tmp/20220505-9795-rkists" # +# Implementation note: +# +# The keyword argument +anonymous=true+ is implemented using FILE_SHARE_DELETE on Windows. +# O_TMPFILE is used on Linux. +# # Related: Tempfile.new. # -def Tempfile.create(basename="", tmpdir=nil, mode: 0, **options) +def Tempfile.create(basename="", tmpdir=nil, mode: 0, anonymous: false, **options, &block) + if anonymous + create_anonymous(basename, tmpdir, mode: mode, **options, &block) + else + create_with_filename(basename, tmpdir, mode: mode, **options, &block) + end +end + +class << Tempfile +private def create_with_filename(basename="", tmpdir=nil, mode: 0, **options) tmpfile = nil Dir::Tmpname.create(basename, tmpdir, **options) do |tmpname, n, opts| mode |= File::RDWR|File::CREAT|File::EXCL @@ -464,3 +588,37 @@ def Tempfile.create(basename="", tmpdir=nil, mode: 0, **options) tmpfile end end + +private def create_anonymous(basename="", tmpdir=nil, mode: 0, **options, &block) + tmpfile = nil + tmpdir = Dir.tmpdir() if tmpdir.nil? + if defined?(File::TMPFILE) # O_TMPFILE since Linux 3.11 + begin + tmpfile = File.open(tmpdir, File::RDWR | File::TMPFILE, 0600) + rescue Errno::EISDIR, Errno::ENOENT, Errno::EOPNOTSUPP + # kernel or the filesystem does not support O_TMPFILE + # fallback to create-and-unlink + end + end + if tmpfile.nil? + mode |= File::SHARE_DELETE | File::BINARY # Windows needs them to unlink the opened file. + tmpfile = create_with_filename(basename, tmpdir, mode: mode, **options) + File.unlink(tmpfile.path) + end + path = File.join(tmpdir, '') + if tmpfile.path != path + # clear path. + tmpfile.autoclose = false + tmpfile = File.new(tmpfile.fileno, mode: File::RDWR, path: path) + end + if block + begin + yield tmpfile + ensure + tmpfile.close + end + else + tmpfile + end +end +end @@ -746,6 +746,7 @@ load_iseq_eval(rb_execution_context_t *ec, VALUE fname) if (*rb_ruby_prism_ptr()) { pm_parse_result_t result = { 0 }; result.options.line = 1; + result.node.coverage_enabled = 1; VALUE error = pm_load_parse_file(&result, fname); @@ -1283,7 +1284,7 @@ require_internal(rb_execution_context_t *ec, VALUE fname, int exception, bool wa else { switch (found) { case 'r': - load_iseq_eval(ec, path); + load_iseq_eval(saved.ec, path); break; case 's': diff --git a/man/ruby.1 b/man/ruby.1 index 3cfe605ce9..905c712f4c 100644 --- a/man/ruby.1 +++ b/man/ruby.1 @@ -430,7 +430,7 @@ Dump some information. Prints the specified target. .Ar target can be one of: -.Bl -hang -offset indent +.Bl -hang -offset indent -width "version" .It Sy version Print version description (same as .Fl -version). @@ -448,18 +448,20 @@ Check syntax (same as .El .Pp Or one of the following, which are intended for debugging the interpreter: -.Bl -hang -offset indent -tag -width "parsetree_with_comment" +.Bl -hang -offset indent -width "parsetree" .It Sy yydebug Enable compiler debug mode (same as .Fl -yydebug). .It Sy parsetree Print a textual representation of the Ruby AST for the program. -.It Sy parsetree_with_comment -Print a textual representation of the Ruby AST for the program, but with each node annotated with the associated Ruby source code. .It Sy insns Print a list of disassembled bytecode instructions. -.It Sy insns_without_opt -Print the list of disassembled bytecode instructions before various optimizations have been applied. +.It Sy -optimize +Disable various optimizations to print a list disassembled bytecode instructions. +.It Sy +error-tolerant +Enable error-tolerant parsing, when yydebug or parsetree. +.It Sy +comment +Annotate a textual representation of the Ruby AST for the program with the associated Ruby source code. .El .Pp .It Fl -verbose @@ -515,6 +517,32 @@ enabled for only mswin32, mingw32, and OS/2 platforms. If this variable is not defined, Ruby refers to .Ev COMSPEC . .Pp +.It Ev RUBY_FREE_AT_EXIT +If set, Ruby tries to free all dynamically allocated memories. +Introduced in Ruby 3.3, default: unset. +.Pp +.It Ev RUBY_IO_BUFFER_DEFAULT_SIZE +The custom default buffer size of +.Li IO::Buffer . +.Pp +.It Ev RUBY_MAX_CPU +The maximum number of native threads used by M:N Threads scheduler +Introduced in Ruby 3.3, default: 8. +.Pp +.It Ev RUBY_MN_THREADS +If set to +.Li 1 , +M:N Thread scheduler is enabled on the main Ractor. +Introduced in Ruby 3.3, default: unset. +.Pp +.It Ev RUBY_PAGER +The pager command that will be used for +.Fl -help +option. +Introduced in Ruby 3.0, default: +.Ev PAGER +environment variable. +.Pp .It Ev PATH Ruby refers to the .Ev PATH @@ -559,14 +587,15 @@ Reaching the old malloc limit. .El .Pp There are currently 4 possible areas where the GC may be tuned by -the following 11 environment variables: -.Bl -hang -compact -width "RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR" -.It Ev RUBY_GC_HEAP_INIT_SLOTS -Initial allocation slots. Applies to all slot sizes. Introduced in Ruby 2.1, default: 10000. +the following environment variables: +.Bl -hang -compact -width "RUBY_GC_HEAP_n_INIT_SLOTS" .Pp -.It Ev RUBY_GC_HEAP_%d_INIT_SLOTS +.It Ev RUBY_GC_HEAP_ Ns Ar n Ns Ev _INIT_SLOTS Initial allocation of slots in a specific heap. -The available heaps can be found in the keys of `GC.stat_heap`. +The available heaps can be found in the keys of +.Li GC.stat_heap . +.Ar n +is a decimal number between 0 and 4. Introduced in Ruby 3.3. .Pp .It Ev RUBY_GC_HEAP_FREE_SLOTS @@ -589,6 +618,12 @@ where R is this factor and N is the number of old objects after the last full GC. Introduced in Ruby 2.1.1, default: 2.0 .Pp +.It Ev RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO +Used to calculate the +.Li remembered_wb_unprotected_objects_limit +using a ratio of +.Li old_objects . +Introduced in Ruby 3.3, default: 0.1, minimum: 0.0 .It Ev RUBY_GC_MALLOC_LIMIT The initial limit of young generation allocation from the malloc-family. GC will start when this limit is reached. @@ -605,6 +640,31 @@ GC frequency but increasing malloc growth until RUBY_GC_MALLOC_LIMIT_MAX is reached. Introduced in Ruby 2.1, default: 1.4, minimum: 1.0 (no growth) .Pp +.It Ev RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO +Allocate additional pages when the number of free slots is +lower than the value +.Li (total_slots * (this ratio)) . +Introduced in Ruby 2.4, default: 0.2, minimum: 0.0, maximum: 1.0 +.Pp +.It Ev RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO +Allow to free pages when the number of free slots is greater than the value +.Li (total_slots * (this ratio)) . +Introduced in Ruby 2.4, default: 0.4, minimum: +.Li RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO, +maximum: 1.0 +.Pp +.It Ev RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO +Allocate slots to satisfy this formula: +.Li free_slots = total_slots * goal_ratio +In other words, prepare +.Li (total_slots * goal_ratio) +free slots. +if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly. +Introduced in Ruby 2.4, default: 0.65, minimum: +.Li RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO, +maximum: +.Li RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO +.Pp .It Ev RUBY_GC_OLDMALLOC_LIMIT The initial limit of old generation allocation from malloc, a full GC will start when this limit is reached. @@ -622,6 +682,11 @@ GC frequency but increasing malloc growth until RUBY_GC_OLDMALLOC_LIMIT_MAX is reached. Introduced in Ruby 2.1, default: 1.2, minimum: 1.0 (no growth) .Pp +.It Ev RUBY_SHARED_FIBER_POOL_FREE_STACKS +Frees stacks of pooled fibers, if set to 1. +Do not free the stacks if set to 0. +Introduced in Ruby 2.7, default: 1 (no growth) +.Pp .El .Sh STACK SIZE ENVIRONMENT Stack size environment variables are implementation-dependent and @@ -128,22 +128,6 @@ static VALUE compat_allocator_tbl_wrapper; static VALUE rb_marshal_dump_limited(VALUE obj, VALUE port, int limit); static VALUE rb_marshal_load_with_proc(VALUE port, VALUE proc, bool freeze); -static int -mark_marshal_compat_i(st_data_t key, st_data_t value, st_data_t _) -{ - marshal_compat_t *p = (marshal_compat_t *)value; - rb_gc_mark(p->newclass); - rb_gc_mark(p->oldclass); - return ST_CONTINUE; -} - -static void -mark_marshal_compat_t(void *tbl) -{ - if (!tbl) return; - st_foreach(tbl, mark_marshal_compat_i, 0); -} - static st_table *compat_allocator_table(void); void @@ -156,11 +140,10 @@ rb_marshal_define_compat(VALUE newclass, VALUE oldclass, VALUE (*dumper)(VALUE), rb_raise(rb_eTypeError, "no allocator"); } + compat_allocator_table(); compat = ALLOC(marshal_compat_t); - compat->newclass = Qnil; - compat->oldclass = Qnil; - compat->newclass = newclass; - compat->oldclass = oldclass; + RB_OBJ_WRITE(compat_allocator_tbl_wrapper, &compat->newclass, newclass); + RB_OBJ_WRITE(compat_allocator_tbl_wrapper, &compat->oldclass, oldclass); compat->dumper = dumper; compat->loader = loader; @@ -1696,6 +1679,11 @@ r_copy_ivar(VALUE v, VALUE data) return v; } +#define override_ivar_error(type, str) \ + rb_raise(rb_eTypeError, \ + "can't override instance variable of "type" '%"PRIsVALUE"'", \ + (str)) + static void r_ivar(VALUE obj, int *has_encoding, struct load_arg *arg) { @@ -1703,6 +1691,12 @@ r_ivar(VALUE obj, int *has_encoding, struct load_arg *arg) len = r_long(arg); if (len > 0) { + if (RB_TYPE_P(obj, T_MODULE)) { + override_ivar_error("module", rb_mod_name(obj)); + } + else if (RB_TYPE_P(obj, T_CLASS)) { + override_ivar_error("class", rb_class_name(obj)); + } do { VALUE sym = r_symbol(arg); VALUE val = r_object(arg); @@ -1795,9 +1789,7 @@ append_extmod(VALUE obj, VALUE extmod) #define prohibit_ivar(type, str) do { \ if (!ivp || !*ivp) break; \ - rb_raise(rb_eTypeError, \ - "can't override instance variable of "type" '%"PRIsVALUE"'", \ - (str)); \ + override_ivar_error(type, str); \ } while (0) static VALUE r_object_for(struct load_arg *arg, bool partial, int *ivp, VALUE extmod, int type); @@ -2516,28 +2508,75 @@ Init_marshal(void) } static int -free_compat_i(st_data_t key, st_data_t value, st_data_t _) +marshal_compat_table_mark_i(st_data_t key, st_data_t value, st_data_t _) +{ + marshal_compat_t *p = (marshal_compat_t *)value; + rb_gc_mark_movable(p->newclass); + rb_gc_mark_movable(p->oldclass); + return ST_CONTINUE; +} + +static void +marshal_compat_table_mark(void *tbl) +{ + if (!tbl) return; + st_foreach(tbl, marshal_compat_table_mark_i, 0); +} + +static int +marshal_compat_table_free_i(st_data_t key, st_data_t value, st_data_t _) { xfree((marshal_compat_t *)value); return ST_CONTINUE; } static void -free_compat_allocator_table(void *data) +marshal_compat_table_free(void *data) { - st_foreach(data, free_compat_i, 0); + st_foreach(data, marshal_compat_table_free_i, 0); st_free_table(data); } +static size_t +marshal_compat_table_memsize(const void *data) +{ + return st_memsize(data) + sizeof(marshal_compat_t) * st_table_size(data); +} + +static int +marshal_compat_table_compact_i(st_data_t key, st_data_t value, st_data_t _) +{ + marshal_compat_t *p = (marshal_compat_t *)value; + p->newclass = rb_gc_location(p->newclass); + p->oldclass = rb_gc_location(p->oldclass); + return ST_CONTINUE; +} + +static void +marshal_compat_table_compact(void *tbl) +{ + if (!tbl) return; + st_foreach(tbl, marshal_compat_table_compact_i, 0); +} + +static const rb_data_type_t marshal_compat_type = { + .wrap_struct_name = "marshal_compat_table", + .function = { + .dmark = marshal_compat_table_mark, + .dfree = marshal_compat_table_free, + .dsize = marshal_compat_table_memsize, + .dcompact = marshal_compat_table_compact, + }, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY, +}; + static st_table * compat_allocator_table(void) { if (compat_allocator_tbl) return compat_allocator_tbl; compat_allocator_tbl = st_init_numtable(); -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 compat_allocator_tbl_wrapper = - Data_Wrap_Struct(0, mark_marshal_compat_t, free_compat_allocator_table, compat_allocator_tbl); + TypedData_Wrap_Struct(0, &marshal_compat_type, compat_allocator_tbl); rb_vm_register_global_object(compat_allocator_tbl_wrapper); return compat_allocator_tbl; } diff --git a/mini_builtin.c b/mini_builtin.c index 810125fa2e..2024d5d4a6 100644 --- a/mini_builtin.c +++ b/mini_builtin.c @@ -12,22 +12,16 @@ static struct st_table *loaded_builtin_table; #endif +bool pm_builtin_ast_value(pm_parse_result_t *result, const char *feature_name, VALUE *name_str); VALUE rb_builtin_ast_value(const char *feature_name, VALUE *name_str); static const rb_iseq_t * builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *table) { VALUE name_str = 0; - rb_ast_t *ast; - VALUE ast_value = rb_builtin_ast_value(feature_name, &name_str); - rb_vm_t *vm = GET_VM(); + const rb_iseq_t *iseq; - if (NIL_P(ast_value)) { - rb_fatal("builtin_iseq_load: can not find %s; " - "probably miniprelude.c is out of date", - feature_name); - } - vm->builtin_function_table = table; + rb_vm_t *vm = GET_VM(); static const rb_compile_option_t optimization = { .inline_const_cache = TRUE, .peephole_optimization = TRUE, @@ -40,11 +34,38 @@ builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *ta .coverage_enabled = FALSE, .debug_level = 0, }; - ast = rb_ruby_ast_data_get(ast_value); - const rb_iseq_t *iseq = rb_iseq_new_with_opt(ast_value, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil); - GET_VM()->builtin_function_table = NULL; - rb_ast_dispose(ast); + if (*rb_ruby_prism_ptr()) { + pm_parse_result_t result = { 0 }; + if (!pm_builtin_ast_value(&result, feature_name, &name_str)) { + rb_fatal("builtin_iseq_load: can not find %s; " + "probably miniprelude.c is out of date", + feature_name); + } + + vm->builtin_function_table = table; + iseq = pm_iseq_new_with_opt(&result.node, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization); + + GET_VM()->builtin_function_table = NULL; + pm_parse_result_free(&result); + } + else { + VALUE ast_value = rb_builtin_ast_value(feature_name, &name_str); + + if (NIL_P(ast_value)) { + rb_fatal("builtin_iseq_load: can not find %s; " + "probably miniprelude.c is out of date", + feature_name); + } + + rb_ast_t *ast = rb_ruby_ast_data_get(ast_value); + + vm->builtin_function_table = table; + iseq = rb_iseq_new_with_opt(ast_value, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil); + + GET_VM()->builtin_function_table = NULL; + rb_ast_dispose(ast); + } // for debug if (0 && strcmp("prelude", feature_name) == 0) { diff --git a/misc/lldb_rb/utils.py b/misc/lldb_rb/utils.py index 86b5bdda2d..a6bbd385cd 100644 --- a/misc/lldb_rb/utils.py +++ b/misc/lldb_rb/utils.py @@ -60,6 +60,9 @@ class RbInspector(LLDBInterface): rbUndef = self.ruby_globals["RUBY_Qundef"] rbImmediateMask = self.ruby_globals["RUBY_IMMEDIATE_MASK"] + if self.inspect_node(val): + return + num = val.GetValueAsSigned() if num == rbFalse: print('false', file=self.result) @@ -135,6 +138,16 @@ class RbInspector(LLDBInterface): else: self.result.write('[enc=%d] ' % encidx) + coderange = rval.flags & self.ruby_globals["RUBY_ENC_CODERANGE_MASK"] + if coderange == self.ruby_globals["RUBY_ENC_CODERANGE_7BIT"]: + self.result.write('[7BIT] ') + elif coderange == self.ruby_globals["RUBY_ENC_CODERANGE_VALID"]: + self.result.write('[VALID] ') + elif coderange == self.ruby_globals["RUBY_ENC_CODERANGE_BROKEN"]: + self.result.write('[BROKEN] ') + else: + self.result.write('[UNKNOWN] ') + ptr, len = self.string2cstr(val.Cast(tRString)) if len == 0: self.result.write("(empty)\n") @@ -245,227 +258,6 @@ class RbInspector(LLDBInterface): print("T_DATA:", file=self.result) self._append_expression("*(struct RData *) %0#x" % val.GetValueAsUnsigned()) - elif rval.is_type("RUBY_T_NODE"): - tRNode = self.target.FindFirstType("struct RNode").GetPointerType() - rbNodeTypeMask = self.ruby_globals["RUBY_NODE_TYPEMASK"] - rbNodeTypeShift = self.ruby_globals["RUBY_NODE_TYPESHIFT"] - - nd_type = (rval.flags & rbNodeTypeMask) >> rbNodeTypeShift - val = val.Cast(tRNode) - - self._append_expression("(node_type) %d" % nd_type) - - if nd_type == self.ruby_globals["NODE_SCOPE"]: - self._append_expression("*(struct RNode_SCOPE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BLOCK"]: - self._append_expression("*(struct RNode_BLOCK *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IF"]: - self._append_expression("*(struct RNode_IF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNLESS"]: - self._append_expression("*(struct RNode_UNLESS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE"]: - self._append_expression("*(struct RNode_CASE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE2"]: - self._append_expression("*(struct RNode_CASE2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE3"]: - self._append_expression("*(struct RNode_CASE3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_WHEN"]: - self._append_expression("*(struct RNode_WHEN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IN"]: - self._append_expression("*(struct RNode_IN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_WHILE"]: - self._append_expression("*(struct RNode_WHILE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNTIL"]: - self._append_expression("*(struct RNode_UNTIL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ITER"]: - self._append_expression("*(struct RNode_ITER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FOR"]: - self._append_expression("*(struct RNode_FOR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FOR_MASGN"]: - self._append_expression("*(struct RNode_FOR_MASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BREAK"]: - self._append_expression("*(struct RNode_BREAK *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NEXT"]: - self._append_expression("*(struct RNode_NEXT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_REDO"]: - self._append_expression("*(struct RNode_REDO *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RETRY"]: - self._append_expression("*(struct RNode_RETRY *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BEGIN"]: - self._append_expression("*(struct RNode_BEGIN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RESCUE"]: - self._append_expression("*(struct RNode_RESCUE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RESBODY"]: - self._append_expression("*(struct RNode_RESBODY *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ENSURE"]: - self._append_expression("*(struct RNode_ENSURE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_AND"]: - self._append_expression("*(struct RNode_AND *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OR"]: - self._append_expression("*(struct RNode_OR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MASGN"]: - self._append_expression("*(struct RNode_MASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LASGN"]: - self._append_expression("*(struct RNode_LASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DASGN"]: - self._append_expression("*(struct RNode_DASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_GASGN"]: - self._append_expression("*(struct RNode_GASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IASGN"]: - self._append_expression("*(struct RNode_IASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CDECL"]: - self._append_expression("*(struct RNode_CDECL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CVASGN"]: - self._append_expression("*(struct RNode_CVASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN1"]: - self._append_expression("*(struct RNode_OP_ASGN1 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN2"]: - self._append_expression("*(struct RNode_OP_ASGN2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN_AND"]: - self._append_expression("*(struct RNode_OP_ASGN_AND *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN_OR"]: - self._append_expression("*(struct RNode_OP_ASGN_OR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_CDECL"]: - self._append_expression("*(struct RNode_OP_CDECL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CALL"]: - self._append_expression("*(struct RNode_CALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OPCALL"]: - self._append_expression("*(struct RNode_OPCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FCALL"]: - self._append_expression("*(struct RNode_FCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_VCALL"]: - self._append_expression("*(struct RNode_VCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_QCALL"]: - self._append_expression("*(struct RNode_QCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SUPER"]: - self._append_expression("*(struct RNode_SUPER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ZSUPER"]: - self._append_expression("*(struct RNode_ZSUPER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LIST"]: - self._append_expression("*(struct RNode_LIST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ZLIST"]: - self._append_expression("*(struct RNode_ZLIST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_HASH"]: - self._append_expression("*(struct RNode_HASH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RETURN"]: - self._append_expression("*(struct RNode_RETURN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_YIELD"]: - self._append_expression("*(struct RNode_YIELD *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LVAR"]: - self._append_expression("*(struct RNode_LVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DVAR"]: - self._append_expression("*(struct RNode_DVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_GVAR"]: - self._append_expression("*(struct RNode_GVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CONST"]: - self._append_expression("*(struct RNode_CONST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CVAR"]: - self._append_expression("*(struct RNode_CVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NTH_REF"]: - self._append_expression("*(struct RNode_NTH_REF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BACK_REF"]: - self._append_expression("*(struct RNode_BACK_REF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH"]: - self._append_expression("*(struct RNode_MATCH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH2"]: - self._append_expression("*(struct RNode_MATCH2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH3"]: - self._append_expression("*(struct RNode_MATCH3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_STR"]: - self._append_expression("*(struct RNode_STR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DSTR"]: - self._append_expression("*(struct RNode_DSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_XSTR"]: - self._append_expression("*(struct RNode_XSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DXSTR"]: - self._append_expression("*(struct RNode_DXSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_EVSTR"]: - self._append_expression("*(struct RNode_EVSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_REGX"]: - self._append_expression("*(struct RNode_REGX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DREGX"]: - self._append_expression("*(struct RNode_DREGX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ONCE"]: - self._append_expression("*(struct RNode_ONCE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGS"]: - self._append_expression("*(struct RNode_ARGS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGS_AUX"]: - self._append_expression("*(struct RNode_ARGS_AUX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OPT_ARG"]: - self._append_expression("*(struct RNode_OPT_ARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_KW_ARG"]: - self._append_expression("*(struct RNode_KW_ARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_POSTARG"]: - self._append_expression("*(struct RNode_POSTARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGSCAT"]: - self._append_expression("*(struct RNode_ARGSCAT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGSPUSH"]: - self._append_expression("*(struct RNode_ARGSPUSH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SPLAT"]: - self._append_expression("*(struct RNode_SPLAT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFN"]: - self._append_expression("*(struct RNode_DEFN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFS"]: - self._append_expression("*(struct RNode_DEFS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ALIAS"]: - self._append_expression("*(struct RNode_ALIAS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_VALIAS"]: - self._append_expression("*(struct RNode_VALIAS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNDEF"]: - self._append_expression("*(struct RNode_UNDEF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CLASS"]: - self._append_expression("*(struct RNode_CLASS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MODULE"]: - self._append_expression("*(struct RNode_MODULE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SCLASS"]: - self._append_expression("*(struct RNode_SCLASS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_COLON2"]: - self._append_expression("*(struct RNode_COLON2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_COLON3"]: - self._append_expression("*(struct RNode_COLON3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DOT2"]: - self._append_expression("*(struct RNode_DOT2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DOT3"]: - self._append_expression("*(struct RNode_DOT3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FLIP2"]: - self._append_expression("*(struct RNode_FLIP2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FLIP3"]: - self._append_expression("*(struct RNode_FLIP3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SELF"]: - self._append_expression("*(struct RNode_SELF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NIL"]: - self._append_expression("*(struct RNode_NIL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_TRUE"]: - self._append_expression("*(struct RNode_TRUE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FALSE"]: - self._append_expression("*(struct RNode_FALSE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ERRINFO"]: - self._append_expression("*(struct RNode_ERRINFO *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFINED"]: - self._append_expression("*(struct RNode_DEFINED *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_POSTEXE"]: - self._append_expression("*(struct RNode_POSTEXE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DSYM"]: - self._append_expression("*(struct RNode_DSYM *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ATTRASGN"]: - self._append_expression("*(struct RNode_ATTRASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LAMBDA"]: - self._append_expression("*(struct RNode_LAMBDA *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARYPTN"]: - self._append_expression("*(struct RNode_ARYPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_HSHPTN"]: - self._append_expression("*(struct RNode_HSHPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FNDPTN"]: - self._append_expression("*(struct RNode_FNDPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ERROR"]: - self._append_expression("*(struct RNode_ERROR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LINE"]: - self._append_expression("*(struct RNode_LINE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FILE"]: - self._append_expression("*(struct RNode_FILE *) %0#x" % val.GetValueAsUnsigned()) - else: - self._append_expression("*(struct RNode *) %0#x" % val.GetValueAsUnsigned()) - elif rval.is_type("RUBY_T_IMEMO"): imemo_type = ((rval.flags >> self.ruby_globals["RUBY_FL_USHIFT"]) & IMEMO_MASK) @@ -492,3 +284,230 @@ class RbInspector(LLDBInterface): else: print("Not-handled type %0#x" % rval.type, file=self.result) print(val, file=self.result) + + def inspect_node(self, val): + tRNode = self.target.FindFirstType("struct RNode").GetPointerType() + + # if val.GetType() != tRNode: does not work for unknown reason + + if val.GetType().GetPointeeType().name != "NODE": + return False + + rbNodeTypeMask = self.ruby_globals["RUBY_NODE_TYPEMASK"] + rbNodeTypeShift = self.ruby_globals["RUBY_NODE_TYPESHIFT"] + flags = val.Cast(tRNode).GetChildMemberWithName("flags").GetValueAsUnsigned() + nd_type = (flags & rbNodeTypeMask) >> rbNodeTypeShift + + self._append_expression("(node_type) %d" % nd_type) + + if nd_type == self.ruby_globals["NODE_SCOPE"]: + self._append_expression("*(rb_node_scope_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BLOCK"]: + self._append_expression("*(rb_node_block_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IF"]: + self._append_expression("*(rb_node_if_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNLESS"]: + self._append_expression("*(rb_node_unless_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE"]: + self._append_expression("*(rb_node_case_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE2"]: + self._append_expression("*(rb_node_case2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE3"]: + self._append_expression("*(rb_node_case3_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_WHEN"]: + self._append_expression("*(rb_node_when_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IN"]: + self._append_expression("*(rb_node_in_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_WHILE"]: + self._append_expression("*(rb_node_while_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNTIL"]: + self._append_expression("*(rb_node_until_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ITER"]: + self._append_expression("*(rb_node_iter_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FOR"]: + self._append_expression("*(rb_node_for_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FOR_MASGN"]: + self._append_expression("*(rb_node_for_masgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BREAK"]: + self._append_expression("*(rb_node_break_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NEXT"]: + self._append_expression("*(rb_node_next_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_REDO"]: + self._append_expression("*(rb_node_redo_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RETRY"]: + self._append_expression("*(rb_node_retry_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BEGIN"]: + self._append_expression("*(rb_node_begin_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RESCUE"]: + self._append_expression("*(rb_node_rescue_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RESBODY"]: + self._append_expression("*(rb_node_resbody_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ENSURE"]: + self._append_expression("*(rb_node_ensure_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_AND"]: + self._append_expression("*(rb_node_and_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OR"]: + self._append_expression("*(rb_node_or_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MASGN"]: + self._append_expression("*(rb_node_masgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LASGN"]: + self._append_expression("*(rb_node_lasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DASGN"]: + self._append_expression("*(rb_node_dasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_GASGN"]: + self._append_expression("*(rb_node_gasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IASGN"]: + self._append_expression("*(rb_node_iasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CDECL"]: + self._append_expression("*(rb_node_cdecl_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CVASGN"]: + self._append_expression("*(rb_node_cvasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN1"]: + self._append_expression("*(rb_node_op_asgn1_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN2"]: + self._append_expression("*(rb_node_op_asgn2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN_AND"]: + self._append_expression("*(rb_node_op_asgn_and_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN_OR"]: + self._append_expression("*(rb_node_op_asgn_or_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_CDECL"]: + self._append_expression("*(rb_node_op_cdecl_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CALL"]: + self._append_expression("*(rb_node_call_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OPCALL"]: + self._append_expression("*(rb_node_opcall_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FCALL"]: + self._append_expression("*(rb_node_fcall_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_VCALL"]: + self._append_expression("*(rb_node_vcall_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_QCALL"]: + self._append_expression("*(rb_node_qcall_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SUPER"]: + self._append_expression("*(rb_node_super_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ZSUPER"]: + self._append_expression("*(rb_node_zsuper_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LIST"]: + self._append_expression("*(rb_node_list_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ZLIST"]: + self._append_expression("*(rb_node_zlist_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_HASH"]: + self._append_expression("*(rb_node_hash_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RETURN"]: + self._append_expression("*(rb_node_return_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_YIELD"]: + self._append_expression("*(rb_node_yield_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LVAR"]: + self._append_expression("*(rb_node_lvar_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DVAR"]: + self._append_expression("*(rb_node_dvar_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_GVAR"]: + self._append_expression("*(rb_node_gvar_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CONST"]: + self._append_expression("*(rb_node_const_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CVAR"]: + self._append_expression("*(rb_node_cvar_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NTH_REF"]: + self._append_expression("*(rb_node_nth_ref_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BACK_REF"]: + self._append_expression("*(rb_node_back_ref_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH"]: + self._append_expression("*(rb_node_match_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH2"]: + self._append_expression("*(rb_node_match2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH3"]: + self._append_expression("*(rb_node_match3_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_STR"]: + self._append_expression("*(rb_node_str_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DSTR"]: + self._append_expression("*(rb_node_dstr_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_XSTR"]: + self._append_expression("*(rb_node_xstr_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DXSTR"]: + self._append_expression("*(rb_node_dxstr_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_EVSTR"]: + self._append_expression("*(rb_node_evstr_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_REGX"]: + self._append_expression("*(rb_node_regx_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DREGX"]: + self._append_expression("*(rb_node_dregx_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ONCE"]: + self._append_expression("*(rb_node_once_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGS"]: + self._append_expression("*(rb_node_args_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGS_AUX"]: + self._append_expression("*(rb_node_args_aux_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OPT_ARG"]: + self._append_expression("*(rb_node_opt_arg_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_KW_ARG"]: + self._append_expression("*(rb_node_kw_arg_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_POSTARG"]: + self._append_expression("*(rb_node_postarg_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGSCAT"]: + self._append_expression("*(rb_node_argscat_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGSPUSH"]: + self._append_expression("*(rb_node_argspush_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SPLAT"]: + self._append_expression("*(rb_node_splat_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFN"]: + self._append_expression("*(rb_node_defn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFS"]: + self._append_expression("*(rb_node_defs_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ALIAS"]: + self._append_expression("*(rb_node_alias_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_VALIAS"]: + self._append_expression("*(rb_node_valias_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNDEF"]: + self._append_expression("*(rb_node_undef_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CLASS"]: + self._append_expression("*(rb_node_class_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MODULE"]: + self._append_expression("*(rb_node_module_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SCLASS"]: + self._append_expression("*(rb_node_sclass_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_COLON2"]: + self._append_expression("*(rb_node_colon2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_COLON3"]: + self._append_expression("*(rb_node_colon3_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DOT2"]: + self._append_expression("*(rb_node_dot2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DOT3"]: + self._append_expression("*(rb_node_dot3_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FLIP2"]: + self._append_expression("*(rb_node_flip2_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FLIP3"]: + self._append_expression("*(rb_node_flip3_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SELF"]: + self._append_expression("*(rb_node_self_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NIL"]: + self._append_expression("*(rb_node_nil_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_TRUE"]: + self._append_expression("*(rb_node_true_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FALSE"]: + self._append_expression("*(rb_node_false_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ERRINFO"]: + self._append_expression("*(rb_node_errinfo_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFINED"]: + self._append_expression("*(rb_node_defined_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_POSTEXE"]: + self._append_expression("*(rb_node_postexe_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DSYM"]: + self._append_expression("*(rb_node_dsym_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ATTRASGN"]: + self._append_expression("*(rb_node_attrasgn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LAMBDA"]: + self._append_expression("*(rb_node_lambda_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARYPTN"]: + self._append_expression("*(rb_node_aryptn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_HSHPTN"]: + self._append_expression("*(rb_node_hshptn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FNDPTN"]: + self._append_expression("*(rb_node_fndptn_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ERROR"]: + self._append_expression("*(rb_node_error_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LINE"]: + self._append_expression("*(rb_node_line_t *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FILE"]: + self._append_expression("*(rb_node_file_t *) %0#x" % val.GetValueAsUnsigned()) + else: + self._append_expression("*(NODE *) %0#x" % val.GetValueAsUnsigned()) + return True @@ -782,6 +782,12 @@ pack_pack(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) return res; } +VALUE +rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) +{ + return pack_pack(ec, ary, fmt, buffer); +} + static const char uu_table[] = "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"; static const char b64_table[] = @@ -83,8 +83,14 @@ static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, con #define compile_callback rb_suppress_tracing #endif /* !UNIVERSAL_PARSER */ +#define NODE_SPECIAL_EMPTY_ARGS ((NODE *)-1) +#define NODE_EMPTY_ARGS_P(node) ((node) == NODE_SPECIAL_EMPTY_ARGS) + static int rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2); + +#ifndef RIPPER static rb_parser_string_t *rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *original); +#endif static int node_integer_cmp(rb_node_integer_t *n1, rb_node_integer_t *n2) @@ -294,7 +300,6 @@ parse_isxdigit(int c) #define STRNCASECMP rb_parser_st_locale_insensitive_strncasecmp #ifdef RIPPER -VALUE rb_ripper_none; #include "ripper_init.h" #endif @@ -517,7 +522,7 @@ struct parser_params { int line_count; int ruby_sourceline; /* current line no. */ const char *ruby_sourcefile; /* current source file */ - rb_parser_string_t *ruby_sourcefile_string; + VALUE ruby_sourcefile_string; rb_encoding *enc; token_info *token_info; st_table *case_labels; @@ -534,8 +539,6 @@ struct parser_params { int end_col; } delayed; - ID cur_arg; - rb_ast_t *ast; int node_id; @@ -634,10 +637,10 @@ static void after_reduce(int len, struct parser_params *p) { for (int i = 0; i < len; i++) { + VALUE tos = rb_ary_pop(p->s_value_stack); if (p->debug) { - rb_parser_printf(p, "after-reduce pop: %+"PRIsVALUE"\n", rb_ary_entry(p->s_value_stack, -1)); + rb_parser_printf(p, "after-reduce pop: %+"PRIsVALUE"\n", tos); } - rb_ary_pop(p->s_value_stack); } if (p->debug) { rb_parser_printf(p, "after-reduce push: %+"PRIsVALUE"\n", p->s_lvalue); @@ -659,10 +662,10 @@ static void after_pop_stack(int len, struct parser_params *p) { for (int i = 0; i < len; i++) { + VALUE tos = rb_ary_pop(p->s_value_stack); if (p->debug) { - rb_parser_printf(p, "after-pop-stack pop: %+"PRIsVALUE"\n", rb_ary_entry(p->s_value_stack, -1)); + rb_parser_printf(p, "after-pop-stack pop: %+"PRIsVALUE"\n", tos); } - rb_ary_pop(p->s_value_stack); } } #else @@ -1158,7 +1161,7 @@ static rb_node_aryptn_t *rb_node_aryptn_new(struct parser_params *p, NODE *pre_a static rb_node_hshptn_t *rb_node_hshptn_new(struct parser_params *p, NODE *nd_pconst, NODE *nd_pkwargs, NODE *nd_pkwrestarg, const YYLTYPE *loc); static rb_node_fndptn_t *rb_node_fndptn_new(struct parser_params *p, NODE *pre_rest_arg, NODE *args, NODE *post_rest_arg, const YYLTYPE *loc); static rb_node_line_t *rb_node_line_new(struct parser_params *p, const YYLTYPE *loc); -static rb_node_file_t *rb_node_file_new(struct parser_params *p, rb_parser_string_t *str, const YYLTYPE *loc); +static rb_node_file_t *rb_node_file_new(struct parser_params *p, VALUE str, const YYLTYPE *loc); static rb_node_error_t *rb_node_error_new(struct parser_params *p, const YYLTYPE *loc); #define NEW_SCOPE(a,b,loc) (NODE *)rb_node_scope_new(p,a,b,loc) @@ -1300,7 +1303,6 @@ struct RNode_DEF_TEMP { ID nd_mid; struct { - ID cur_arg; int max_numparam; NODE *numparam_save; struct lex_context ctxt; @@ -1446,7 +1448,7 @@ static NODE *assignable(struct parser_params*,ID,NODE*,const YYLTYPE*); static NODE *aryset(struct parser_params*,NODE*,NODE*,const YYLTYPE*); static NODE *attrset(struct parser_params*,NODE*,ID,ID,const YYLTYPE*); -static void rb_backref_error(struct parser_params*,NODE*); +static VALUE rb_backref_error(struct parser_params*,NODE*); static NODE *node_assign(struct parser_params*,NODE*,NODE*,struct lex_context,const YYLTYPE*); static NODE *new_op_assign(struct parser_params *p, NODE *lhs, ID op, NODE *rhs, struct lex_context, const YYLTYPE *loc); @@ -1488,14 +1490,8 @@ static NODE *heredoc_dedent(struct parser_params*,NODE*); static void check_literal_when(struct parser_params *p, NODE *args, const YYLTYPE *loc); #ifdef RIPPER -static VALUE var_field(struct parser_params *p, VALUE a); #define get_value(idx) (rb_ary_entry(p->s_value_stack, idx)) #define set_value(val) (p->s_lvalue = val) -static VALUE defs(struct parser_params *p, VALUE head, VALUE args, VALUE bodystmt); -static VALUE backref_error(struct parser_params*, NODE *, VALUE); -static VALUE ripper_assignable(struct parser_params *p, ID id, VALUE lhs); -static VALUE ripper_const_decl(struct parser_params *p, VALUE path); -static VALUE ripper_heredoc_dedent(struct parser_params *p, int indent, VALUE array); static VALUE assign_error(struct parser_params *p, const char *mesg, VALUE a); static int id_is_var(struct parser_params *p, ID id); #endif @@ -1518,7 +1514,7 @@ RUBY_SYMBOL_EXPORT_END static void error_duplicate_pattern_variable(struct parser_params *p, ID id, const YYLTYPE *loc); static void error_duplicate_pattern_key(struct parser_params *p, ID id, const YYLTYPE *loc); -static ID formal_argument(struct parser_params*, ID); +static VALUE formal_argument_error(struct parser_params*, ID); static ID shadowing_lvar(struct parser_params*,ID); static void new_bv(struct parser_params*,ID); @@ -1572,7 +1568,7 @@ static void numparam_pop(struct parser_params *p, NODE *prev_inner); #define CASE_LABELS_ENABLED_P(case_labels) (case_labels && case_labels != CHECK_LITERAL_WHEN) #define yytnamerr(yyres, yystr) (YYSIZE_T)rb_yytnamerr(p, yyres, yystr) -size_t rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr); +RUBY_FUNC_EXPORTED size_t rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr); #define TOKEN2ID(tok) ( \ tTOKEN_LOCAL_BEGIN<(tok)&&(tok)<tTOKEN_LOCAL_END ? TOKEN2LOCALID(tok) : \ @@ -1601,34 +1597,19 @@ static VALUE ripper_dispatch5(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,V static VALUE ripper_dispatch7(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); void ripper_error(struct parser_params *p); -#define dispatch0(n) ripper_dispatch0(p, TOKEN_PASTE(ripper_id_, n)) -#define dispatch1(n,a) ripper_dispatch1(p, TOKEN_PASTE(ripper_id_, n), (a)) -#define dispatch2(n,a,b) ripper_dispatch2(p, TOKEN_PASTE(ripper_id_, n), (a), (b)) -#define dispatch3(n,a,b,c) ripper_dispatch3(p, TOKEN_PASTE(ripper_id_, n), (a), (b), (c)) -#define dispatch4(n,a,b,c,d) ripper_dispatch4(p, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d)) -#define dispatch5(n,a,b,c,d,e) ripper_dispatch5(p, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d), (e)) -#define dispatch7(n,a,b,c,d,e,f,g) ripper_dispatch7(p, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d), (e), (f), (g)) +#define dispatch0(n) ripper_dispatch0(p, RIPPER_ID(n)) +#define dispatch1(n,a) ripper_dispatch1(p, RIPPER_ID(n), (a)) +#define dispatch2(n,a,b) ripper_dispatch2(p, RIPPER_ID(n), (a), (b)) +#define dispatch3(n,a,b,c) ripper_dispatch3(p, RIPPER_ID(n), (a), (b), (c)) +#define dispatch4(n,a,b,c,d) ripper_dispatch4(p, RIPPER_ID(n), (a), (b), (c), (d)) +#define dispatch5(n,a,b,c,d,e) ripper_dispatch5(p, RIPPER_ID(n), (a), (b), (c), (d), (e)) +#define dispatch7(n,a,b,c,d,e,f,g) ripper_dispatch7(p, RIPPER_ID(n), (a), (b), (c), (d), (e), (f), (g)) #define yyparse ripper_yyparse -static void ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs); - -static VALUE -ripper_new_args(struct parser_params *p, VALUE pre_args, VALUE opt_args, VALUE rest_arg, VALUE post_args, VALUE tail) -{ - VALUE kw_args = rb_ary_entry(tail, 0); - VALUE kw_rest_arg = rb_ary_entry(tail, 1); - VALUE block = rb_ary_entry(tail, 2); - return dispatch7(params, pre_args, opt_args, rest_arg, post_args, kw_args, kw_rest_arg, block); -} - static VALUE -ripper_new_array_pattern(struct parser_params *p, VALUE constant, VALUE pre_arg, VALUE aryptn) +aryptn_pre_args(struct parser_params *p, VALUE pre_arg, VALUE pre_args) { - VALUE pre_args = rb_ary_entry(aryptn, 0); - VALUE rest_arg = rb_ary_entry(aryptn, 1); - VALUE post_args = rb_ary_entry(aryptn, 2); - if (!NIL_P(pre_arg)) { if (!NIL_P(pre_args)) { rb_ary_unshift(pre_args, pre_arg); @@ -1637,50 +1618,7 @@ ripper_new_array_pattern(struct parser_params *p, VALUE constant, VALUE pre_arg, pre_args = rb_ary_new_from_args(1, pre_arg); } } - return dispatch4(aryptn, constant, pre_args, rest_arg, post_args); -} - -static VALUE -ripper_new_array_pattern_tail(struct parser_params *p, VALUE pre_args, VALUE rest_arg, VALUE post_args) -{ - return rb_ary_new_from_args(3, pre_args, rest_arg, post_args); -} - -static VALUE -ripper_new_hash_pattern(struct parser_params *p, VALUE constant, VALUE hshptn) -{ - VALUE kw_args = rb_ary_entry(hshptn, 0); - VALUE kw_rest_arg = rb_ary_entry(hshptn, 1); - - return dispatch3(hshptn, constant, kw_args, kw_rest_arg); -} - -static VALUE -ripper_new_hash_pattern_tail(struct parser_params *p, VALUE kw_args, VALUE kw_rest_arg) -{ - if (kw_rest_arg) { - kw_rest_arg = dispatch1(var_field, kw_rest_arg); - } - else { - kw_rest_arg = Qnil; - } - return rb_ary_new_from_args(2, kw_args, kw_rest_arg); -} - -static VALUE -ripper_new_find_pattern(struct parser_params *p, VALUE constant, VALUE fndptn) -{ - VALUE pre_rest_arg = rb_ary_entry(fndptn, 0); - VALUE args = rb_ary_entry(fndptn, 1); - VALUE post_rest_arg = rb_ary_entry(fndptn, 2); - - return dispatch4(fndptn, constant, pre_rest_arg, args, post_rest_arg); -} - -static VALUE -ripper_new_find_pattern_tail(struct parser_params *p, VALUE pre_rest_arg, VALUE args, VALUE post_rest_arg) -{ - return rb_ary_new_from_args(3, pre_rest_arg, args, post_rest_arg); + return pre_args; } #define ID2VAL(id) STATIC_ID2SYM(id) @@ -1727,7 +1665,6 @@ restore_defun(struct parser_params *p, rb_node_def_temp_t *temp) { /* See: def_name action */ struct lex_context ctxt = temp->save.ctxt; - p->cur_arg = temp->save.cur_arg; p->ctxt.in_def = ctxt.in_def; p->ctxt.shareable_constant_value = ctxt.shareable_constant_value; p->ctxt.in_rescue = ctxt.in_rescue; @@ -1769,8 +1706,10 @@ endless_method_name(struct parser_params *p, ID mid, const YYLTYPE *loc) #ifndef RIPPER # define ifndef_ripper(x) (x) +# define ifdef_ripper(r,x) (x) #else # define ifndef_ripper(x) +# define ifdef_ripper(r,x) (r) #endif # define rb_warn0(fmt) WARN_CALL(WARN_ARGS(fmt, 1)) @@ -1853,7 +1792,7 @@ add_block_exit(struct parser_params *p, NODE *node) switch (nd_type(node)) { case NODE_BREAK: case NODE_NEXT: case NODE_REDO: break; default: - compile_error(p, "unexpected node: %s", parser_node_name(nd_type(node))); + compile_error(p, "add_block_exit: unexpected node: %s", parser_node_name(nd_type(node))); return node; } if (!p->ctxt.in_defined) { @@ -1944,7 +1883,7 @@ get_nd_value(struct parser_params *p, NODE *node) case NODE_CDECL: return RNODE_CDECL(node)->nd_value; default: - compile_error(p, "unexpected node: %s", parser_node_name(nd_type(node))); + compile_error(p, "get_nd_value: unexpected node: %s", parser_node_name(nd_type(node))); return 0; } } @@ -1975,7 +1914,7 @@ set_nd_value(struct parser_params *p, NODE *node, NODE *rhs) RNODE_CVASGN(node)->nd_value = rhs; break; default: - compile_error(p, "unexpected node: %s", parser_node_name(nd_type(node))); + compile_error(p, "set_nd_value: unexpected node: %s", parser_node_name(nd_type(node))); break; } } @@ -1997,7 +1936,7 @@ get_nd_vid(struct parser_params *p, NODE *node) case NODE_CVASGN: return RNODE_CVASGN(node)->nd_vid; default: - compile_error(p, "unexpected node: %s", parser_node_name(nd_type(node))); + compile_error(p, "get_nd_vid: unexpected node: %s", parser_node_name(nd_type(node))); return 0; } } @@ -2024,7 +1963,7 @@ get_nd_args(struct parser_params *p, NODE *node) case NODE_NEXT: return 0; default: - compile_error(p, "unexpected node: %s", parser_node_name(nd_type(node))); + compile_error(p, "get_nd_args: unexpected node: %s", parser_node_name(nd_type(node))); return 0; } } @@ -2100,46 +2039,13 @@ rb_parser_encoding_string_new(rb_parser_t *p, const char *ptr, long len, rb_enco } #ifndef RIPPER -static bool -zero_filled(const char *s, int n) -{ - for (; n > 0; --n) { - if (*s++) return false; - } - return true; -} - -static bool -str_null_char(rb_parser_t *p, const char *s, long len, const int minlen, rb_encoding *enc) -{ - const char *e = s + len; - - for (; s + minlen <= e; s += rb_enc_mbclen(s, e, enc)) { - if (zero_filled(s, minlen)) return true; - } - return false; -} - -static bool -cstr_null_check(rb_parser_t *p, const char *s, long len, rb_encoding *enc, int *w) -{ - const int minlen = rb_enc_mbminlen(enc); - - if (minlen > 1) { - *w = 1; - return str_null_char(p, s, len, minlen, enc); - } - else { - *w = 0; - return (!s || memchr(s, 0, len)); - } -} - rb_parser_string_t * rb_str_to_parser_string(rb_parser_t *p, VALUE str) { /* Type check */ - return rb_parser_encoding_string_new(p, RSTRING_PTR(str), RSTRING_LEN(str), rb_enc_get(str)); + rb_parser_string_t *ret = rb_parser_encoding_string_new(p, RSTRING_PTR(str), RSTRING_LEN(str), rb_enc_get(str)); + RB_GC_GUARD(str); + return ret; } #endif @@ -2675,21 +2581,21 @@ rb_parser_ast_token_free(rb_parser_t *p, rb_parser_ast_token_t *token) static void rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) { - void (*free_func)(rb_parser_t *, rb_parser_ary_data) = NULL; +# define foreach_ary(ptr) \ + for (rb_parser_ary_data *ptr = ary->data, *const end_ary_data = ptr + ary->len; \ + ptr < end_ary_data; ptr++) switch (ary->data_type) { case PARSER_ARY_DATA_AST_TOKEN: - free_func = (void (*)(rb_parser_t *, rb_parser_ary_data))rb_parser_ast_token_free; + foreach_ary(data) {rb_parser_ast_token_free(p, *data);} break; case PARSER_ARY_DATA_SCRIPT_LINE: - free_func = (void (*)(rb_parser_t *, rb_parser_ary_data))rb_parser_string_free; + foreach_ary(data) {rb_parser_string_free(p, *data);} break; default: rb_bug("unexpected rb_parser_ary_data_type: %d", ary->data_type); break; } - for (long i = 0; i < ary->len; i++) { - free_func(p, ary->data[i]); - } +# undef foreach_ary xfree(ary); } @@ -2770,6 +2676,7 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) const struct vtable *vars; struct rb_strterm_struct *strterm; struct lex_context ctxt; + enum lex_state_e state; } %token <id> @@ -2934,7 +2841,7 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %token tSTAR "*" %token tDSTAR "**arg" %token tAMPER "&" -%token tLAMBDA "->" +%token <num> tLAMBDA "->" %token tSYMBEG "symbol literal" %token tSTRING_BEG "string literal" %token tXSTRING_BEG "backtick literal" @@ -2945,7 +2852,8 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %token tQSYMBOLS_BEG "verbatim symbol list" %token tSTRING_END "terminator" %token tSTRING_DEND "'}'" -%token tSTRING_DBEG tSTRING_DVAR tLAMBEG tLABEL_END +%token <state> tSTRING_DBEG "'#{'" +%token tSTRING_DVAR tLAMBEG tLABEL_END %token tIGNORED_NL tCOMMENT tEMBDOC_BEG tEMBDOC tEMBDOC_END %token tHEREDOC_BEG tHEREDOC_END k__END__ @@ -2985,46 +2893,45 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) */ %rule f_opt(value) <node_opt_arg>: f_arg_asgn f_eq value { - p->cur_arg = 0; p->ctxt.in_argdef = 1; $$ = NEW_OPT_ARG(assignable(p, $1, $3, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:3)) %*/ + /*% ripper: [$:$, $:3] %*/ } ; %rule f_optarg(value) <node_opt_arg>: f_opt(value) { $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | f_optarg(value) ',' f_opt(value) { $$ = opt_arg_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; %rule f_kwarg(kw) <node_kw_arg>: kw { $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | f_kwarg(kw) ',' kw { $$ = kwd_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; %rule opt_args_tail(tail) <node_args>: ',' tail { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | /* none */ { $$ = new_args_tail(p, 0, 0, 0, &@0); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, Qnil); %*/ + /*% ripper: [Qnil, Qnil, Qnil] %*/ } ; @@ -3093,7 +3000,7 @@ top_stmt : stmt | keyword_BEGIN begin_block { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -3182,7 +3089,7 @@ k_END : keyword_END lex_ctxt { $$ = $2; p->ctxt.in_rescue = before_rescue; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ }; stmt : keyword_alias fitem {SET_LEX_STATE(EXPR_FNAME|EXPR_FITEM);} fitem @@ -3353,12 +3260,8 @@ command_asgn : lhs '=' lex_ctxt command_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFN($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = dispatch3(def, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: def!($:head, $:args, $:$) %*/ local_pop(p); } | defs_head[head] f_opt_paren_args[args] '=' endless_command[bodystmt] @@ -3368,21 +3271,15 @@ command_asgn : lhs '=' lex_ctxt command_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = defs(p, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:$) %*/ local_pop(p); } | backref tOP_ASGN lex_ctxt command_rhs { - /*%%%*/ - rb_backref_error(p, $1); - /*% %*/ + VALUE MAYBE_UNUSED(e) = rb_backref_error(p, $1); $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, RNODE($:1), assign!(var_field(p, get_value($:1)), $:4)) %*/ + /*% ripper[error]: assign_error!(?e, opassign!(var_field!($:1), $:2, $:4)) %*/ } ; @@ -3471,7 +3368,6 @@ def_name : fname ID fname = $1; numparam_name(p, fname); local_push(p, 0); - p->cur_arg = 0; p->ctxt.in_def = 1; p->ctxt.in_rescue = before_rescue; $$ = $1; @@ -3483,7 +3379,7 @@ defn_head : k_def def_name $$ = def_head_save(p, $k_def); $$->nd_mid = $def_name; $$->nd_def = NEW_DEFN($def_name, 0, &@$); - /*% ripper: get_value($:def_name); %*/ + /*% ripper: $:def_name %*/ } ; @@ -3498,10 +3394,7 @@ defs_head : k_def singleton dot_or_colon $$ = def_head_save(p, $k_def); $$->nd_mid = $def_name; $$->nd_def = NEW_DEFS($singleton, $def_name, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:singleton), get_value($:dot_or_colon), get_value($:def_name))); - %*/ + /*% ripper: [$:singleton, $:dot_or_colon, $:def_name] %*/ } ; @@ -3519,7 +3412,7 @@ expr_value : expr expr_value_do : {COND_PUSH(1);} expr_value do {COND_POP();} { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -3539,14 +3432,14 @@ cmd_brace_block : tLBRACE_ARG brace_body '}' { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; fcall : operation { $$ = NEW_FCALL($1, 0, &@$); - /*% ripper: get_value($:1); %*/ + /*% ripper: $:1 %*/ } ; @@ -3613,14 +3506,14 @@ command : fcall command_args %prec tLOWEST { NODE *args = 0; args = ret_args(p, $2); - $<node>$ = add_block_exit(p, NEW_BREAK(args, &@$)); + $$ = add_block_exit(p, NEW_BREAK(args, &@$)); /*% ripper: break!($:2) %*/ } | keyword_next call_args { NODE *args = 0; args = ret_args(p, $2); - $<node>$ = add_block_exit(p, NEW_NEXT(args, &@$)); + $$ = add_block_exit(p, NEW_NEXT(args, &@$)); /*% ripper: next!($:2) %*/ } ; @@ -3644,7 +3537,7 @@ mlhs_inner : mlhs_basic mlhs_basic : mlhs_head { $$ = NEW_MASGN($1, 0, &@$); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | mlhs_head mlhs_item { @@ -3727,13 +3620,13 @@ mlhs_post : mlhs_item mlhs_node : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | primary_value '[' opt_call_args rbracket { @@ -3759,33 +3652,31 @@ mlhs_node : user_variable } | primary_value tCOLON2 tCONSTANT { + /*% ripper: const_path_field!($:1, $:3) %*/ $$ = const_decl(p, NEW_COLON2($1, $3, &@$), &@$); - /*% ripper: ripper_const_decl(p, const_path_field!($:1, $:3)) %*/ } | tCOLON3 tCONSTANT { + /*% ripper: top_const_field!($:2) %*/ $$ = const_decl(p, NEW_COLON3($2, &@$), &@$); - /*% ripper: ripper_const_decl(p, top_const_field!($:2)) %*/ } | backref { - /*%%%*/ - rb_backref_error(p, $1); - /*% %*/ + VALUE MAYBE_UNUSED(e) = rb_backref_error(p, $1); $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, $1, var_field(p, get_value($:1))) %*/ + /*% ripper[error]: assign_error!(?e, var_field!($:1)) %*/ } ; lhs : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | primary_value '[' opt_call_args rbracket { @@ -3809,21 +3700,19 @@ lhs : user_variable } | primary_value tCOLON2 tCONSTANT { + /*% ripper: const_path_field!($:1, $:3) %*/ $$ = const_decl(p, NEW_COLON2($1, $3, &@$), &@$); - /*% ripper: ripper_const_decl(p, const_path_field!($:1, $:3)) %*/ } | tCOLON3 tCONSTANT { + /*% ripper: top_const_field!($:2) %*/ $$ = const_decl(p, NEW_COLON3($2, &@$), &@$); - /*% ripper: ripper_const_decl(p, top_const_field!($:2)) %*/ } | backref { - /*%%%*/ - rb_backref_error(p, $1); - /*% %*/ + VALUE MAYBE_UNUSED(e) = rb_backref_error(p, $1); $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, $1, var_field(p, get_value($:1))) %*/ + /*% ripper[error]: assign_error!(?e, var_field!($:1)) %*/ } ; @@ -3877,13 +3766,13 @@ fitem : fname undef_list : fitem { $$ = NEW_UNDEF($1, &@$); - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | undef_list ',' {SET_LEX_STATE(EXPR_FNAME|EXPR_FITEM);} fitem { NODE *undef = NEW_UNDEF($4, &@4); $$ = block_append(p, $1, undef); - /*% ripper: rb_ary_push(get_value($:1), get_value($:4)) %*/ + /*% ripper: rb_ary_push($:1, $:4) %*/ } ; @@ -3977,11 +3866,9 @@ arg : lhs '=' lex_ctxt arg_rhs } | backref tOP_ASGN lex_ctxt arg_rhs { - rb_backref_error(p, $1); - /*%%%*/ + VALUE MAYBE_UNUSED(e) = rb_backref_error(p, $1); $$ = NEW_ERROR(&@$); - /*% %*/ - /*% ripper[error]: backref_error(p, RNODE($:1), opassign!(var_field(p, get_value($:1)), $:2, $:4)) %*/ + /*% ripper[error]: assign_error!(?e, opassign!(var_field!($:1), $:2, $:4)) %*/ } | arg tDOT2 arg { @@ -4054,12 +3941,7 @@ arg : lhs '=' lex_ctxt arg_rhs | tUMINUS_NUM simple_numeric tPOW arg { $$ = call_uni_op(p, call_bin_op(p, $2, idPow, $4, &@2, &@$), idUMinus, &@1, &@$); - /*%%%*/ - /*% - VALUE val = dispatch3(binary, get_value($:2), ID2VAL(idPow), get_value($:4)); - val = dispatch2(unary, ID2VAL(idUMinus), val); - set_value(val); - %*/ + /*% ripper: unary!(ID2VAL(idUMinus), binary!($:2, ID2VAL(idPow), $:4)) %*/ } | tUPLUS arg { @@ -4167,12 +4049,8 @@ arg : lhs '=' lex_ctxt arg_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFN($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = dispatch3(def, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: def!($:head, $:args, $:$) %*/ local_pop(p); } | defs_head[head] f_opt_paren_args[args] '=' endless_arg[bodystmt] @@ -4182,12 +4060,8 @@ arg : lhs '=' lex_ctxt arg_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = defs(p, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:$) %*/ local_pop(p); } | primary @@ -4316,13 +4190,16 @@ paren_args : '(' opt_call_args rparen opt_paren_args : none | paren_args + { + $$ = $1 ? $1 : NODE_SPECIAL_EMPTY_ARGS; + } ; opt_call_args : none | call_args | args ',' { - $$ = $1; + $$ = $1; } | args ',' assocs ',' { @@ -4397,14 +4274,14 @@ command_args : { CMDARG_POP(); if (lookahead) CMDARG_PUSH(0); $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; block_arg : tAMPER arg_value { $$ = NEW_BLOCK_PASS($2, &@$); - /*% ripper: get_value($:2) %*/ + /*% ripper: $:2 %*/ } | tAMPER { @@ -4417,7 +4294,7 @@ block_arg : tAMPER arg_value opt_block_arg : ',' block_arg { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none { @@ -4453,7 +4330,7 @@ args : arg_value arg_splat : tSTAR arg_value { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | tSTAR /* none */ { @@ -4641,7 +4518,7 @@ primary : literal k_end { if (CASE_LABELS_ENABLED_P(p->case_labels)) st_free_table(p->case_labels); - p->case_labels = $<labels>4; + p->case_labels = $4; $$ = NEW_CASE($2, $5, &@$); fixpos($$, $2); /*% ripper: case!($:2, $:5) %*/ @@ -4655,7 +4532,7 @@ primary : literal k_end { if (p->case_labels) st_free_table(p->case_labels); - p->case_labels = $<labels>3; + p->case_labels = $3; $$ = NEW_CASE2($4, &@$); /*% ripper: case!(Qnil, $:4) %*/ } @@ -4785,22 +4662,22 @@ primary : literal $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*% ripper: defs(p, get_value($:head), get_value($:args), get_value($:bodystmt)) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:bodystmt) %*/ local_pop(p); } | keyword_break { - $<node>$ = add_block_exit(p, NEW_BREAK(0, &@$)); + $$ = add_block_exit(p, NEW_BREAK(0, &@$)); /*% ripper: break!(args_new!) %*/ } | keyword_next { - $<node>$ = add_block_exit(p, NEW_NEXT(0, &@$)); + $$ = add_block_exit(p, NEW_NEXT(0, &@$)); /*% ripper: next!(args_new!) %*/ } | keyword_redo { - $<node>$ = add_block_exit(p, NEW_REDO(&@$)); + $$ = add_block_exit(p, NEW_REDO(&@$)); /*% ripper: redo! %*/ } | keyword_retry @@ -5032,7 +4909,6 @@ f_marg : f_norm_arg { $$ = assignable(p, $1, 0, &@$); mark_lvar_used(p, $$); - /*% ripper: ripper_assignable(p, $1, get_value($:1)) %*/ } | tLPAREN f_margs rparen { @@ -5056,7 +4932,7 @@ f_marg_list : f_marg f_margs : f_marg_list { $$ = NEW_MASGN($1, 0, &@$); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | f_marg_list ',' f_rest_marg { @@ -5082,9 +4958,9 @@ f_margs : f_marg_list f_rest_marg : tSTAR f_norm_arg { + /*% ripper: $:2 %*/ $$ = assignable(p, $2, 0, &@$); mark_lvar_used(p, $$); - /*% ripper: ripper_assignable(p, $2, get_value($:2)) %*/ } | tSTAR { @@ -5106,22 +4982,22 @@ f_eq : {p->ctxt.in_argdef = 0;} '='; block_args_tail : f_kwarg(f_block_kw) ',' f_kwrest opt_f_block_arg { $$ = new_args_tail(p, $1, $3, $4, &@3); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), get_value($:3), get_value($:4)); %*/ + /*% ripper: [$:1, $:3, $:4] %*/ } | f_kwarg(f_block_kw) opt_f_block_arg { $$ = new_args_tail(p, $1, 0, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), Qnil, get_value($:2)); %*/ + /*% ripper: [$:1, Qnil, $:2] %*/ } | f_any_kwrest opt_f_block_arg { $$ = new_args_tail(p, 0, $1, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), get_value($:2)); %*/ + /*% ripper: [Qnil, $:1, $:2] %*/ } | f_block_arg { $$ = new_args_tail(p, 0, 0, $1, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, get_value($:1)); %*/ + /*% ripper: [Qnil, Qnil, $:1] %*/ } ; @@ -5136,78 +5012,78 @@ excessed_comma : ',' block_param : f_arg ',' f_optarg(primary_value) ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, $5, 0, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), Qnil, get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, $:5, Qnil, *$:6[0..2]) %*/ } | f_arg ',' f_optarg(primary_value) ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, $5, $7, $8, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), get_value($:7), get_value($:8)) %*/ + /*% ripper: params!($:1, $:3, $:5, $:7, *$:8[0..2]) %*/ } | f_arg ',' f_optarg(primary_value) opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, 0, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, $:3, Qnil, Qnil, *$:4[0..2]) %*/ } | f_arg ',' f_optarg(primary_value) ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, 0, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, Qnil, $:5, *$:6[0..2]) %*/ } | f_arg ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, Qnil, $:3, Qnil, *$:4[0..2]) %*/ } | f_arg excessed_comma { $$ = new_args_tail(p, 0, 0, 0, &@2); $$ = new_args(p, $1, 0, $2, 0, $$, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:2), Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!($:1, Qnil, $:2, Qnil, Qnil, Qnil, Qnil) %*/ } | f_arg ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, Qnil, $:3, $:5, *$:6[0..2]) %*/ } | f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!($:1, Qnil, Qnil, Qnil, *$:2[0..2]) %*/ } | f_optarg(primary_value) ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, Qnil, *$:4[0..2]) %*/ } | f_optarg(primary_value) ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, $:5, *$:6[0..2]) %*/ } | f_optarg(primary_value) opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, Qnil, *$:2[0..2]) %*/ } | f_optarg(primary_value) ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, 0, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, $:3, *$:4[0..2]) %*/ } | f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, 0, $1, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, Qnil, *$:2[0..2]) %*/ } | f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, 0, $1, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, $:3, *$:4[0..2]) %*/ } | block_args_tail { $$ = new_args(p, 0, 0, 0, 0, $1, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, get_value($:1)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, *$:1[0..2]) %*/ } ; @@ -5220,20 +5096,13 @@ opt_block_param : none block_param_def : '|' opt_bv_decl '|' { - p->cur_arg = 0; p->max_numparam = ORDINAL_PARAM; p->ctxt.in_argdef = 0; $$ = 0; - /*%%%*/ - /*% - VALUE val = dispatch7(params, Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil); - val = dispatch2(block_var, val, get_value($:2)); - set_value(val); - %*/ + /*% ripper: block_var!(params!(Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil), $:2) %*/ } | '|' block_param opt_bv_decl '|' { - p->cur_arg = 0; p->max_numparam = ORDINAL_PARAM; p->ctxt.in_argdef = 0; $$ = $2; @@ -5250,20 +5119,20 @@ opt_bv_decl : '\n'? | '\n'? ';' bv_decls '\n'? { $$ = 0; - /*% ripper: get_value($:3) %*/ + /*% ripper: $:3 %*/ } ; bv_decls : bvar - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | bv_decls ',' bvar - /*% ripper[brace]: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper[brace]: rb_ary_push($:1, $:3) %*/ ; bvar : tIDENTIFIER { new_bv(p, $1); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | f_bad_arg { @@ -5288,13 +5157,12 @@ it_id : { } ; -lambda : tLAMBDA[dyna] +lambda : tLAMBDA[lpar] { token_info_push(p, "->", &@1); - $<vars>dyna = dyna_push(p); - $<num>$ = p->lex.lpar_beg; + $$ = dyna_push(p); p->lex.lpar_beg = p->lex.paren_nest; - }[lpar] + }[dyna]<vars> max_numparam numparam it_id allow_exits f_larglist[args] { @@ -5304,7 +5172,7 @@ lambda : tLAMBDA[dyna] { int max_numparam = p->max_numparam; ID it_id = p->it_id; - p->lex.lpar_beg = $<num>lpar; + p->lex.lpar_beg = $lpar; p->max_numparam = $max_numparam; p->it_id = $it_id; restore_block_exit(p, $allow_exits); @@ -5319,7 +5187,7 @@ lambda : tLAMBDA[dyna] } /*% ripper: lambda!($:args, $:body) %*/ numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; @@ -5343,7 +5211,7 @@ lambda_body : tLAMBEG compstmt '}' { token_info_pop(p, "}", &@3); $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | keyword_do_LAMBDA { @@ -5352,7 +5220,7 @@ lambda_body : tLAMBEG compstmt '}' bodystmt k_end { $$ = $3; - /*% ripper: get_value($:3); %*/ + /*% ripper: $:3 %*/ } ; @@ -5360,7 +5228,7 @@ do_block : k_do_block do_body k_end { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -5378,13 +5246,23 @@ block_call : command do_block } | block_call call_op2 operation2 opt_paren_args { + bool has_args = $4 != 0; + if (NODE_EMPTY_ARGS_P($4)) $4 = 0; $$ = new_qcall(p, $2, $1, $3, $4, &@3, &@$); - /*% ripper: opt_event(:method_add_arg!, call!($:1, $:2, $:3), $:4) %*/ + /*% ripper: call!($:1, $:2, $:3) %*/ + if (has_args) { + /*% ripper: method_add_arg!($:$, $:4) %*/ + } } | block_call call_op2 operation2 opt_paren_args brace_block { + bool has_args = $5 != 0; + if (NODE_EMPTY_ARGS_P($5)) $5 = 0; $$ = new_command_qcall(p, $2, $1, $3, $4, $5, &@3, &@$); - /*% ripper: opt_event(:method_add_block!, command_call!($:1, $:2, $:3, $:4), $:5) %*/ + /*% ripper: command_call!($:1, $:2, $:3, $:4) %*/ + if (has_args) { + /*% ripper: method_add_block!($:$, $:5) %*/ + } } | block_call call_op2 operation2 command_args do_block { @@ -5402,9 +5280,14 @@ method_call : fcall paren_args } | primary_value call_op operation2 opt_paren_args { + bool has_args = $4 != 0; + if (NODE_EMPTY_ARGS_P($4)) $4 = 0; $$ = new_qcall(p, $2, $1, $3, $4, &@3, &@$); nd_set_line($$, @3.end_pos.lineno); - /*% ripper: opt_event(:method_add_arg!, call!($:1, $:2, $:3), $:4) %*/ + /*% ripper: call!($:1, $:2, $:3) %*/ + if (has_args) { + /*% ripper: method_add_arg!($:$, $:4) %*/ + } } | primary_value tCOLON2 operation2 paren_args { @@ -5451,17 +5334,17 @@ brace_block : '{' brace_body '}' { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | k_do do_body k_end { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; -brace_body : {$<vars>$ = dyna_push(p);}[dyna] +brace_body : {$$ = dyna_push(p);}[dyna]<vars> max_numparam numparam it_id allow_exits opt_block_param[args] compstmt { @@ -5474,28 +5357,28 @@ brace_body : {$<vars>$ = dyna_push(p);}[dyna] /*% ripper: brace_block!($:args, $:compstmt) %*/ restore_block_exit(p, $allow_exits); numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; do_body : { - $<vars>$ = dyna_push(p); + $$ = dyna_push(p); CMDARG_PUSH(0); - }[dyna] + }[dyna]<vars> max_numparam numparam it_id allow_exits opt_block_param[args] bodystmt { int max_numparam = p->max_numparam; ID it_id = p->it_id; p->max_numparam = $max_numparam; - p->it_id = $<id>it_id; + p->it_id = $it_id; $args = args_with_numbered(p, $args, max_numparam, it_id); $$ = NEW_ITER($args, $bodystmt, &@$); /*% ripper: do_block!($:args, $:bodystmt) %*/ CMDARG_POP(); restore_block_exit(p, $allow_exits); numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; @@ -5588,28 +5471,28 @@ p_top_expr_body : p_expr { $$ = new_array_pattern_tail(p, 0, 1, 0, 0, &@$); $$ = new_array_pattern(p, 0, $1, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, get_value($:1), rb_ary_new()); %*/ + /*% ripper: aryptn!(Qnil, [$:1], Qnil, Qnil) %*/ } | p_expr ',' p_args { $$ = new_array_pattern(p, 0, $1, $3, &@$); nd_set_first_loc($$, @1.beg_pos); - /*% ripper: ripper_new_array_pattern(p, Qnil, get_value($:1), get_value($:3)); %*/ + /*% ripper: aryptn!(Qnil, aryptn_pre_args(p, $:1, $:3[0]), *$:3[1..2]) %*/ } | p_find { $$ = new_find_pattern(p, 0, $1, &@$); - /*% ripper: ripper_new_find_pattern(p, Qnil, get_value($:1)); %*/ + /*% ripper: fndptn!(Qnil, *$:1[0..2]) %*/ } | p_args_tail { $$ = new_array_pattern(p, 0, 0, $1, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, get_value($:1)); %*/ + /*% ripper: aryptn!(Qnil, *$:1[0..2]) %*/ } | p_kwargs { $$ = new_hash_pattern(p, 0, $1, &@$); - /*% ripper: ripper_new_hash_pattern(p, Qnil, get_value($:1)); %*/ + /*% ripper: hshptn!(Qnil, *$:1[0..1]) %*/ } ; @@ -5637,14 +5520,14 @@ p_alt : p_alt '|' p_expr_basic p_lparen : '(' p_pktbl { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; p_lbracket : '[' p_pktbl { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -5655,70 +5538,70 @@ p_expr_basic : p_value pop_pktbl(p, $p_pktbl); $$ = new_array_pattern(p, $p_const, 0, $p_args, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!($:p_const, *$:p_args[0..2]) %*/ } | p_const p_lparen[p_pktbl] p_find rparen { pop_pktbl(p, $p_pktbl); $$ = new_find_pattern(p, $p_const, $p_find, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_find_pattern(p, get_value($:p_const), get_value($:p_find)); %*/ + /*% ripper: fndptn!($:p_const, *$:p_find[0..2]) %*/ } | p_const p_lparen[p_pktbl] p_kwargs rparen { pop_pktbl(p, $p_pktbl); $$ = new_hash_pattern(p, $p_const, $p_kwargs, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_hash_pattern(p, get_value($:p_const), get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!($:p_const, *$:p_kwargs[0..1]) %*/ } | p_const '(' rparen { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, $p_const, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!($:p_const, Qnil, Qnil, Qnil) %*/ } | p_const p_lbracket[p_pktbl] p_args rbracket { pop_pktbl(p, $p_pktbl); $$ = new_array_pattern(p, $p_const, 0, $p_args, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!($:p_const, *$:p_args[0..2]) %*/ } | p_const p_lbracket[p_pktbl] p_find rbracket { pop_pktbl(p, $p_pktbl); $$ = new_find_pattern(p, $p_const, $p_find, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_find_pattern(p, get_value($:p_const), get_value($:p_find)); %*/ + /*% ripper: fndptn!($:p_const, *$:p_find[0..2]) %*/ } | p_const p_lbracket[p_pktbl] p_kwargs rbracket { pop_pktbl(p, $p_pktbl); $$ = new_hash_pattern(p, $p_const, $p_kwargs, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_hash_pattern(p, get_value($:p_const), get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!($:p_const, *$:p_kwargs[0..1]) %*/ } | p_const '[' rbracket { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, $1, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, get_value($:1), Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!($:1, Qnil, Qnil, Qnil) %*/ } | tLBRACK p_args rbracket { $$ = new_array_pattern(p, 0, 0, $p_args, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!(Qnil, *$:p_args[0..2]) %*/ } | tLBRACK p_find rbracket { $$ = new_find_pattern(p, 0, $p_find, &@$); - /*% ripper: ripper_new_find_pattern(p, Qnil, get_value($:p_find)); %*/ + /*% ripper: fndptn!(Qnil, *$:p_find[0..2]) %*/ } | tLBRACK rbracket { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, 0, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!(Qnil, Qnil, Qnil, Qnil) %*/ } | tLBRACE p_pktbl lex_ctxt[ctxt] { @@ -5729,24 +5612,19 @@ p_expr_basic : p_value pop_pktbl(p, $p_pktbl); p->ctxt.in_kwarg = $ctxt.in_kwarg; $$ = new_hash_pattern(p, 0, $p_kwargs, &@$); - /*% ripper: ripper_new_hash_pattern(p, Qnil, get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!(Qnil, *$:p_kwargs[0..1]) %*/ } | tLBRACE rbrace { $$ = new_hash_pattern_tail(p, 0, 0, &@$); $$ = new_hash_pattern(p, 0, $$, &@$); - /*%%%*/ - /*% - VALUE val = ripper_new_hash_pattern_tail(p, Qnil, 0); - val = ripper_new_hash_pattern(p, Qnil, val); - set_value(val); - %*/ + /*% ripper: hshptn!(Qnil, Qnil, Qnil) %*/ } | tLPAREN p_pktbl p_expr rparen { pop_pktbl(p, $p_pktbl); $$ = $p_expr; - /*% ripper: get_value($:p_expr); %*/ + /*% ripper: $:p_expr %*/ } ; @@ -5754,44 +5632,27 @@ p_args : p_expr { NODE *pre_args = NEW_LIST($1, &@$); $$ = new_array_pattern_tail(p, pre_args, 0, 0, 0, &@$); - /*%%%*/ - /*% - VALUE ary = rb_ary_new_from_args(1, get_value($:1)); - set_value(rb_ary_new_from_args(3, ary, Qnil, Qnil)); - %*/ + /*% ripper: [[$:1], Qnil, Qnil] %*/ } | p_args_head { $$ = new_array_pattern_tail(p, $1, 1, 0, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), Qnil, Qnil)); - %*/ + /*% ripper: [$:1, Qnil, Qnil] %*/ } | p_args_head p_arg { $$ = new_array_pattern_tail(p, list_concat($1, $2), 0, 0, 0, &@$); - /*%%%*/ - /*% - VALUE pre_args = rb_ary_concat(get_value($:1), get_value($:2)); - set_value(rb_ary_new_from_args(3, pre_args, Qnil, Qnil)); - %*/ + /*% ripper: [rb_ary_concat($:1, $:2), Qnil, Qnil] %*/ } | p_args_head p_rest { $$ = new_array_pattern_tail(p, $1, 1, $2, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), get_value($:2), Qnil)); - %*/ + /*% ripper: [$:1, $:2, Qnil] %*/ } | p_args_head p_rest ',' p_args_post { $$ = new_array_pattern_tail(p, $1, 1, $2, $4, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), get_value($:2), get_value($:4))); - %*/ + /*% ripper: [$:1, $:2, $:4] %*/ } | p_args_tail ; @@ -5803,26 +5664,26 @@ p_args_head : p_arg ',' | p_args_head p_arg ',' { $$ = list_concat($1, $2); - /*% ripper: rb_ary_concat(get_value($:1), get_value($:2)) %*/ + /*% ripper: rb_ary_concat($:1, $:2) %*/ } ; p_args_tail : p_rest { $$ = new_array_pattern_tail(p, 0, 1, $1, 0, &@$); - /*% ripper: ripper_new_array_pattern_tail(p, Qnil, get_value($:1), Qnil); %*/ + /*% ripper: [Qnil, $:1, Qnil] %*/ } | p_rest ',' p_args_post { $$ = new_array_pattern_tail(p, 0, 1, $1, $3, &@$); - /*% ripper: ripper_new_array_pattern_tail(p, Qnil, get_value($:1), get_value($:3)); %*/ + /*% ripper: [Qnil, $:1, $:3] %*/ } ; p_find : p_rest ',' p_args_post ',' p_rest { $$ = new_find_pattern_tail(p, $1, $3, $5, &@$); - /*% ripper: ripper_new_find_pattern_tail(p, get_value($:1), get_value($:3), get_value($:5)) %*/ + /*% ripper: [$:1, $:3, $:5] %*/ } ; @@ -5830,13 +5691,13 @@ p_find : p_rest ',' p_args_post ',' p_rest p_rest : tSTAR tIDENTIFIER { error_duplicate_pattern_variable(p, $2, &@2); + /*% ripper: var_field!($:2) %*/ $$ = assignable(p, $2, 0, &@$); - /*% ripper: ripper_assignable(p, $2, var_field(p, get_value($:2))) %*/ } | tSTAR { $$ = 0; - /*% ripper: var_field(p, Qnil) %*/ + /*% ripper: var_field!(Qnil) %*/ } ; @@ -5844,45 +5705,45 @@ p_args_post : p_arg | p_args_post ',' p_arg { $$ = list_concat($1, $3); - /*% ripper: rb_ary_concat(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_concat($:1, $:3) %*/ } ; p_arg : p_expr { $$ = NEW_LIST($1, &@$); - /*% ripper: rb_ary_new_from_args(1, get_value($:1)) %*/ + /*% ripper: [$:1] %*/ } ; p_kwargs : p_kwarg ',' p_any_kwrest { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), $3, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), get_value($:3)) %*/ + /*% ripper: [$:1, $:3] %*/ } | p_kwarg { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), 0, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), 0) %*/ + /*% ripper: [$:1, Qnil] %*/ } | p_kwarg ',' { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), 0, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), 0) %*/ + /*% ripper: [$:1, Qnil] %*/ } | p_any_kwrest { $$ = new_hash_pattern_tail(p, new_hash(p, 0, &@$), $1, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, rb_ary_new(), get_value($:1)) %*/ + /*% ripper: [[], $:1] %*/ } ; p_kwarg : p_kw - /*% ripper[brace]: rb_ary_new_from_args(1, get_value($:1)) %*/ + /*% ripper[brace]: [$:1] %*/ | p_kwarg ',' p_kw { $$ = list_concat($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; @@ -5890,7 +5751,7 @@ p_kw : p_kw_label p_expr { error_duplicate_pattern_key(p, $1, &@1); $$ = list_append(p, NEW_LIST(NEW_SYM(rb_id2str($1), &@1), &@$), $2); - /*% ripper: rb_ary_new_from_args(2, get_value($:1), get_value($:2)) %*/ + /*% ripper: [$:1, $:2] %*/ } | p_kw_label { @@ -5900,7 +5761,7 @@ p_kw : p_kw_label p_expr } error_duplicate_pattern_variable(p, $1, &@1); $$ = list_append(p, NEW_LIST(NEW_SYM(rb_id2str($1), &@$), &@$), assignable(p, $1, 0, &@$)); - /*% ripper: rb_ary_new_from_args(2, ripper_assignable(p, $1, get_value($:1)), Qnil) %*/ + /*% ripper: [$:1, Qnil] %*/ } ; @@ -5916,19 +5777,19 @@ p_kw_label : tLABEL yyerror1(&loc, "symbol literal with interpolation is not allowed"); $$ = rb_intern_str(STR_NEW0()); } - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; p_kwrest : kwrest_mark tIDENTIFIER { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: var_field!($:2) %*/ } | kwrest_mark { $$ = 0; - /*% ripper: 0; %*/ + /*% ripper: Qnil %*/ } ; @@ -5942,7 +5803,7 @@ p_any_kwrest : p_kwrest | p_kwnorest { $$ = idNil; - /*% ripper: ID2VAL(idNil) %*/ + /*% ripper: var_field!(ID2VAL(idNil)) %*/ } ; @@ -6009,8 +5870,8 @@ p_primitive : literal p_variable : tIDENTIFIER { error_duplicate_pattern_variable(p, $1, &@1); + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } ; @@ -6085,7 +5946,7 @@ opt_rescue : k_rescue exc_list exc_var then exc_list : arg_value { $$ = NEW_LIST($1, &@$); - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | mrhs { @@ -6097,7 +5958,7 @@ exc_list : arg_value exc_var : tASSOC lhs { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none ; @@ -6125,7 +5986,7 @@ strings : string node = evstr2dstr(p, node); } $$ = node; - /*% ripper: get_value($:1); %*/ + /*% ripper: $:1 %*/ } ; @@ -6140,32 +6001,26 @@ string : tCHAR string1 : tSTRING_BEG string_contents tSTRING_END { - /*%%%*/ - /*% - int indent = p->heredoc_indent; - %*/ $$ = heredoc_dedent(p, $2); if ($$) nd_set_loc($$, &@$); - /*%%%*/ - /*% - VALUE val = dispatch1(string_literal, ripper_heredoc_dedent(p, indent, get_value($:2))); - set_value(val); - %*/ + /*% ripper: $:2 %*/ + if (p->heredoc_indent > 0) { + /*% ripper: heredoc_dedent!($:$, INT2NUM(%{p->heredoc_indent})) %*/ + p->heredoc_indent = 0; + } + /*% ripper: string_literal!($:$) %*/ } ; xstring : tXSTRING_BEG xstring_contents tSTRING_END { - /*%%%*/ - /*% - int indent = p->heredoc_indent; - %*/ $$ = new_xstring(p, heredoc_dedent(p, $2), &@$); - /*%%%*/ - /*% - VALUE val = dispatch1(xstring_literal, ripper_heredoc_dedent(p, indent, get_value($:2))); - set_value(val); - %*/ + /*% ripper: $:2 %*/ + if (p->heredoc_indent > 0) { + /*% ripper: heredoc_dedent!($:$, INT2NUM(%{p->heredoc_indent})) %*/ + p->heredoc_indent = 0; + } + /*% ripper: xstring_literal!($:$) %*/ } ; @@ -6248,8 +6103,6 @@ qsym_list : /* none */ string_contents : /* none */ { $$ = 0; - /*%%%*/ - /*% %*/ /*% ripper: string_content! %*/ } | string_contents string_content @@ -6303,47 +6156,46 @@ regexp_contents: /* none */ ; string_content : tSTRING_CONTENT - /*% ripper[brace]: get_value($:1); %*/ + /*% ripper[brace]: $:1 %*/ | tSTRING_DVAR { /* need to backup p->lex.strterm so that a string literal `%&foo,#$&,bar&` can be parsed */ - $<strterm>$ = p->lex.strterm; + $$ = p->lex.strterm; p->lex.strterm = 0; SET_LEX_STATE(EXPR_BEG); - } + }<strterm> string_dvar { - p->lex.strterm = $<strterm>2; + p->lex.strterm = $2; $$ = NEW_EVSTR($3, &@$); nd_set_line($$, @3.end_pos.lineno); /*% ripper: string_dvar!($:3) %*/ } - | tSTRING_DBEG[term] + | tSTRING_DBEG[state] { CMDARG_PUSH(0); COND_PUSH(0); /* need to backup p->lex.strterm so that a string literal `%!foo,#{ !0 },bar!` can be parsed */ - $<strterm>term = p->lex.strterm; + $$ = p->lex.strterm; p->lex.strterm = 0; - $<num>$ = p->lex.state; SET_LEX_STATE(EXPR_BEG); - }[state] + }[term]<strterm> { - $<num>$ = p->lex.brace_nest; + $$ = p->lex.brace_nest; p->lex.brace_nest = 0; - }[brace] + }[brace]<num> { - $<num>$ = p->heredoc_indent; + $$ = p->heredoc_indent; p->heredoc_indent = 0; - }[indent] + }[indent]<num> compstmt string_dend { COND_POP(); CMDARG_POP(); - p->lex.strterm = $<strterm>term; - SET_LEX_STATE($<num>state); - p->lex.brace_nest = $<num>brace; - p->heredoc_indent = $<num>indent; + p->lex.strterm = $term; + SET_LEX_STATE($state); + p->lex.brace_nest = $brace; + p->heredoc_indent = $indent; p->heredoc_line_indent = -1; if ($compstmt) nd_unset_fl_newline($compstmt); $$ = new_evstr(p, $compstmt, &@$); @@ -6432,17 +6284,12 @@ keyword_variable: keyword_nil {$$ = KWD2EID(nil, $1);} var_ref : user_variable { if (!($$ = gettable(p, $1, &@$))) $$ = NEW_ERROR(&@$); - /*%%%*/ - /*% - if (id_is_var(p, $1)) { - VALUE val = dispatch1(var_ref, get_value($:1)); - set_value(val); + if (ifdef_ripper(id_is_var(p, $1), false)) { + /*% ripper: var_ref!($:1) %*/ } else { - VALUE val = dispatch1(vcall, get_value($:1)); - set_value(val); + /*% ripper: vcall!($:1) %*/ } - %*/ } | keyword_variable { @@ -6453,13 +6300,13 @@ var_ref : user_variable var_lhs : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } ; @@ -6475,7 +6322,7 @@ superclass : '<' expr_value term { $$ = $3; - /*% ripper: get_value($:3); %*/ + /*% ripper: $:3 %*/ } | /* none */ { @@ -6490,7 +6337,7 @@ f_opt_paren_args: f_paren_args p->ctxt.in_argdef = 0; $$ = new_args_tail(p, 0, 0, 0, &@0); $$ = new_args(p, 0, 0, 0, 0, $$, &@0); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, Qnil, Qnil, Qnil) %*/ } ; @@ -6506,126 +6353,126 @@ f_paren_args : '(' f_args rparen f_arglist : f_paren_args | { - $<ctxt>$ = p->ctxt; + $$ = p->ctxt; p->ctxt.in_kwarg = 1; p->ctxt.in_argdef = 1; SET_LEX_STATE(p->lex.state|EXPR_LABEL); /* force for args */ - } + }<ctxt> f_args term { - p->ctxt.in_kwarg = $<ctxt>1.in_kwarg; + p->ctxt.in_kwarg = $1.in_kwarg; p->ctxt.in_argdef = 0; $$ = $2; SET_LEX_STATE(EXPR_BEG); p->command_start = TRUE; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; args_tail : f_kwarg(f_kw) ',' f_kwrest opt_f_block_arg { $$ = new_args_tail(p, $1, $3, $4, &@3); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), get_value($:3), get_value($:4)); %*/ + /*% ripper: [$:1, $:3, $:4] %*/ } | f_kwarg(f_kw) opt_f_block_arg { $$ = new_args_tail(p, $1, 0, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), Qnil, get_value($:2)); %*/ + /*% ripper: [$:1, Qnil, $:2] %*/ } | f_any_kwrest opt_f_block_arg { $$ = new_args_tail(p, 0, $1, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), get_value($:2)); %*/ + /*% ripper: [Qnil, $:1, $:2] %*/ } | f_block_arg { $$ = new_args_tail(p, 0, 0, $1, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, get_value($:1)); %*/ + /*% ripper: [Qnil, Qnil, $:1] %*/ } | args_forward { add_forwarding_args(p); $$ = new_args_tail(p, 0, $1, arg_FWD_BLOCK, &@1); $$->nd_ainfo.forwarding = 1; - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), Qnil); %*/ + /*% ripper: [Qnil, $:1, Qnil] %*/ } ; f_args : f_arg ',' f_optarg(arg_value) ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, $5, 0, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), Qnil, get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, $:5, Qnil, *$:6[0..2]) %*/ } | f_arg ',' f_optarg(arg_value) ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, $5, $7, $8, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), get_value($:7), get_value($:8)) %*/ + /*% ripper: params!($:1, $:3, $:5, $:7, *$:8[0..2]) %*/ } | f_arg ',' f_optarg(arg_value) opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, 0, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, $:3, Qnil, Qnil, *$:4[0..2]) %*/ } | f_arg ',' f_optarg(arg_value) ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, 0, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, Qnil, $:5, *$:6[0..2]) %*/ } | f_arg ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, Qnil, $:3, Qnil, *$:4[0..2]) %*/ } | f_arg ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, Qnil, $:3, $:5, *$:6[0..2]) %*/ } | f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!($:1, Qnil, Qnil, Qnil, *$:2[0..2]) %*/ } | f_optarg(arg_value) ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, Qnil, *$:4[0..2]) %*/ } | f_optarg(arg_value) ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, $:5, *$:6[0..2]) %*/ } | f_optarg(arg_value) opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, Qnil, *$:2[0..2]) %*/ } | f_optarg(arg_value) ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, 0, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, $:3, *$:4[0..2]) %*/ } | f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, 0, $1, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, Qnil, *$:2[0..2]) %*/ } | f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, 0, $1, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, $:3, *$:4[0..2]) %*/ } | args_tail { $$ = new_args(p, 0, 0, 0, 0, $1, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, get_value($:1)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, *$:1[0..2]) %*/ } | /* none */ { $$ = new_args_tail(p, 0, 0, 0, &@0); $$ = new_args(p, 0, 0, 0, 0, $$, &@0); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, Qnil, Qnil, Qnil) %*/ } ; @@ -6681,13 +6528,11 @@ f_bad_arg : tCONSTANT f_norm_arg : f_bad_arg | tIDENTIFIER { - formal_argument(p, $1); + VALUE e = formal_argument_error(p, $$ = $1); + if (e) { + /*% ripper[error]: param_error!(?e, $:1) %*/ + } p->max_numparam = ORDINAL_PARAM; - $$ = $1; - /*%%%*/ - /*% - ripper_formal_argument(p, $1, get_value($:1)); - %*/ } ; @@ -6695,16 +6540,14 @@ f_arg_asgn : f_norm_arg { ID id = $1; arg_var(p, id); - p->cur_arg = id; $$ = $1; } ; f_arg_item : f_arg_asgn { - p->cur_arg = 0; $$ = NEW_ARGS_AUX($1, 1, &NULL_LOC); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | tLPAREN f_margs rparen { @@ -6726,45 +6569,49 @@ f_arg_item : f_arg_asgn ; f_arg : f_arg_item - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | f_arg ',' f_arg_item { $$ = $1; $$->nd_plen++; $$->nd_next = block_append(p, $$->nd_next, $3->nd_next); rb_discard_node(p, (NODE *)$3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; f_label : tLABEL { - arg_var(p, formal_argument(p, $1)); - p->cur_arg = $1; + VALUE e = formal_argument_error(p, $$ = $1); + if (e) { + $$ = 0; + /*% ripper[error]: param_error!(?e, $:1) %*/ + } + /* + * Workaround for Prism::ParseTest#test_filepath for + * "unparser/corpus/literal/def.txt" + * + * See the discussion on https://github.com/ruby/ruby/pull/9923 + */ + arg_var(p, ifdef_ripper(0, $1)); + /*% ripper: $:1 %*/ p->max_numparam = ORDINAL_PARAM; p->ctxt.in_argdef = 0; - $$ = $1; - /*%%%*/ - /*% - ripper_formal_argument(p, $1, get_value($:1)); - %*/ } ; f_kw : f_label arg_value { - p->cur_arg = 0; p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, $2, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:2)) %*/ + /*% ripper: [$:$, $:2] %*/ } | f_label { - p->cur_arg = 0; p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, NODE_SPECIAL_REQUIRED_KEYWORD, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), 0) %*/ + /*% ripper: [$:$, 0] %*/ } ; @@ -6772,13 +6619,13 @@ f_block_kw : f_label primary_value { p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, $2, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:2)) %*/ + /*% ripper: [$:$, $:2] %*/ } | f_label { p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, NODE_SPECIAL_REQUIRED_KEYWORD, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), 0) %*/ + /*% ripper: [$:$, 0] %*/ } ; @@ -6845,12 +6692,12 @@ f_block_arg : blkarg_mark tIDENTIFIER opt_f_block_arg : ',' f_block_arg { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none { $$ = 0; - /*% ripper: Qnil; %*/ + /*% ripper: Qnil %*/ } ; @@ -6900,7 +6747,7 @@ assoc_list : none ; assocs : assoc - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | assocs ',' assoc { NODE *assocs = $1; @@ -6922,7 +6769,7 @@ assocs : assoc } } $$ = assocs; - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; @@ -7017,10 +6864,6 @@ terms : term none : /* none */ { $$ = 0; - /*%%%*/ - /*% - set_value(rb_ripper_none); - %*/ } ; %% @@ -7678,7 +7521,7 @@ yycompile0(VALUE arg) struct parser_params *p = (struct parser_params *)arg; int cov = FALSE; - if (!compile_for_eval && p->ruby_sourcefile_string && !e_option_supplied(p)) { + if (!compile_for_eval && !NIL_P(p->ruby_sourcefile_string) && !e_option_supplied(p)) { cov = TRUE; } @@ -7732,39 +7575,17 @@ yycompile0(VALUE arg) return TRUE; } -static void -set_arg_error(struct parser_params *p, const char *err) -{ - VALUE excargs[3]; - - excargs[0] = rb_eArgError; - excargs[1] = rb_str_new_cstr(err); - excargs[2] = rb_make_backtrace(); - rb_set_errinfo(rb_make_exception(3, excargs)); -} - static rb_ast_t * -yycompile(struct parser_params *p, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, int line) +yycompile(struct parser_params *p, VALUE fname, int line) { rb_ast_t *ast; - if (!fname_ptr) { - p->ruby_sourcefile_string = NULL; + if (NIL_P(fname)) { + p->ruby_sourcefile_string = Qnil; p->ruby_sourcefile = "(none)"; } else { - int w; - if (cstr_null_check(p, fname_ptr, fname_len, fname_enc, &w)) { - if (w) { - set_arg_error(p, "string contains null char"); - } - else { - set_arg_error(p, "string contains null byte"); - } - return rb_ast_new(); - } - - p->ruby_sourcefile_string = rb_parser_encoding_string_new(p, fname_ptr, fname_len, fname_enc); - p->ruby_sourcefile = fname_ptr; + p->ruby_sourcefile_string = rb_str_to_interned_str(fname); + p->ruby_sourcefile = StringValueCStr(fname); } p->ruby_sourceline = line - 1; @@ -7805,14 +7626,13 @@ lex_getline(struct parser_params *p) #ifndef RIPPER rb_ast_t* -rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, - const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line) +rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line) { p->lex.gets = gets; p->lex.input = input; p->lex.pbeg = p->lex.pcur = p->lex.pend = 0; - return yycompile(p, fname_ptr, fname_len, fname_enc, line); + return yycompile(p, fname, line); } #endif /* !RIPPER */ @@ -8320,7 +8140,11 @@ read_escape(struct parser_params *p, int flags) } return read_escape(p, flags|ESCAPE_META) | 0x80; } - else if (c == -1 || !ISASCII(c)) goto eof; + else if (c == -1) goto eof; + else if (!ISASCII(c)) { + tokskip_mbchar(p); + goto eof; + } else { int c2 = escaped_control_code(c); if (c2) { @@ -8578,6 +8402,10 @@ parser_update_heredoc_indent(struct parser_params *p, int c) } p->heredoc_line_indent = -1; } + else { + /* Whitespace only line has no indentation */ + p->heredoc_line_indent = 0; + } } return FALSE; } @@ -8785,7 +8613,6 @@ flush_string_content(struct parser_params *p, rb_encoding *enc) dispatch_scan_event(p, tSTRING_CONTENT); } -RUBY_FUNC_EXPORTED const uint_least32_t ruby_global_name_punct_bits[(0x7e - 0x20 + 31) / 32]; /* this can be shared with ripper, since it's independent from struct * parser_params. */ #ifndef RIPPER @@ -8833,6 +8660,7 @@ parser_peek_variable_name(struct parser_params *p) case '{': p->lex.pcur = ptr; p->command_start = TRUE; + yylval.state = p->lex.state; return tSTRING_DBEG; default: return 0; @@ -9111,7 +8939,6 @@ heredoc_dedent(struct parser_params *p, NODE *root) rb_parser_string_t *prev_lit = 0; if (indent <= 0) return root; - p->heredoc_indent = 0; if (!root) return root; prev_node = node = str_node = root; @@ -9155,17 +8982,6 @@ heredoc_dedent(struct parser_params *p, NODE *root) return root; } -#ifdef RIPPER -static VALUE -ripper_heredoc_dedent(struct parser_params *p, int indent, VALUE array) -{ - if (indent <= 0) return array; - p->heredoc_indent = 0; - dispatch2(heredoc_dedent, array, INT2NUM(indent)); - return array; -} -#endif - static int whole_match_p(struct parser_params *p, const char *eos, long len, int indent) { @@ -9492,71 +9308,35 @@ arg_ambiguous(struct parser_params *p, char c) return TRUE; } -static ID -formal_argument(struct parser_params *p, ID id) +/* returns true value if formal argument error; + * Qtrue, or error message if ripper */ +static VALUE +formal_argument_error(struct parser_params *p, ID id) { switch (id_type(id)) { case ID_LOCAL: break; -#define ERR(mesg) yyerror0(mesg) - case ID_CONST: - ERR("formal argument cannot be a constant"); - return 0; - case ID_INSTANCE: - ERR("formal argument cannot be an instance variable"); - return 0; - case ID_GLOBAL: - ERR("formal argument cannot be a global variable"); - return 0; - case ID_CLASS: - ERR("formal argument cannot be a class variable"); - return 0; - default: - ERR("formal argument must be local variable"); - return 0; -#undef ERR - } - shadowing_lvar(p, id); - -/* - * Workaround for Prism::ParseTest#test_filepath for "unparser/corpus/literal/def.txt" - * - * See the discussion on https://github.com/ruby/ruby/pull/9923 - */ #ifndef RIPPER - return id; +# define ERR(mesg) (yyerror0(mesg), Qtrue) #else - return 0; +# define ERR(mesg) WARN_S(mesg) #endif -} - -#ifdef RIPPER -static void -ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs) -{ - switch (id_type(id)) { - case ID_LOCAL: - break; -#define ERR(mesg) (dispatch2(param_error, WARN_S(mesg), lhs), ripper_error(p)) case ID_CONST: - ERR("formal argument cannot be a constant"); - return; + return ERR("formal argument cannot be a constant"); case ID_INSTANCE: - ERR("formal argument cannot be an instance variable"); - return; + return ERR("formal argument cannot be an instance variable"); case ID_GLOBAL: - ERR("formal argument cannot be a global variable"); - return; + return ERR("formal argument cannot be a global variable"); case ID_CLASS: - ERR("formal argument cannot be a class variable"); - return; + return ERR("formal argument cannot be a class variable"); default: - ERR("formal argument must be local variable"); - return; + return ERR("formal argument must be local variable"); #undef ERR } + shadowing_lvar(p, id); + + return Qfalse; } -#endif static int lvar_defined(struct parser_params *p, ID id) @@ -9607,7 +9387,7 @@ parser_set_encode(struct parser_params *p, const char *name) error: excargs[0] = rb_eArgError; excargs[2] = rb_make_backtrace(); - rb_ary_unshift(excargs[2], rb_sprintf("%"PRIsVALUE":%d", rb_str_new_mutable_parser_string(p->ruby_sourcefile_string), p->ruby_sourceline)); + rb_ary_unshift(excargs[2], rb_sprintf("%"PRIsVALUE":%d", p->ruby_sourcefile_string, p->ruby_sourceline)); VALUE exc = rb_make_exception(3, excargs); ruby_show_error_line(p, exc, &(YYLTYPE)RUBY_INIT_YYLLOC(), p->ruby_sourceline, p->lex.lastline); rb_exc_raise(exc); @@ -10485,7 +10265,7 @@ parse_gvar(struct parser_params *p, const enum lex_state_e last_state) case '.': /* $.: last read line number */ case '=': /* $=: ignorecase */ case ':': /* $:: load path */ - case '<': /* $<: reading filename */ + case '<': /* $<: default input handle */ case '>': /* $>: default output handle */ case '\"': /* $": already loaded files */ tokadd(p, '$'); @@ -11215,6 +10995,7 @@ parser_yylex(struct parser_params *p) } if (c == '>') { SET_LEX_STATE(EXPR_ENDFN); + yylval.num = p->lex.lpar_beg; return tLAMBDA; } if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous(p, '-'))) { @@ -12279,7 +12060,7 @@ rb_node_dxstr_new(struct parser_params *p, rb_parser_string_t *string, long nd_a { rb_node_dxstr_t *n = NODE_NEWNODE(NODE_DXSTR, rb_node_dxstr_t, loc); n->string = string; - n->nd_alen = nd_alen; + n->as.nd_alen = nd_alen; n->nd_next = (rb_node_list_t *)nd_next; return n; @@ -12299,7 +12080,7 @@ rb_node_dsym_new(struct parser_params *p, rb_parser_string_t *string, long nd_al { rb_node_dsym_t *n = NODE_NEWNODE(NODE_DSYM, rb_node_dsym_t, loc); n->string = string; - n->nd_alen = nd_alen; + n->as.nd_alen = nd_alen; n->nd_next = (rb_node_list_t *)nd_next; return n; @@ -12584,10 +12365,10 @@ rb_node_line_new(struct parser_params *p, const YYLTYPE *loc) } static rb_node_file_t * -rb_node_file_new(struct parser_params *p, rb_parser_string_t *str, const YYLTYPE *loc) +rb_node_file_new(struct parser_params *p, VALUE str, const YYLTYPE *loc) { rb_node_file_t *n = NODE_NEWNODE(NODE_FILE, rb_node_file_t, loc); - n->path = str; + n->path = rb_str_to_parser_string(p, str); return n; } @@ -12666,7 +12447,6 @@ static rb_node_def_temp_t * rb_node_def_temp_new(struct parser_params *p, const YYLTYPE *loc) { rb_node_def_temp_t *n = NODE_NEWNODE((enum node_type)NODE_DEF_TEMP, rb_node_def_temp_t, loc); - n->save.cur_arg = p->cur_arg; n->save.numparam_save = 0; n->save.max_numparam = 0; n->save.ctxt = p->ctxt; @@ -12836,6 +12616,7 @@ string_literal_head(struct parser_params *p, enum node_type htype, NODE *head) return lit; } +#ifndef RIPPER static rb_parser_string_t * rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *orig) { @@ -12846,6 +12627,7 @@ rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *or copy->enc = orig->enc; return copy; } +#endif /* concat two string literals */ static NODE * @@ -13179,13 +12961,9 @@ gettable(struct parser_params *p, ID id, const YYLTYPE *loc) return NEW_FALSE(loc); case keyword__FILE__: { - rb_parser_string_t *file; - if (p->ruby_sourcefile_string) { - file = rb_parser_string_deep_copy(p, p->ruby_sourcefile_string); - } - else { - file = STRING_NEW0(); - } + VALUE file = p->ruby_sourcefile_string; + if (NIL_P(file)) + file = rb_str_new(0, 0); node = NEW_FILE(file, loc); } return node; @@ -13199,19 +12977,11 @@ gettable(struct parser_params *p, ID id, const YYLTYPE *loc) case ID_LOCAL: if (dyna_in_block(p) && dvar_defined_ref(p, id, &vidp)) { if (NUMPARAM_ID_P(id) && (numparam_nested_p(p) || it_used_p(p))) return 0; - if (id == p->cur_arg) { - compile_error(p, "circular argument reference - %"PRIsWARN, rb_id2str(id)); - return 0; - } if (vidp) *vidp |= LVAR_USED; node = NEW_DVAR(id, loc); return node; } if (local_id_ref(p, id, &vidp)) { - if (id == p->cur_arg) { - compile_error(p, "circular argument reference - %"PRIsWARN, rb_id2str(id)); - return 0; - } if (vidp) *vidp |= LVAR_USED; node = NEW_LVAR(id, loc); return node; @@ -13360,15 +13130,17 @@ new_regexp(struct parser_params *p, NODE *node, int options, const YYLTYPE *loc) case NODE_DSTR: nd_set_type(node, NODE_DREGX); nd_set_loc(node, loc); - RNODE_DREGX(node)->nd_cflag = options & RE_OPTION_MASK; - if (RNODE_DREGX(node)->string) reg_fragment_check(p, RNODE_DREGX(node)->string, options); - for (list = RNODE_DREGX(prev = node)->nd_next; list; list = RNODE_LIST(list->nd_next)) { + rb_node_dregx_t *const dreg = RNODE_DREGX(node); + dreg->as.nd_cflag = options & RE_OPTION_MASK; + if (dreg->string) reg_fragment_check(p, dreg->string, options); + prev = node; + for (list = dreg->nd_next; list; list = RNODE_LIST(list->nd_next)) { NODE *frag = list->nd_head; enum node_type type = nd_type(frag); if (type == NODE_STR || (type == NODE_DSTR && !RNODE_DSTR(frag)->nd_next)) { rb_parser_string_t *tail = RNODE_STR(frag)->string; if (reg_fragment_check(p, tail, options) && prev && RNODE_DREGX(prev)->string) { - rb_parser_string_t *lit = prev == node ? RNODE_DREGX(prev)->string : RNODE_STR(RNODE_LIST(prev)->nd_head)->string; + rb_parser_string_t *lit = prev == node ? dreg->string : RNODE_STR(RNODE_LIST(prev)->nd_head)->string; if (!literal_concat0(p, lit, tail)) { return NEW_NIL(loc); /* dummy node on error */ } @@ -13386,9 +13158,9 @@ new_regexp(struct parser_params *p, NODE *node, int options, const YYLTYPE *loc) prev = 0; } } - if (!RNODE_DREGX(node)->nd_next) { + if (!dreg->nd_next) { /* Check string is valid regex */ - reg_compile(p, RNODE_DREGX(node)->string, options); + reg_compile(p, dreg->string, options); } if (options & RE_OPTION_ONCE) { node = NEW_ONCE(node, loc); @@ -13449,8 +13221,8 @@ check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) else { st_data_t line; if (st_lookup(p->case_labels, (st_data_t)arg, &line)) { - rb_warning1("duplicated 'when' clause with line %d is ignored", - WARN_I((int)line)); + rb_warning2("'when' clause on line %d duplicates 'when' clause on line %d and is ignored", + WARN_I((int)nd_line(arg)), WARN_I((int)line)); return; } } @@ -13743,19 +13515,11 @@ assignable(struct parser_params *p, ID id, NODE *val, const YYLTYPE *loc) /* TODO: FIXME */ #ifndef RIPPER if (err) yyerror1(loc, err); +#else + if (err) set_value(assign_error(p, err, p->s_lvalue)); #endif return NEW_ERROR(loc); } -#ifdef RIPPER -static VALUE -ripper_assignable(struct parser_params *p, ID id, VALUE lhs) -{ - const char *err = 0; - assignable0(p, id, &err); - if (err) lhs = assign_error(p, err, lhs); - return lhs; -} -#endif static int is_private_local_id(struct parser_params *p, ID name) @@ -13838,10 +13602,10 @@ aryset_check(struct parser_params *p, NODE *args) } } if (kwds && nd_type_p(kwds, NODE_HASH) && !RNODE_HASH(kwds)->nd_brace) { - yyerror1(&kwds->nd_loc, "keyword arg given in index"); + yyerror1(&kwds->nd_loc, "keyword arg given in index assignment"); } if (block) { - yyerror1(&block->nd_loc, "block arg given in index"); + yyerror1(&block->nd_loc, "block arg given in index assignment"); } } @@ -13867,46 +13631,23 @@ attrset(struct parser_params *p, NODE *recv, ID atype, ID id, const YYLTYPE *loc return NEW_ATTRASGN(recv, id, 0, loc); } -static void -rb_backref_error(struct parser_params *p, NODE *node) -{ - switch (nd_type(node)) { - case NODE_NTH_REF: - compile_error(p, "Can't set variable $%ld", RNODE_NTH_REF(node)->nd_nth); - break; - case NODE_BACK_REF: - compile_error(p, "Can't set variable $%c", (int)RNODE_BACK_REF(node)->nd_nth); - break; - } -} - -#ifdef RIPPER static VALUE -defs(struct parser_params *p, VALUE head, VALUE args, VALUE bodystmt) -{ - return dispatch5(defs, - rb_ary_entry(head, 0), /* nd_recv */ - rb_ary_entry(head, 1), /* dot_or_colon */ - rb_ary_entry(head, 2), /* nd_mid */ - args, - bodystmt); -} - -static VALUE -backref_error(struct parser_params *p, NODE *node, VALUE expr) +rb_backref_error(struct parser_params *p, NODE *node) { - VALUE mesg = rb_str_new_cstr("Can't set variable "); +#ifndef RIPPER +# define ERR(...) (compile_error(p, __VA_ARGS__), Qtrue) +#else +# define ERR(...) rb_sprintf(__VA_ARGS__) +#endif switch (nd_type(node)) { case NODE_NTH_REF: - rb_str_catf(mesg, "$%ld", RNODE_NTH_REF(node)->nd_nth); - break; + return ERR("Can't set variable $%ld", RNODE_NTH_REF(node)->nd_nth); case NODE_BACK_REF: - rb_str_catf(mesg, "$%c", (int)RNODE_BACK_REF(node)->nd_nth); - break; + return ERR("Can't set variable $%c", (int)RNODE_BACK_REF(node)->nd_nth); } - return dispatch2(assign_error, mesg, expr); +#undef ERR + UNREACHABLE_RETURN(Qfalse); /* only called on syntax error */ } -#endif static NODE * arg_append(struct parser_params *p, NODE *node1, NODE *node2, const YYLTYPE *loc) @@ -14480,8 +14221,18 @@ cond0(struct parser_params *p, NODE *node, enum cond_type type, const YYLTYPE *l if (!top) break; RNODE_DOT2(node)->nd_beg = range_op(p, RNODE_DOT2(node)->nd_beg, loc); RNODE_DOT2(node)->nd_end = range_op(p, RNODE_DOT2(node)->nd_end, loc); - if (nd_type_p(node, NODE_DOT2)) nd_set_type(node,NODE_FLIP2); - else if (nd_type_p(node, NODE_DOT3)) nd_set_type(node, NODE_FLIP3); + switch (nd_type(node)) { + case NODE_DOT2: + nd_set_type(node,NODE_FLIP2); + rb_node_flip2_t *flip2 = RNODE_FLIP2(node); /* for debug info */ + (void)flip2; + break; + case NODE_DOT3: + nd_set_type(node, NODE_FLIP3); + rb_node_flip3_t *flip3 = RNODE_FLIP3(node); /* for debug info */ + (void)flip3; + break; + } break; case NODE_SYM: @@ -15047,20 +14798,16 @@ static NODE * const_decl(struct parser_params *p, NODE *path, const YYLTYPE *loc) { if (p->ctxt.in_def) { +#ifndef RIPPER yyerror1(loc, "dynamic constant assignment"); +#else + set_value(assign_error(p, "dynamic constant assignment", p->s_lvalue)); +#endif } return NEW_CDECL(0, 0, (path), p->ctxt.shareable_constant_value, loc); } -#ifdef RIPPER -static VALUE -ripper_const_decl(struct parser_params *p, VALUE path) -{ - if (p->ctxt.in_def) { - path = assign_error(p, "dynamic constant assignment", path); - } - return path; -} +#ifdef RIPPER static VALUE assign_error(struct parser_params *p, const char *mesg, VALUE a) { @@ -15068,12 +14815,6 @@ assign_error(struct parser_params *p, const char *mesg, VALUE a) ripper_error(p); return a; } - -static VALUE -var_field(struct parser_params *p, VALUE a) -{ - return dispatch1(var_field, a); -} #endif static NODE * @@ -15087,9 +14828,6 @@ new_bodystmt(struct parser_params *p, NODE *head, NODE *rescue, NODE *rescue_els result = NEW_RESCUE(head, rescue, rescue_else, &rescue_loc); nd_set_line(result, rescue->nd_loc.beg_pos.lineno); } - else if (rescue_else) { - result = block_append(p, result, rescue_else); - } if (ensure) { result = NEW_ENSURE(result, ensure, loc); } @@ -15768,7 +15506,7 @@ parser_initialize(struct parser_params *p) { /* note: we rely on TypedData_Make_Struct to set most fields to 0 */ p->command_start = TRUE; - p->ruby_sourcefile_string = NULL; + p->ruby_sourcefile_string = Qnil; p->lex.lpar_beg = -1; /* make lambda_beginning_p() == FALSE at first */ string_buffer_init(p); p->node_id = 0; @@ -15803,6 +15541,7 @@ rb_ruby_parser_mark(void *ptr) { struct parser_params *p = (struct parser_params*)ptr; + rb_gc_mark(p->ruby_sourcefile_string); #ifndef RIPPER rb_gc_mark(p->error_buffer); #else @@ -15833,10 +15572,6 @@ rb_ruby_parser_free(void *ptr) ruby_sized_xfree(p->tokenbuf, p->toksiz); } - if (p->ruby_sourcefile_string) { - rb_parser_string_free(p, p->ruby_sourcefile_string); - } - for (local = p->lvtbl; local; local = prev) { prev = local->prev; local_free(p, local); @@ -15859,6 +15594,9 @@ rb_ruby_parser_free(void *ptr) st_free_table(p->case_labels); } + xfree(p->lex.strterm); + p->lex.strterm = 0; + xfree(ptr); } @@ -16019,7 +15757,7 @@ rb_ruby_parser_ripper_initialize(rb_parser_t *p, rb_parser_lex_gets_func *gets, p->lex.gets = gets; p->lex.input = input; p->eofp = 0; - p->ruby_sourcefile_string = rb_str_to_parser_string(p, sourcefile_string); + p->ruby_sourcefile_string = sourcefile_string; p->ruby_sourcefile = sourcefile; p->ruby_sourceline = sourceline; } @@ -16039,7 +15777,7 @@ rb_ruby_parser_enc(rb_parser_t *p) VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p) { - return rb_str_new_parser_string(p->ruby_sourcefile_string); + return p->ruby_sourcefile_string; } int @@ -16176,7 +15914,7 @@ parser_compile_error(struct parser_params *p, const rb_code_location_t *loc, con va_start(ap, fmt); p->error_buffer = rb_syntax_error_append(p->error_buffer, - p->ruby_sourcefile_string ? rb_str_new_parser_string(p->ruby_sourcefile_string) : Qnil, + p->ruby_sourcefile_string, lineno, column, p->enc, fmt, ap); va_end(ap); @@ -16196,7 +15934,7 @@ count_char(const char *str, int c) * * "\"`class' keyword\"" => "`class' keyword" */ -RUBY_FUNC_EXPORTED size_t +size_t rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr) { if (*yystr == '"') { @@ -16260,7 +15998,7 @@ rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr) #endif #ifdef RIPPER -#define validate(x) ((x) = (x) == rb_ripper_none ? Qnil : x) +#define validate(x) (void)(x) static VALUE ripper_dispatch0(struct parser_params *p, ID mid) diff --git a/prism/config.yml b/prism/config.yml index 4e85a6f34d..0400f60e01 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -98,6 +98,7 @@ errors: - EXPECT_EXPRESSION_AFTER_SPLAT_HASH - EXPECT_EXPRESSION_AFTER_STAR - EXPECT_IDENT_REQ_PARAMETER + - EXPECT_IN_DELIMITER - EXPECT_LPAREN_REQ_PARAMETER - EXPECT_MESSAGE - EXPECT_RBRACKET @@ -113,6 +114,7 @@ errors: - EXPRESSION_NOT_WRITABLE_FILE - EXPRESSION_NOT_WRITABLE_LINE - EXPRESSION_NOT_WRITABLE_NIL + - EXPRESSION_NOT_WRITABLE_NUMBERED - EXPRESSION_NOT_WRITABLE_SELF - EXPRESSION_NOT_WRITABLE_TRUE - FLOAT_PARSE @@ -137,6 +139,7 @@ errors: - INVALID_BLOCK_EXIT - INVALID_CHARACTER - INVALID_ENCODING_MAGIC_COMMENT + - INVALID_ESCAPE_CHARACTER - INVALID_FLOAT_EXPONENT - INVALID_LOCAL_VARIABLE_READ - INVALID_LOCAL_VARIABLE_WRITE @@ -151,6 +154,7 @@ errors: - INVALID_NUMBER_UNDERSCORE_INNER - INVALID_NUMBER_UNDERSCORE_TRAILING - INVALID_PERCENT + - INVALID_PERCENT_EOF - INVALID_PRINTABLE_CHARACTER - INVALID_RETRY_AFTER_ELSE - INVALID_RETRY_AFTER_ENSURE @@ -182,15 +186,17 @@ errors: - NO_LOCAL_VARIABLE - NOT_EXPRESSION - NUMBER_LITERAL_UNDERSCORE + - NUMBERED_PARAMETER_INNER_BLOCK - NUMBERED_PARAMETER_IT - NUMBERED_PARAMETER_ORDINARY - - NUMBERED_PARAMETER_OUTER_SCOPE + - NUMBERED_PARAMETER_OUTER_BLOCK - OPERATOR_MULTI_ASSIGN - OPERATOR_WRITE_ARGUMENTS - OPERATOR_WRITE_BLOCK - PARAMETER_ASSOC_SPLAT_MULTI - PARAMETER_BLOCK_MULTI - PARAMETER_CIRCULAR + - PARAMETER_FORWARDING_AFTER_REST - PARAMETER_METHOD_NAME - PARAMETER_NAME_DUPLICATED - PARAMETER_NO_DEFAULT @@ -200,8 +206,8 @@ errors: - PARAMETER_SPLAT_MULTI - PARAMETER_STAR - PARAMETER_UNEXPECTED_FWD - - PARAMETER_WILD_LOOSE_COMMA - PARAMETER_UNEXPECTED_NO_KW + - PARAMETER_WILD_LOOSE_COMMA - PATTERN_CAPTURE_DUPLICATE - PATTERN_EXPRESSION_AFTER_BRACKET - PATTERN_EXPRESSION_AFTER_COMMA @@ -213,8 +219,10 @@ errors: - PATTERN_EXPRESSION_AFTER_PIPE - PATTERN_EXPRESSION_AFTER_RANGE - PATTERN_EXPRESSION_AFTER_REST + - PATTERN_HASH_IMPLICIT - PATTERN_HASH_KEY - PATTERN_HASH_KEY_DUPLICATE + - PATTERN_HASH_KEY_INTERPOLATED - PATTERN_HASH_KEY_LABEL - PATTERN_HASH_KEY_LOCALS - PATTERN_IDENT_AFTER_HROCKET @@ -228,6 +236,7 @@ errors: - REGEXP_INCOMPAT_CHAR_ENCODING - REGEXP_INVALID_UNICODE_RANGE - REGEXP_NON_ESCAPED_MBC + - REGEXP_PARSE_ERROR - REGEXP_TERM - REGEXP_UNKNOWN_OPTIONS - REGEXP_UTF8_CHAR_NON_UTF8_REGEXP @@ -268,6 +277,7 @@ errors: - WRITE_TARGET_UNEXPECTED - XSTRING_TERM warnings: + - AMBIGUOUS_BINARY_OPERATOR - AMBIGUOUS_FIRST_ARGUMENT_MINUS - AMBIGUOUS_FIRST_ARGUMENT_PLUS - AMBIGUOUS_PREFIX_AMPERSAND @@ -290,6 +300,7 @@ warnings: - KEYWORD_EOL - LITERAL_IN_CONDITION_DEFAULT - LITERAL_IN_CONDITION_VERBOSE + - SHAREABLE_CONSTANT_VALUE_LINE - SHEBANG_CARRIAGE_RETURN - UNEXPECTED_CARRIAGE_RETURN - UNREACHABLE_STATEMENT @@ -1207,9 +1218,9 @@ nodes: type: constant - name: write_name type: constant - - name: operator + - name: binary_operator type: constant - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node @@ -1365,11 +1376,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a class variable using an operator that isn't `=`. @@ -1475,11 +1486,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a constant using an operator that isn't `=`. @@ -1563,11 +1574,11 @@ nodes: - name: target type: node kind: ConstantPathNode - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a constant path using an operator that isn't `=`. @@ -1870,19 +1881,56 @@ nodes: fields: - name: index type: node + comment: | + The index expression for `for` loops. + + for i in a end + ^ - name: collection type: node + comment: | + The collection to iterate over. + + for i in a end + ^ - name: statements type: node? kind: StatementsNode + comment: | + Represents the body of statements to execute for each iteration of the loop. + + for i in a + foo(i) + ^^^^^^ + end - name: for_keyword_loc type: location + comment: | + The location of the `for` keyword. + + for i in a end + ^^^ - name: in_keyword_loc type: location + comment: | + The location of the `in` keyword. + + for i in a end + ^^ - name: do_keyword_loc type: location? + comment: | + The location of the `do` keyword, if present. + + for i in a do end + ^^ - name: end_keyword_loc type: location + comment: | + The location of the `end` keyword. + + for i in a end + ^^^ comment: | Represents the use of the `for` keyword. @@ -1934,11 +1982,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a global variable using an operator that isn't `=`. @@ -2270,9 +2318,9 @@ nodes: type: location - name: block type: node? - - name: operator + - name: binary_operator type: constant - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node @@ -2358,11 +2406,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to an instance variable using an operator that isn't `=`. @@ -2560,6 +2608,12 @@ nodes: `foo #{bar} baz` ^^^^^^^^^^^^^^^^ + - name: ItLocalVariableReadNode + comment: | + Represents reading from the implicit `it` local variable. + + -> { it } + ^^ - name: ItParametersNode comment: | Represents an implicit set of parameters through the use of the `it` keyword within a block or lambda. @@ -2638,13 +2692,13 @@ nodes: fields: - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - name: name type: constant - - name: operator + - name: binary_operator type: constant - name: depth type: uint32 @@ -2685,10 +2739,6 @@ nodes: _1 # name `:_1` - Finally, for the default `it` block parameter, the name is `0it`. This is to distinguish it from an `it` local variable that is explicitly declared. - - it # name `:0it` - - name: depth type: uint32 comment: | @@ -3079,6 +3129,8 @@ nodes: # On parsing error of `f(**kwargs, ...)` or `f(**nil, ...)`, the keyword_rest value is moved here: - KeywordRestParameterNode - NoKeywordsParameterNode + # On parsing error of `f(..., ...)`, the first forwarding parameter is moved here: + - ForwardingParameterNode - name: keywords type: node[] kind: @@ -3219,8 +3271,21 @@ nodes: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: RationalNode fields: - - name: numeric - type: node + - name: flags + type: flags + kind: IntegerBaseFlags + - name: numerator + type: integer + comment: | + The numerator of the rational number. + + 1.5r # numerator 3 + - name: denominator + type: integer + comment: | + The denominator of the rational number. + + 1.5r # denominator 2 comment: | Represents a rational number literal. @@ -3559,7 +3624,7 @@ nodes: ^^^^ - name: then_keyword_loc type: location? - comment: + comment: | The location of the `then` keyword, if present. unless cond then bar end diff --git a/prism/extension.c b/prism/extension.c index 84872914c4..091cac79ce 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -25,14 +25,14 @@ VALUE rb_cPrismParseLexResult; VALUE rb_cPrismDebugEncoding; -ID rb_option_id_command_line; -ID rb_option_id_encoding; -ID rb_option_id_filepath; -ID rb_option_id_frozen_string_literal; -ID rb_option_id_line; -ID rb_option_id_scopes; -ID rb_option_id_version; -ID rb_prism_source_id_for; +ID rb_id_option_command_line; +ID rb_id_option_encoding; +ID rb_id_option_filepath; +ID rb_id_option_frozen_string_literal; +ID rb_id_option_line; +ID rb_id_option_scopes; +ID rb_id_option_version; +ID rb_id_source_for; /******************************************************************************/ /* IO of Ruby code */ @@ -52,7 +52,7 @@ check_string(VALUE value) { // Check if the value is a string. If it's not, then raise a type error. if (!RB_TYPE_P(value, T_STRING)) { - rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(value)); + rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(value)); } // Otherwise, return the value as a C string. @@ -66,7 +66,7 @@ static void input_load_string(pm_string_t *input, VALUE string) { // Check if the string is a string. If it's not, then raise a type error. if (!RB_TYPE_P(string, T_STRING)) { - rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(string)); + rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(string)); } pm_string_constant_init(input, RSTRING_PTR(string), RSTRING_LEN(string)); @@ -135,15 +135,15 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { pm_options_t *options = (pm_options_t *) argument; ID key_id = SYM2ID(key); - if (key_id == rb_option_id_filepath) { + if (key_id == rb_id_option_filepath) { if (!NIL_P(value)) pm_options_filepath_set(options, check_string(value)); - } else if (key_id == rb_option_id_encoding) { + } else if (key_id == rb_id_option_encoding) { if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value))); - } else if (key_id == rb_option_id_line) { + } else if (key_id == rb_id_option_line) { if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value)); - } else if (key_id == rb_option_id_frozen_string_literal) { + } else if (key_id == rb_id_option_frozen_string_literal) { if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value)); - } else if (key_id == rb_option_id_version) { + } else if (key_id == rb_id_option_version) { if (!NIL_P(value)) { const char *version = check_string(value); @@ -151,9 +151,9 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value); } } - } else if (key_id == rb_option_id_scopes) { + } else if (key_id == rb_id_option_scopes) { if (!NIL_P(value)) build_options_scopes(options, value); - } else if (key_id == rb_option_id_command_line) { + } else if (key_id == rb_id_option_command_line) { if (!NIL_P(value)) { const char *string = check_string(value); uint8_t command_line = 0; @@ -600,7 +600,7 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input)); VALUE offsets = rb_ary_new(); - VALUE source = rb_funcall(rb_cPrismSource, rb_prism_source_id_for, 3, source_string, LONG2NUM(parser.start_line), offsets); + VALUE source = rb_funcall(rb_cPrismSource, rb_id_source_for, 3, source_string, LONG2NUM(parser.start_line), offsets); parse_lex_data_t parse_lex_data = { .source = source, @@ -762,6 +762,82 @@ parse(int argc, VALUE *argv, VALUE self) { } /** + * call-seq: + * Prism::parse_file(filepath, **options) -> ParseResult + * + * Parse the given file and return a ParseResult instance. For supported + * options, see Prism::parse. + */ +static VALUE +parse_file(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + file_options(argc, argv, &input, &options); + + VALUE value = parse_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return value; +} + +/** + * Parse the given input and return nothing. + */ +static void +profile_input(pm_string_t *input, const pm_options_t *options) { + pm_parser_t parser; + pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options); + + pm_node_t *node = pm_parse(&parser); + pm_node_destroy(&parser, node); + pm_parser_free(&parser); +} + +/** + * call-seq: + * Prism::profile(source, **options) -> nil + * + * Parse the given string and return nothing. This method is meant to allow + * profilers to avoid the overhead of reifying the AST to Ruby. For supported + * options, see Prism::parse. + */ +static VALUE +profile(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + string_options(argc, argv, &input, &options); + profile_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return Qnil; +} + +/** + * call-seq: + * Prism::profile_file(filepath, **options) -> nil + * + * Parse the given file and return nothing. This method is meant to allow + * profilers to avoid the overhead of reifying the AST to Ruby. For supported + * options, see Prism::parse. + */ +static VALUE +profile_file(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + file_options(argc, argv, &input, &options); + profile_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return Qnil; +} + +/** * An implementation of fgets that is suitable for use with Ruby IO objects. */ static char * @@ -816,27 +892,6 @@ parse_stream(int argc, VALUE *argv, VALUE self) { } /** - * call-seq: - * Prism::parse_file(filepath, **options) -> ParseResult - * - * Parse the given file and return a ParseResult instance. For supported - * options, see Prism::parse. - */ -static VALUE -parse_file(int argc, VALUE *argv, VALUE self) { - pm_string_t input; - pm_options_t options = { 0 }; - - file_options(argc, argv, &input, &options); - - VALUE value = parse_input(&input, &options); - pm_string_free(&input); - pm_options_free(&options); - - return value; -} - -/** * Parse the given input and return an array of Comment objects. */ static VALUE @@ -1035,310 +1090,13 @@ parse_file_failure_p(int argc, VALUE *argv, VALUE self) { } /******************************************************************************/ -/* Utility functions exposed to make testing easier */ -/******************************************************************************/ - -/** - * call-seq: - * Debug::named_captures(source) -> Array - * - * Returns an array of strings corresponding to the named capture groups in the - * given source string. If prism was unable to parse the regular expression, - * this function returns nil. - */ -static VALUE -named_captures(VALUE self, VALUE source) { - pm_string_list_t string_list = { 0 }; - - if (!pm_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, PM_ENCODING_UTF_8_ENTRY)) { - pm_string_list_free(&string_list); - return Qnil; - } - - VALUE names = rb_ary_new(); - for (size_t index = 0; index < string_list.length; index++) { - const pm_string_t *string = &string_list.strings[index]; - rb_ary_push(names, rb_str_new((const char *) pm_string_source(string), pm_string_length(string))); - } - - pm_string_list_free(&string_list); - return names; -} - -/** - * call-seq: - * Debug::integer_parse(source) -> [Integer, String] - * - * Parses the given source string and returns the integer it represents, as well - * as a decimal string representation. - */ -static VALUE -integer_parse(VALUE self, VALUE source) { - const uint8_t *start = (const uint8_t *) RSTRING_PTR(source); - size_t length = RSTRING_LEN(source); - - pm_integer_t integer = { 0 }; - pm_integer_parse(&integer, PM_INTEGER_BASE_UNKNOWN, start, start + length); - - pm_buffer_t buffer = { 0 }; - pm_integer_string(&buffer, &integer); - - VALUE string = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer)); - pm_buffer_free(&buffer); - - VALUE result = rb_ary_new_capa(2); - rb_ary_push(result, pm_integer_new(&integer)); - rb_ary_push(result, string); - pm_integer_free(&integer); - - return result; -} - -/** - * call-seq: - * Debug::memsize(source) -> { length: xx, memsize: xx, node_count: xx } - * - * Return a hash of information about the given source string's memory usage. - */ -static VALUE -memsize(VALUE self, VALUE string) { - pm_parser_t parser; - size_t length = RSTRING_LEN(string); - pm_parser_init(&parser, (const uint8_t *) RSTRING_PTR(string), length, NULL); - - pm_node_t *node = pm_parse(&parser); - pm_memsize_t memsize; - pm_node_memsize(node, &memsize); - - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - - VALUE result = rb_hash_new(); - rb_hash_aset(result, ID2SYM(rb_intern("length")), INT2FIX(length)); - rb_hash_aset(result, ID2SYM(rb_intern("memsize")), INT2FIX(memsize.memsize)); - rb_hash_aset(result, ID2SYM(rb_intern("node_count")), INT2FIX(memsize.node_count)); - return result; -} - -/** - * call-seq: - * Debug::profile_file(filepath) -> nil - * - * Parse the file, but do nothing with the result. This is used to profile the - * parser for memory and speed. - */ -static VALUE -profile_file(VALUE self, VALUE filepath) { - pm_string_t input; - - const char *checked = check_string(filepath); - Check_Type(filepath, T_STRING); - - if (!pm_string_mapped_init(&input, checked)) { -#ifdef _WIN32 - int e = rb_w32_map_errno(GetLastError()); -#else - int e = errno; -#endif - - rb_syserr_fail(e, checked); - } - - pm_options_t options = { 0 }; - pm_options_filepath_set(&options, checked); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options); - - pm_node_t *node = pm_parse(&parser); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_options_free(&options); - pm_string_free(&input); - - return Qnil; -} - -#ifndef PRISM_EXCLUDE_PRETTYPRINT - -/** - * call-seq: - * Debug::inspect_node(source) -> inspected - * - * Inspect the AST that represents the given source using the prism pretty print - * as opposed to the Ruby implementation. - */ -static VALUE -inspect_node(VALUE self, VALUE source) { - pm_string_t input; - input_load_string(&input, source); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL); - - pm_node_t *node = pm_parse(&parser); - pm_buffer_t buffer = { 0 }; - - pm_prettyprint(&buffer, &parser, node); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE string = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - - return string; -} - -#endif - -/** - * call-seq: - * Debug::format_errors(source, colorize) -> String - * - * Format the errors that are found when parsing the given source string. - */ -static VALUE -format_errors(VALUE self, VALUE source, VALUE colorize) { - pm_string_t input; - input_load_string(&input, source); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL); - - pm_node_t *node = pm_parse(&parser); - pm_buffer_t buffer = { 0 }; - - pm_parser_errors_format(&parser, &parser.error_list, &buffer, RTEST(colorize), true); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_string_free(&input); - - return result; -} - -/** - * call-seq: - * Debug::static_inspect(source) -> String - * - * Inspect the node as it would be inspected by the warnings used in static - * literal sets. - */ -static VALUE -static_inspect(int argc, VALUE *argv, VALUE self) { - pm_string_t input; - pm_options_t options = { 0 }; - string_options(argc, argv, &input, &options); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options); - - pm_node_t *program = pm_parse(&parser); - pm_node_t *node = ((pm_program_node_t *) program)->statements->body.nodes[0]; - - pm_buffer_t buffer = { 0 }; - pm_static_literal_inspect(&buffer, &parser.newline_list, parser.start_line, parser.encoding->name, node); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, program); - pm_parser_free(&parser); - pm_string_free(&input); - pm_options_free(&options); - - return result; -} - -/** - * call-seq: Debug::Encoding.all -> Array[Debug::Encoding] - * - * Return an array of all of the encodings that prism knows about. - */ -static VALUE -encoding_all(VALUE self) { - VALUE encodings = rb_ary_new(); - - for (size_t index = 0; index < PM_ENCODING_MAXIMUM; index++) { - const pm_encoding_t *encoding = &pm_encodings[index]; - - VALUE encoding_argv[] = { rb_str_new_cstr(encoding->name), encoding->multibyte ? Qtrue : Qfalse }; - rb_ary_push(encodings, rb_class_new_instance(2, encoding_argv, rb_cPrismDebugEncoding)); - } - - return encodings; -} - -static const pm_encoding_t * -encoding_find(VALUE name) { - const uint8_t *source = (const uint8_t *) RSTRING_PTR(name); - size_t length = RSTRING_LEN(name); - - const pm_encoding_t *encoding = pm_encoding_find(source, source + length); - if (encoding == NULL) { rb_raise(rb_eArgError, "Unknown encoding: %s", source); } - - return encoding; -} - -/** - * call-seq: Debug::Encoding.width(source) -> Integer - * - * Returns the width of the first character in the given string if it is valid - * in the encoding. If it is not, this function returns 0. - */ -static VALUE -encoding_char_width(VALUE self, VALUE name, VALUE value) { - return ULONG2NUM(encoding_find(name)->char_width((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value))); -} - -/** - * call-seq: Debug::Encoding.alnum?(source) -> true | false - * - * Returns true if the first character in the given string is an alphanumeric - * character in the encoding. - */ -static VALUE -encoding_alnum_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->alnum_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse; -} - -/** - * call-seq: Debug::Encoding.alpha?(source) -> true | false - * - * Returns true if the first character in the given string is an alphabetic - * character in the encoding. - */ -static VALUE -encoding_alpha_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->alpha_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse; -} - -/** - * call-seq: Debug::Encoding.upper?(source) -> true | false - * - * Returns true if the first character in the given string is an uppercase - * character in the encoding. - */ -static VALUE -encoding_isupper_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->isupper_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) ? Qtrue : Qfalse; -} - -/******************************************************************************/ /* Initialization of the extension */ /******************************************************************************/ /** * The init function that Ruby calls when loading this extension. */ -RUBY_FUNC_EXPORTED void +PRISM_EXPORTED_FUNCTION void Init_prism(void) { // Make sure that the prism library version matches the expected version. // Otherwise something was compiled incorrectly. @@ -1364,22 +1122,20 @@ Init_prism(void) { rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject); rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject); rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject); - rb_cPrismResult = rb_define_class_under(rb_cPrism, "Result", rb_cObject); rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cPrismResult); rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult); - // Intern all of the options that we support so that we don't have to do it - // every time we parse. - rb_option_id_command_line = rb_intern_const("command_line"); - rb_option_id_encoding = rb_intern_const("encoding"); - rb_option_id_filepath = rb_intern_const("filepath"); - rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal"); - rb_option_id_line = rb_intern_const("line"); - rb_option_id_scopes = rb_intern_const("scopes"); - rb_option_id_version = rb_intern_const("version"); - - rb_prism_source_id_for = rb_intern("for"); + // Intern all of the IDs eagerly that we support so that we don't have to do + // it every time we parse. + rb_id_option_command_line = rb_intern_const("command_line"); + rb_id_option_encoding = rb_intern_const("encoding"); + rb_id_option_filepath = rb_intern_const("filepath"); + rb_id_option_frozen_string_literal = rb_intern_const("frozen_string_literal"); + rb_id_option_line = rb_intern_const("line"); + rb_id_option_scopes = rb_intern_const("scopes"); + rb_id_option_version = rb_intern_const("version"); + rb_id_source_for = rb_intern("for"); /** * The version of the prism library. @@ -1390,8 +1146,10 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "lex", lex, -1); rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1); rb_define_singleton_method(rb_cPrism, "parse", parse, -1); - rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1); rb_define_singleton_method(rb_cPrism, "parse_file", parse_file, -1); + rb_define_singleton_method(rb_cPrism, "profile", profile, -1); + rb_define_singleton_method(rb_cPrism, "profile_file", profile_file, -1); + rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1); rb_define_singleton_method(rb_cPrism, "parse_comments", parse_comments, -1); rb_define_singleton_method(rb_cPrism, "parse_file_comments", parse_file_comments, -1); rb_define_singleton_method(rb_cPrism, "parse_lex", parse_lex, -1); @@ -1406,29 +1164,6 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); #endif - // Next, the functions that will be called by the parser to perform various - // internal tasks. We expose these to make them easier to test. - VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug"); - rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1); - rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1); - rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1); - rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1); - rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2); - rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1); - -#ifndef PRISM_EXCLUDE_PRETTYPRINT - rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); -#endif - - // Next, define the functions that are exposed through the private - // Debug::Encoding class. - rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject); - rb_define_singleton_method(rb_cPrismDebugEncoding, "all", encoding_all, 0); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_width", encoding_char_width, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_alnum?", encoding_alnum_char, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_alpha?", encoding_alpha_char, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_upper?", encoding_isupper_char, 2); - // Next, initialize the other APIs. Init_prism_api_node(); Init_prism_pack(); diff --git a/prism/extension.h b/prism/extension.h index 9134959783..fe1b238ff8 100644 --- a/prism/extension.h +++ b/prism/extension.h @@ -1,7 +1,7 @@ #ifndef PRISM_EXT_NODE_H #define PRISM_EXT_NODE_H -#define EXPECTED_PRISM_VERSION "0.27.0" +#define EXPECTED_PRISM_VERSION "0.30.0" #include <ruby.h> #include <ruby/encoding.h> diff --git a/prism/node.h b/prism/node.h index 8736e59a94..e8686a327c 100644 --- a/prism/node.h +++ b/prism/node.h @@ -57,27 +57,6 @@ void pm_node_list_free(pm_node_list_t *list); PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node); /** - * This struct stores the information gathered by the pm_node_memsize function. - * It contains both the memory footprint and additionally metadata about the - * shape of the tree. - */ -typedef struct { - /** The total memory footprint of the node and all of its children. */ - size_t memsize; - - /** The number of children the node has. */ - size_t node_count; -} pm_memsize_t; - -/** - * Calculates the memory footprint of a given node. - * - * @param node The node to calculate the memory footprint of. - * @param memsize The memory footprint of the node and all of its children. - */ -PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize); - -/** * Returns a string representation of the given node type. * * @param node_type The node type to convert to a string. diff --git a/prism/parser.h b/prism/parser.h index 8054e332f7..c5f8ab9df4 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -364,6 +364,9 @@ typedef enum { /** a rescue statement within a lambda expression */ PM_CONTEXT_LAMBDA_RESCUE, + /** the predicate clause of a loop statement */ + PM_CONTEXT_LOOP_PREDICATE, + /** the top level context */ PM_CONTEXT_MAIN, @@ -546,6 +549,17 @@ typedef struct pm_locals { pm_local_t *locals; } pm_locals_t; +/** The flags about scope parameters that can be set. */ +typedef uint8_t pm_scope_parameters_t; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40; + /** * This struct represents a node in a linked list of scopes. Some scopes can see * into their parent scopes, while others cannot. @@ -558,9 +572,18 @@ typedef struct pm_scope { pm_locals_t locals; /** + * This is a list of the implicit parameters contained within the block. + * These will be processed after the block is parsed to determine the kind + * of parameters node that should be used and to check if any errors need to + * be added. + */ + pm_node_list_t implicit_parameters; + + /** * This is a bitfield that indicates the parameters that are being used in - * this scope. It is a combination of the PM_SCOPE_PARAMS_* constants. There - * are three different kinds of parameters that can be used in a scope: + * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants. + * There are three different kinds of parameters that can be used in a + * scope: * * - Ordinary parameters (e.g., def foo(bar); end) * - Numbered parameters (e.g., def foo; _1; end) @@ -575,15 +598,7 @@ typedef struct pm_scope { * - def foo(&); end * - def foo(...); end */ - uint8_t parameters; - - /** - * An integer indicating the number of numbered parameters on this scope. - * This is necessary to determine if child blocks are allowed to use - * numbered parameters, and to pass information to consumers of the AST - * about how many numbered parameters exist. - */ - int8_t numbered_parameters; + pm_scope_parameters_t parameters; /** * The current state of constant shareability for this scope. This is @@ -598,20 +613,6 @@ typedef struct pm_scope { bool closed; } pm_scope_t; -static const uint8_t PM_SCOPE_PARAMETERS_NONE = 0x0; -static const uint8_t PM_SCOPE_PARAMETERS_ORDINARY = 0x1; -static const uint8_t PM_SCOPE_PARAMETERS_NUMBERED = 0x2; -static const uint8_t PM_SCOPE_PARAMETERS_IT = 0x4; -static const uint8_t PM_SCOPE_PARAMETERS_TYPE_MASK = 0x7; - -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x8; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x10; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x20; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x40; - -static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED = -1; -static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_NONE = 0; - /** * A struct that represents a stack of boolean values. */ diff --git a/prism/prism.c b/prism/prism.c index 44b4dc5ae5..41381d7312 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -14,180 +14,6 @@ pm_version(void) { */ #define PM_TAB_WHITESPACE_SIZE 8 -#ifndef PM_DEBUG_LOGGING -/** - * Debugging logging will provide you with additional debugging functions as - * well as automatically replace some functions with their debugging - * counterparts. - */ -#define PM_DEBUG_LOGGING 0 -#endif - -#if PM_DEBUG_LOGGING - -/******************************************************************************/ -/* Debugging */ -/******************************************************************************/ - -PRISM_ATTRIBUTE_UNUSED static const char * -debug_context(pm_context_t context) { - switch (context) { - case PM_CONTEXT_BEGIN: return "BEGIN"; - case PM_CONTEXT_BEGIN_ENSURE: return "BEGIN_ENSURE"; - case PM_CONTEXT_BEGIN_ELSE: return "BEGIN_ELSE"; - case PM_CONTEXT_BEGIN_RESCUE: return "BEGIN_RESCUE"; - case PM_CONTEXT_BLOCK_BRACES: return "BLOCK_BRACES"; - case PM_CONTEXT_BLOCK_KEYWORDS: return "BLOCK_KEYWORDS"; - case PM_CONTEXT_BLOCK_ENSURE: return "BLOCK_ENSURE"; - case PM_CONTEXT_BLOCK_ELSE: return "BLOCK_ELSE"; - case PM_CONTEXT_BLOCK_RESCUE: return "BLOCK_RESCUE"; - case PM_CONTEXT_CASE_IN: return "CASE_IN"; - case PM_CONTEXT_CASE_WHEN: return "CASE_WHEN"; - case PM_CONTEXT_CLASS: return "CLASS"; - case PM_CONTEXT_CLASS_ELSE: return "CLASS_ELSE"; - case PM_CONTEXT_CLASS_ENSURE: return "CLASS_ENSURE"; - case PM_CONTEXT_CLASS_RESCUE: return "CLASS_RESCUE"; - case PM_CONTEXT_DEF: return "DEF"; - case PM_CONTEXT_DEF_PARAMS: return "DEF_PARAMS"; - case PM_CONTEXT_DEF_ENSURE: return "DEF_ENSURE"; - case PM_CONTEXT_DEF_ELSE: return "DEF_ELSE"; - case PM_CONTEXT_DEF_RESCUE: return "DEF_RESCUE"; - case PM_CONTEXT_DEFAULT_PARAMS: return "DEFAULT_PARAMS"; - case PM_CONTEXT_DEFINED: return "DEFINED"; - case PM_CONTEXT_ELSE: return "ELSE"; - case PM_CONTEXT_ELSIF: return "ELSIF"; - case PM_CONTEXT_EMBEXPR: return "EMBEXPR"; - case PM_CONTEXT_FOR_INDEX: return "FOR_INDEX"; - case PM_CONTEXT_FOR: return "FOR"; - case PM_CONTEXT_IF: return "IF"; - case PM_CONTEXT_LAMBDA_BRACES: return "LAMBDA_BRACES"; - case PM_CONTEXT_LAMBDA_DO_END: return "LAMBDA_DO_END"; - case PM_CONTEXT_LAMBDA_ENSURE: return "LAMBDA_ENSURE"; - case PM_CONTEXT_LAMBDA_ELSE: return "LAMBDA_ELSE"; - case PM_CONTEXT_LAMBDA_RESCUE: return "LAMBDA_RESCUE"; - case PM_CONTEXT_MAIN: return "MAIN"; - case PM_CONTEXT_MODULE: return "MODULE"; - case PM_CONTEXT_MODULE_ELSE: return "MODULE_ELSE"; - case PM_CONTEXT_MODULE_ENSURE: return "MODULE_ENSURE"; - case PM_CONTEXT_MODULE_RESCUE: return "MODULE_RESCUE"; - case PM_CONTEXT_NONE: return "NONE"; - case PM_CONTEXT_PARENS: return "PARENS"; - case PM_CONTEXT_POSTEXE: return "POSTEXE"; - case PM_CONTEXT_PREDICATE: return "PREDICATE"; - case PM_CONTEXT_PREEXE: return "PREEXE"; - case PM_CONTEXT_RESCUE_MODIFIER: return "RESCUE_MODIFIER"; - case PM_CONTEXT_SCLASS: return "SCLASS"; - case PM_CONTEXT_SCLASS_ENSURE: return "SCLASS_ENSURE"; - case PM_CONTEXT_SCLASS_ELSE: return "SCLASS_ELSE"; - case PM_CONTEXT_SCLASS_RESCUE: return "SCLASS_RESCUE"; - case PM_CONTEXT_TERNARY: return "TERNARY"; - case PM_CONTEXT_UNLESS: return "UNLESS"; - case PM_CONTEXT_UNTIL: return "UNTIL"; - case PM_CONTEXT_WHILE: return "WHILE"; - } - return NULL; -} - -PRISM_ATTRIBUTE_UNUSED static void -debug_contexts(pm_parser_t *parser) { - pm_context_node_t *context_node = parser->current_context; - fprintf(stderr, "CONTEXTS: "); - - if (context_node != NULL) { - while (context_node != NULL) { - fprintf(stderr, "%s", debug_context(context_node->context)); - context_node = context_node->prev; - if (context_node != NULL) { - fprintf(stderr, " <- "); - } - } - } else { - fprintf(stderr, "NONE"); - } - - fprintf(stderr, "\n"); -} - -PRISM_ATTRIBUTE_UNUSED static void -debug_node(const pm_parser_t *parser, const pm_node_t *node) { - pm_buffer_t output_buffer = { 0 }; - pm_prettyprint(&output_buffer, parser, node); - - fprintf(stderr, "%.*s", (int) output_buffer.length, output_buffer.value); - pm_buffer_free(&output_buffer); -} - -PRISM_ATTRIBUTE_UNUSED static void -debug_lex_mode(pm_parser_t *parser) { - pm_lex_mode_t *lex_mode = parser->lex_modes.current; - bool first = true; - - while (lex_mode != NULL) { - if (first) { - first = false; - } else { - fprintf(stderr, " <- "); - } - - switch (lex_mode->mode) { - case PM_LEX_DEFAULT: fprintf(stderr, "DEFAULT"); break; - case PM_LEX_EMBEXPR: fprintf(stderr, "EMBEXPR"); break; - case PM_LEX_EMBVAR: fprintf(stderr, "EMBVAR"); break; - case PM_LEX_HEREDOC: fprintf(stderr, "HEREDOC"); break; - case PM_LEX_LIST: fprintf(stderr, "LIST (terminator=%c, interpolation=%d)", lex_mode->as.list.terminator, lex_mode->as.list.interpolation); break; - case PM_LEX_REGEXP: fprintf(stderr, "REGEXP (terminator=%c)", lex_mode->as.regexp.terminator); break; - case PM_LEX_STRING: fprintf(stderr, "STRING (terminator=%c, interpolation=%d)", lex_mode->as.string.terminator, lex_mode->as.string.interpolation); break; - } - - lex_mode = lex_mode->prev; - } - - fprintf(stderr, "\n"); -} - -PRISM_ATTRIBUTE_UNUSED static void -debug_state(pm_parser_t *parser) { - fprintf(stderr, "STATE: "); - bool first = true; - - if (parser->lex_state == PM_LEX_STATE_NONE) { - fprintf(stderr, "NONE\n"); - return; - } - -#define CHECK_STATE(state) \ - if (parser->lex_state & state) { \ - if (!first) fprintf(stderr, "|"); \ - fprintf(stderr, "%s", #state); \ - first = false; \ - } - - CHECK_STATE(PM_LEX_STATE_BEG) - CHECK_STATE(PM_LEX_STATE_END) - CHECK_STATE(PM_LEX_STATE_ENDARG) - CHECK_STATE(PM_LEX_STATE_ENDFN) - CHECK_STATE(PM_LEX_STATE_ARG) - CHECK_STATE(PM_LEX_STATE_CMDARG) - CHECK_STATE(PM_LEX_STATE_MID) - CHECK_STATE(PM_LEX_STATE_FNAME) - CHECK_STATE(PM_LEX_STATE_DOT) - CHECK_STATE(PM_LEX_STATE_CLASS) - CHECK_STATE(PM_LEX_STATE_LABEL) - CHECK_STATE(PM_LEX_STATE_LABELED) - CHECK_STATE(PM_LEX_STATE_FITEM) - -#undef CHECK_STATE - - fprintf(stderr, "\n"); -} - -PRISM_ATTRIBUTE_UNUSED static void -debug_token(pm_token_t * token) { - fprintf(stderr, "%s: \"%.*s\"\n", pm_token_type_human(token->type), (int) (token->end - token->start), token->start); -} - -#endif - // Macros for min/max. #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) @@ -423,7 +249,7 @@ lex_mode_pop(pm_parser_t *parser) { * This is the equivalent of IS_lex_state is CRuby. */ static inline bool -lex_state_p(pm_parser_t *parser, pm_lex_state_t state) { +lex_state_p(const pm_parser_t *parser, pm_lex_state_t state) { return parser->lex_state & state; } @@ -491,8 +317,52 @@ lex_state_set(pm_parser_t *parser, pm_lex_state_t state) { parser->lex_state = state; } +#ifndef PM_DEBUG_LOGGING +/** + * Debugging logging will print additional information to stdout whenever the + * lexer state changes. + */ +#define PM_DEBUG_LOGGING 0 +#endif + #if PM_DEBUG_LOGGING -static inline void +PRISM_ATTRIBUTE_UNUSED static void +debug_state(pm_parser_t *parser) { + fprintf(stderr, "STATE: "); + bool first = true; + + if (parser->lex_state == PM_LEX_STATE_NONE) { + fprintf(stderr, "NONE\n"); + return; + } + +#define CHECK_STATE(state) \ + if (parser->lex_state & state) { \ + if (!first) fprintf(stderr, "|"); \ + fprintf(stderr, "%s", #state); \ + first = false; \ + } + + CHECK_STATE(PM_LEX_STATE_BEG) + CHECK_STATE(PM_LEX_STATE_END) + CHECK_STATE(PM_LEX_STATE_ENDARG) + CHECK_STATE(PM_LEX_STATE_ENDFN) + CHECK_STATE(PM_LEX_STATE_ARG) + CHECK_STATE(PM_LEX_STATE_CMDARG) + CHECK_STATE(PM_LEX_STATE_MID) + CHECK_STATE(PM_LEX_STATE_FNAME) + CHECK_STATE(PM_LEX_STATE_DOT) + CHECK_STATE(PM_LEX_STATE_CLASS) + CHECK_STATE(PM_LEX_STATE_LABEL) + CHECK_STATE(PM_LEX_STATE_LABELED) + CHECK_STATE(PM_LEX_STATE_FITEM) + +#undef CHECK_STATE + + fprintf(stderr, "\n"); +} + +static void debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * caller_name, int line_number) { fprintf(stderr, "Caller: %s:%d\nPrevious: ", caller_name, line_number); debug_state(parser); @@ -708,7 +578,7 @@ pm_parser_scope_push(pm_parser_t *parser, bool closed) { .previous = parser->current_scope, .locals = { 0 }, .parameters = PM_SCOPE_PARAMETERS_NONE, - .numbered_parameters = PM_SCOPE_NUMBERED_PARAMETERS_NONE, + .implicit_parameters = { 0 }, .shareable_constant = (closed || parser->current_scope == NULL) ? PM_SCOPE_SHAREABLE_CONSTANT_NONE : parser->current_scope->shareable_constant, .closed = closed }; @@ -1183,6 +1053,31 @@ pm_check_value_expression(pm_node_t *node) { return NULL; case PM_BEGIN_NODE: { pm_begin_node_t *cast = (pm_begin_node_t *) node; + + if (cast->statements == NULL && cast->ensure_clause != NULL) { + node = (pm_node_t *) cast->ensure_clause; + } + else { + if (cast->rescue_clause != NULL) { + if (cast->rescue_clause->statements == NULL) { + return NULL; + } + else if (cast->else_clause != NULL) { + node = (pm_node_t *) cast->else_clause; + } + else { + node = (pm_node_t *) cast->statements; + } + } + else { + node = (pm_node_t *) cast->statements; + } + } + + break; + } + case PM_ENSURE_NODE: { + pm_ensure_node_t *cast = (pm_ensure_node_t *) node; node = (pm_node_t *) cast->statements; break; } @@ -1630,7 +1525,7 @@ not_provided(pm_parser_t *parser) { return (pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }; } -#define PM_LOCATION_NULL_VALUE(parser) ((pm_location_t) { .start = parser->start, .end = parser->start }) +#define PM_LOCATION_NULL_VALUE(parser) ((pm_location_t) { .start = (parser)->start, .end = (parser)->start }) #define PM_LOCATION_TOKEN_VALUE(token) ((pm_location_t) { .start = (token)->start, .end = (token)->end }) #define PM_LOCATION_NODE_VALUE(node) ((pm_location_t) { .start = (node)->location.start, .end = (node)->location.end }) #define PM_LOCATION_NODE_BASE_VALUE(node) ((pm_location_t) { .start = (node)->base.location.start, .end = (node)->base.location.end }) @@ -1758,7 +1653,7 @@ char_is_identifier_utf8(const uint8_t *b, const uint8_t *end) { * it's important that it be as fast as possible. */ static inline size_t -char_is_identifier(pm_parser_t *parser, const uint8_t *b) { +char_is_identifier(const pm_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { size_t width; if ((width = parser->encoding->alnum_char(b, parser->end - b)) != 0) { @@ -2827,8 +2722,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location.start = parser->start; - node->base.location.end = parser->start; + node->base.location = PM_LOCATION_NULL_VALUE(parser); node->arguments = arguments; node->name = name; @@ -3080,8 +2974,8 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3119,8 +3013,8 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = target->block, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3464,9 +3358,9 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -3580,9 +3474,9 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat } }, .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -3707,9 +3601,9 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -4291,7 +4185,7 @@ pm_float_node_imaginary_create(pm_parser_t *parser, const pm_token_t *token) { } /** - * Allocate and initialize a new FloatNode node from a FLOAT_RATIONAL token. + * Allocate and initialize a new RationalNode node from a FLOAT_RATIONAL token. */ static pm_rational_node_t * pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) { @@ -4301,16 +4195,44 @@ pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) { *node = (pm_rational_node_t) { { .type = PM_RATIONAL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, + .flags = PM_INTEGER_BASE_FLAGS_DECIMAL | PM_NODE_FLAG_STATIC_LITERAL, .location = PM_LOCATION_TOKEN_VALUE(token) }, - .numeric = (pm_node_t *) pm_float_node_create(parser, &((pm_token_t) { - .type = PM_TOKEN_FLOAT, - .start = token->start, - .end = token->end - 1 - })) + .numerator = { 0 }, + .denominator = { 0 } }; + const uint8_t *start = token->start; + const uint8_t *end = token->end - 1; // r + + while (start < end && *start == '0') start++; // 0.1 -> .1 + while (end > start && end[-1] == '0') end--; // 1.0 -> 1. + + size_t length = (size_t) (end - start); + if (length == 1) { + node->denominator.value = 1; + return node; + } + + const uint8_t *point = memchr(start, '.', length); + assert(point && "should have a decimal point"); + + uint8_t *digits = malloc(length); + if (digits == NULL) { + fputs("[pm_float_node_rational_create] Failed to allocate memory", stderr); + abort(); + } + + memcpy(digits, start, (unsigned long) (point - start)); + memcpy(digits + (point - start), point + 1, (unsigned long) (end - point - 1)); + pm_integer_parse(&node->numerator, PM_INTEGER_BASE_DEFAULT, digits, digits + length - 1); + + digits[0] = '1'; + if (end - point > 1) memset(digits + 1, '0', (size_t) (end - point - 1)); + pm_integer_parse(&node->denominator, PM_INTEGER_BASE_DEFAULT, digits, digits + (end - point)); + free(digits); + + pm_integers_reduce(&node->numerator, &node->denominator); return node; } @@ -4560,9 +4482,9 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta }, .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -4621,7 +4543,7 @@ pm_global_variable_read_node_synthesized_create(pm_parser_t *parser, pm_constant *node = (pm_global_variable_read_node_t) { { .type = PM_GLOBAL_VARIABLE_READ_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, .name = name }; @@ -4663,11 +4585,11 @@ pm_global_variable_write_node_synthesized_create(pm_parser_t *parser, pm_constan *node = (pm_global_variable_write_node_t) { { .type = PM_GLOBAL_VARIABLE_WRITE_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, .name = name, - .name_loc = { .start = parser->start, .end = parser->start }, - .operator_loc = { .start = parser->start, .end = parser->start }, + .name_loc = PM_LOCATION_NULL_VALUE(parser), + .operator_loc = PM_LOCATION_NULL_VALUE(parser), .value = value }; @@ -4944,7 +4866,7 @@ pm_integer_node_imaginary_create(pm_parser_t *parser, pm_node_flags_t base, cons } /** - * Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL + * Allocate and initialize a new RationalNode node from an INTEGER_RATIONAL * token. */ static pm_rational_node_t * @@ -4955,16 +4877,24 @@ pm_integer_node_rational_create(pm_parser_t *parser, pm_node_flags_t base, const *node = (pm_rational_node_t) { { .type = PM_RATIONAL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, + .flags = base | PM_NODE_FLAG_STATIC_LITERAL, .location = PM_LOCATION_TOKEN_VALUE(token) }, - .numeric = (pm_node_t *) pm_integer_node_create(parser, base, &((pm_token_t) { - .type = PM_TOKEN_INTEGER, - .start = token->start, - .end = token->end - 1 - })) + .numerator = { 0 }, + .denominator = { .value = 1, 0 } }; + pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL; + switch (base) { + case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break; + case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break; + case PM_INTEGER_BASE_FLAGS_DECIMAL: break; + case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break; + default: assert(false && "unreachable"); break; + } + + pm_integer_parse(&node->numerator, integer_base, token->start, token->end - 1); + return node; } @@ -5068,9 +4998,9 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -5463,6 +5393,23 @@ pm_interpolated_xstring_node_closing_set(pm_interpolated_x_string_node_t *node, } /** + * Create a local variable read that is reading the implicit 'it' variable. + */ +static pm_it_local_variable_read_node_t * +pm_it_local_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name) { + pm_it_local_variable_read_node_t *node = PM_ALLOC_NODE(parser, pm_it_local_variable_read_node_t); + + *node = (pm_it_local_variable_read_node_t) { + { + .type = PM_IT_LOCAL_VARIABLE_READ_NODE, + .location = PM_LOCATION_TOKEN_VALUE(name) + } + }; + + return node; +} + +/** * Allocate and initialize a new ItParametersNode node. */ static pm_it_parameters_node_t * @@ -5664,10 +5611,10 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar } }, .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, .name = name, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), .depth = depth }; @@ -5775,28 +5722,6 @@ pm_token_is_it(const uint8_t *start, const uint8_t *end) { } /** - * Returns true if the given node is `it` default parameter. - */ -static inline bool -pm_node_is_it(pm_parser_t *parser, pm_node_t *node) { - // Check if it's a local variable reference - if (node->type != PM_CALL_NODE) { - return false; - } - - // Check if it's a variable call - pm_call_node_t *call_node = (pm_call_node_t *) node; - if (!PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - return false; - } - - // Check if it's called `it` - pm_constant_id_t id = ((pm_call_node_t *)node)->name; - pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, id); - return pm_token_is_it(constant->start, constant->start + constant->length); -} - -/** * Returns true if the given bounds comprise a numbered parameter (i.e., they * are of the form /^_\d$/). */ @@ -6946,7 +6871,7 @@ pm_statements_node_body_append(pm_parser_t *parser, pm_statements_node_t *node, case PM_REDO_NODE: case PM_RETRY_NODE: case PM_RETURN_NODE: - pm_parser_warn_node(parser, previous, PM_WARN_UNREACHABLE_STATEMENT); + pm_parser_warn_node(parser, statement, PM_WARN_UNREACHABLE_STATEMENT); break; default: break; @@ -7355,9 +7280,9 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { { .type = PM_SYMBOL_NODE, .flags = PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, - .value_loc = { .start = parser->start, .end = parser->start }, + .value_loc = PM_LOCATION_NULL_VALUE(parser), .unescaped = { 0 } }; @@ -7758,10 +7683,10 @@ pm_while_node_synthesized_create(pm_parser_t *parser, pm_node_t *predicate, pm_s *node = (pm_while_node_t) { { .type = PM_WHILE_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, - .keyword_loc = { .start = parser->start, .end = parser->start }, - .closing_loc = { .start = parser->start, .end = parser->start }, + .keyword_loc = PM_LOCATION_NULL_VALUE(parser), + .closing_loc = PM_LOCATION_NULL_VALUE(parser), .predicate = predicate, .statements = statements }; @@ -7917,51 +7842,6 @@ pm_parser_local_add_constant(pm_parser_t *parser, const char *start, size_t leng } /** - * Create a local variable read that is reading the implicit 'it' variable. - */ -static pm_local_variable_read_node_t * -pm_local_variable_read_node_create_it(pm_parser_t *parser, const pm_token_t *name) { - if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_ORDINARY) { - pm_parser_err_token(parser, name, PM_ERR_IT_NOT_ALLOWED_ORDINARY); - return NULL; - } - - if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED) { - pm_parser_err_token(parser, name, PM_ERR_IT_NOT_ALLOWED_NUMBERED); - return NULL; - } - - parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_IT; - - pm_constant_id_t name_id = pm_parser_constant_id_constant(parser, "0it", 3); - pm_parser_local_add(parser, name_id, name->start, name->end, 0); - - return pm_local_variable_read_node_create_constant_id(parser, name, name_id, 0, false); -} - -/** - * Convert a `it` variable call node to a node for `it` default parameter. - */ -static pm_node_t * -pm_node_check_it(pm_parser_t *parser, pm_node_t *node) { - if ( - (parser->version != PM_OPTIONS_VERSION_CRUBY_3_3) && - !parser->current_scope->closed && - (parser->current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED) && - pm_node_is_it(parser, node) - ) { - pm_local_variable_read_node_t *read = pm_local_variable_read_node_create_it(parser, &parser->previous); - - if (read != NULL) { - pm_node_destroy(parser, node); - node = (pm_node_t *) read; - } - } - - return node; -} - -/** * Add a parameter name to the current scope and check whether the name of the * parameter is unique or not. * @@ -7996,6 +7876,7 @@ pm_parser_scope_pop(pm_parser_t *parser) { pm_scope_t *scope = parser->current_scope; parser->current_scope = scope->previous; pm_locals_free(&scope->locals); + pm_node_list_free(&scope->implicit_parameters); xfree(scope); } @@ -8067,7 +7948,7 @@ pm_do_loop_stack_p(pm_parser_t *parser) { * is beyond the end of the source then return '\0'. */ static inline uint8_t -peek_at(pm_parser_t *parser, const uint8_t *cursor) { +peek_at(const pm_parser_t *parser, const uint8_t *cursor) { if (cursor < parser->end) { return *cursor; } else { @@ -8090,7 +7971,7 @@ peek_offset(pm_parser_t *parser, ptrdiff_t offset) { * that position is beyond the end of the source then return '\0'. */ static inline uint8_t -peek(pm_parser_t *parser) { +peek(const pm_parser_t *parser) { return peek_at(parser, parser->current.end); } @@ -8156,6 +8037,14 @@ next_newline(const uint8_t *cursor, ptrdiff_t length) { } /** + * This is equivalent to the predicate of warn_balanced in CRuby. + */ +static inline bool +ambiguous_operator_p(const pm_parser_t *parser, bool space_seen) { + return !lex_state_p(parser, PM_LEX_STATE_CLASS | PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME | PM_LEX_STATE_ENDFN) && space_seen && !pm_char_is_whitespace(peek(parser)); +} + +/** * Here we're going to check if this is a "magic" comment, and perform whatever * actions are necessary for it here. */ @@ -8394,7 +8283,12 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // If we have hit a ractor pragma, attempt to lex that. uint32_t value_length = (uint32_t) (value_end - value_start); if (key_length == 24 && pm_strncasecmp(key_source, (const uint8_t *) "shareable_constant_value", 24) == 0) { - if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "none", 4) == 0) { + const uint8_t *cursor = parser->current.start; + while ((cursor > parser->start) && ((cursor[-1] == ' ') || (cursor[-1] == '\t'))) cursor--; + + if (!((cursor == parser->start) || (cursor[-1] == '\n'))) { + pm_parser_warn_token(parser, &parser->current, PM_WARN_SHAREABLE_CONSTANT_VALUE_LINE); + } else if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "none", 4) == 0) { pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_NONE); } else if (value_length == 7 && pm_strncasecmp(value_start, (const uint8_t *) "literal", 7) == 0) { pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_LITERAL); @@ -8468,6 +8362,8 @@ context_terminator(pm_context_t context, pm_token_t *token) { case PM_CONTEXT_MODULE_ENSURE: case PM_CONTEXT_SCLASS_ENSURE: return token->type == PM_TOKEN_KEYWORD_END; + case PM_CONTEXT_LOOP_PREDICATE: + return token->type == PM_TOKEN_KEYWORD_DO || token->type == PM_TOKEN_KEYWORD_THEN; case PM_CONTEXT_FOR_INDEX: return token->type == PM_TOKEN_KEYWORD_IN; case PM_CONTEXT_CASE_WHEN: @@ -8640,6 +8536,7 @@ context_human(pm_context_t context) { case PM_CONTEXT_IF: return "if statement"; case PM_CONTEXT_LAMBDA_BRACES: return "'{'..'}' lambda block"; case PM_CONTEXT_LAMBDA_DO_END: return "'do'..'end' lambda block"; + case PM_CONTEXT_LOOP_PREDICATE: return "loop predicate"; case PM_CONTEXT_MAIN: return "top level context"; case PM_CONTEXT_MODULE: return "module definition"; case PM_CONTEXT_PARENS: return "parentheses"; @@ -8990,8 +8887,8 @@ lex_global_variable(pm_parser_t *parser) { // If we get here, then we have a $ followed by something that // isn't recognized as a global variable. pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; - size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + const uint8_t *end = parser->current.end + parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_FORMAT(parser, parser->current.start, end, diag_id, (int) (end - parser->current.start), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -9362,12 +9259,20 @@ escape_hexadecimal_digit(const uint8_t value) { * validated. */ static inline uint32_t -escape_unicode(const uint8_t *string, size_t length) { +escape_unicode(pm_parser_t *parser, const uint8_t *string, size_t length) { uint32_t value = 0; for (size_t index = 0; index < length; index++) { if (index != 0) value <<= 4; value |= escape_hexadecimal_digit(string[index]); } + + // Here we're going to verify that the value is actually a valid Unicode + // codepoint and not a surrogate pair. + if (value >= 0xD800 && value <= 0xDFFF) { + pm_parser_err(parser, string, string + length, PM_ERR_ESCAPE_INVALID_UNICODE); + return 0xFFFD; + } + return value; } @@ -9376,7 +9281,7 @@ escape_unicode(const uint8_t *string, size_t length) { */ static inline uint8_t escape_byte(uint8_t value, const uint8_t flags) { - if (flags & PM_ESCAPE_FLAG_CONTROL) value &= 0x1f; + if (flags & PM_ESCAPE_FLAG_CONTROL) value &= 0x9f; if (flags & PM_ESCAPE_FLAG_META) value |= 0x80; return value; } @@ -9476,22 +9381,7 @@ escape_write_escape_encoded(pm_parser_t *parser, pm_buffer_t *buffer) { static inline void escape_write_byte(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expression_buffer, uint8_t flags, uint8_t byte) { if (flags & PM_ESCAPE_FLAG_REGEXP) { - pm_buffer_append_bytes(regular_expression_buffer, (const uint8_t *) "\\x", 2); - - uint8_t byte1 = (uint8_t) ((byte >> 4) & 0xF); - uint8_t byte2 = (uint8_t) (byte & 0xF); - - if (byte1 >= 0xA) { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) ((byte1 - 0xA) + 'A')); - } else { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte1 + '0')); - } - - if (byte2 >= 0xA) { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte2 - 0xA + 'A')); - } else { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte2 + '0')); - } + pm_buffer_append_format(regular_expression_buffer, "\\x%02X", byte); } escape_write_byte_encoded(parser, buffer, byte); @@ -9526,57 +9416,57 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre switch (peek(parser)) { case '\\': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\\', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\\', flags)); return; } case '\'': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\'', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\'', flags)); return; } case 'a': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\a', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\a', flags)); return; } case 'b': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\b', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\b', flags)); return; } case 'e': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\033', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\033', flags)); return; } case 'f': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\f', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\f', flags)); return; } case 'n': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\n', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\n', flags)); return; } case 'r': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\r', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\r', flags)); return; } case 's': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte(' ', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte(' ', flags)); return; } case 't': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\t', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\t', flags)); return; } case 'v': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\v', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\v', flags)); return; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { @@ -9593,7 +9483,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre } } - escape_write_byte_encoded(parser, buffer, value); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, value); return; } case 'x': { @@ -9612,8 +9502,13 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; } + value = escape_byte(value, flags); if (flags & PM_ESCAPE_FLAG_REGEXP) { - pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); + if (flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) { + pm_buffer_append_format(regular_expression_buffer, "\\x%02X", value); + } else { + pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); + } } escape_write_byte_encoded(parser, buffer, value); @@ -9645,7 +9540,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE_LONG); } else if (hexadecimal_length == 0) { // there are not hexadecimal characters - pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); return; } @@ -9655,7 +9551,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre extra_codepoints_start = unicode_start; } - uint32_t value = escape_unicode(unicode_start, hexadecimal_length); + uint32_t value = escape_unicode(parser, unicode_start, hexadecimal_length); escape_write_unicode(parser, buffer, flags, unicode_start, parser->current.end, value); parser->current.end += pm_strspn_whitespace(parser->current.end, parser->end - parser->current.end); @@ -9680,7 +9576,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre size_t length = pm_strspn_hexadecimal_digit(parser->current.end, MIN(parser->end - parser->current.end, 4)); if (length == 4) { - uint32_t value = escape_unicode(parser->current.end, 4); + uint32_t value = escape_unicode(parser, parser->current.end, 4); if (flags & PM_ESCAPE_FLAG_REGEXP) { pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end + 4 - start)); @@ -9716,6 +9612,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_CONTROL); return; case ' ': @@ -9743,7 +9645,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre case 'C': { parser->current.end++; if (peek(parser) != '-') { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9766,6 +9669,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_CONTROL); return; case ' ': @@ -9780,7 +9689,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; default: { if (!char_is_ascii_printable(peeked)) { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9793,7 +9703,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre case 'M': { parser->current.end++; if (peek(parser) != '-') { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); return; } @@ -9811,6 +9722,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_META); return; case ' ': @@ -9825,7 +9742,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; default: if (!char_is_ascii_printable(peeked)) { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); return; } @@ -10786,6 +10704,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_USTAR_STAR; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR_STAR; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10809,6 +10729,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_USTAR; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10925,6 +10847,7 @@ parser_lex(pm_parser_t *parser) { // If we have quotes, then we're going to go until we find the // end quote. while ((parser->current.end < parser->end) && quote != (pm_heredoc_quote_t) (*parser->current.end)) { + if (*parser->current.end == '\r' || *parser->current.end == '\n') break; parser->current.end++; } } @@ -10982,6 +10905,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_LESS_LESS_EQUAL); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); + } + if (lex_state_operator_p(parser)) { lex_state_set(parser, PM_LEX_STATE_ARG); } else { @@ -11095,6 +11022,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_UAMPERSAND; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_UAMPERSAND; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -11169,6 +11098,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_UPLUS); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); + } + lex_state_set(parser, PM_LEX_STATE_BEG); LEX(PM_TOKEN_PLUS); } @@ -11206,6 +11139,10 @@ parser_lex(pm_parser_t *parser) { LEX(pm_char_is_decimal_digit(peek(parser)) ? PM_TOKEN_UMINUS_NUM : PM_TOKEN_UMINUS); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); + } + lex_state_set(parser, PM_LEX_STATE_BEG); LEX(PM_TOKEN_MINUS); } @@ -11304,6 +11241,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_REGEXP_BEGIN); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); + } + if (lex_state_operator_p(parser)) { lex_state_set(parser, PM_LEX_STATE_ARG); } else { @@ -11339,7 +11280,7 @@ parser_lex(pm_parser_t *parser) { // operator because we don't want to move into the string // lex mode unnecessarily. if ((lex_state_beg_p(parser) || lex_state_arg_p(parser)) && (parser->current.end >= parser->end)) { - pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT); + pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT_EOF); LEX(PM_TOKEN_PERCENT); } @@ -11358,10 +11299,7 @@ parser_lex(pm_parser_t *parser) { const uint8_t delimiter = pm_lex_percent_delimiter(parser); lex_mode_push_string(parser, true, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter)); - - if (parser->current.end < parser->end) { - LEX(PM_TOKEN_STRING_BEGIN); - } + LEX(PM_TOKEN_STRING_BEGIN); } // Delimiters for %-literals cannot be alphanumeric. We @@ -11488,6 +11426,10 @@ parser_lex(pm_parser_t *parser) { } } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); + } + lex_state_set(parser, lex_state_operator_p(parser) ? PM_LEX_STATE_ARG : PM_LEX_STATE_BEG); LEX(PM_TOKEN_PERCENT); } @@ -12298,9 +12240,10 @@ parser_lex(pm_parser_t *parser) { // If we are immediately following a newline and we have hit the // terminator, then we need to return the ending of the heredoc. - if (!line_continuation && current_token_starts_line(parser)) { + if (current_token_starts_line(parser)) { const uint8_t *start = parser->current.start; - if (start + ident_length <= parser->end) { + + if (!line_continuation && (start + ident_length <= parser->end)) { const uint8_t *newline = next_newline(start, parser->end - start); const uint8_t *ident_end = newline; const uint8_t *terminator_end = newline; @@ -12456,11 +12399,8 @@ parser_lex(pm_parser_t *parser) { } parser->current.end = breakpoint + 1; - - if (!was_line_continuation) { - pm_token_buffer_flush(parser, &token_buffer); - LEX(PM_TOKEN_STRING_CONTENT); - } + pm_token_buffer_flush(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); } // Otherwise we hit a newline and it wasn't followed by @@ -13096,10 +13036,39 @@ parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) { } /** + * When an implicit local variable is written to or targeted, it becomes a + * regular, named local variable. This function removes it from the list of + * implicit parameters when that happens. + */ +static void +parse_target_implicit_parameter(pm_parser_t *parser, pm_node_t *node) { + pm_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters; + + for (size_t index = 0; index < implicit_parameters->size; index++) { + if (implicit_parameters->nodes[index] == node) { + // If the node is not the last one in the list, we need to shift the + // remaining nodes down to fill the gap. This is extremely unlikely + // to happen. + if (index != implicit_parameters->size - 1) { + memcpy(&implicit_parameters->nodes[index], &implicit_parameters->nodes[index + 1], (implicit_parameters->size - index - 1) * sizeof(pm_node_t *)); + } + + implicit_parameters->size--; + break; + } + } +} + +/** * Convert the given node into a valid target node. + * + * @param multiple Whether or not this target is part of a larger set of + * targets. If it is, then the &. operator is not allowed. + * @param splat Whether or not this target is a child of a splat target. If it + * is, then fewer patterns are allowed. */ static pm_node_t * -parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple) { +parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_parent) { switch (PM_NODE_TYPE(target)) { case PM_MISSING_NODE: return target; @@ -13145,7 +13114,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple) { target->type = PM_GLOBAL_VARIABLE_TARGET_NODE; return target; case PM_LOCAL_VARIABLE_READ_NODE: { - pm_refute_numbered_parameter(parser, target->location.start, target->location.end); + if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, target->location.start); + parse_target_implicit_parameter(parser, target); + } const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) target; uint32_t name = cast->name; @@ -13157,17 +13129,32 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple) { return target; } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); + pm_node_t *node = (pm_node_t *) pm_local_variable_target_node_create(parser, &target->location, name, 0); + + parse_target_implicit_parameter(parser, target); + pm_node_destroy(parser, target); + + return node; + } case PM_INSTANCE_VARIABLE_READ_NODE: assert(sizeof(pm_instance_variable_target_node_t) == sizeof(pm_instance_variable_read_node_t)); target->type = PM_INSTANCE_VARIABLE_TARGET_NODE; return target; case PM_MULTI_TARGET_NODE: + if (splat_parent) { + // Multi target is not accepted in all positions. If this is one + // of them, then we need to add an error. + pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_UNEXPECTED); + } + return target; case PM_SPLAT_NODE: { pm_splat_node_t *splat = (pm_splat_node_t *) target; if (splat->expression != NULL) { - splat->expression = parse_target(parser, splat->expression, multiple); + splat->expression = parse_target(parser, splat->expression, multiple, true); } return (pm_node_t *) splat; @@ -13237,9 +13224,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple) { */ static pm_node_t * parse_target_validate(pm_parser_t *parser, pm_node_t *target, bool multiple) { - pm_node_t *result = parse_target(parser, target, multiple); + pm_node_t *result = parse_target(parser, target, multiple, false); - // Ensure that we have one of an =, an 'in' in for indexes, and a ')' in parens after the targets. + // Ensure that we have one of an =, an 'in' in for indexes, and a ')' in + // parens after the targets. if ( !match1(parser, PM_TOKEN_EQUAL) && !(context_p(parser, PM_CONTEXT_FOR_INDEX) && match1(parser, PM_TOKEN_KEYWORD_IN)) && @@ -13309,18 +13297,34 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod return (pm_node_t *) node; } case PM_LOCAL_VARIABLE_READ_NODE: { - pm_refute_numbered_parameter(parser, target->location.start, target->location.end); pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target; pm_constant_id_t name = local_read->name; + pm_location_t name_loc = target->location; + uint32_t depth = local_read->depth; - pm_locals_unread(&pm_parser_scope_find(parser, depth)->locals, name); + pm_scope_t *scope = pm_parser_scope_find(parser, depth); - pm_location_t name_loc = target->location; + if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED; + PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, diag_id, target->location.start); + parse_target_implicit_parameter(parser, target); + } + + pm_locals_unread(&scope->locals, name); pm_node_destroy(parser, target); return (pm_node_t *) pm_local_variable_write_node_create(parser, name, depth, value, &name_loc, operator); } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); + pm_node_t *node = (pm_node_t *) pm_local_variable_write_node_create(parser, name, 0, value, &target->location, operator); + + parse_target_implicit_parameter(parser, target); + pm_node_destroy(parser, target); + + return node; + } case PM_INSTANCE_VARIABLE_READ_NODE: { pm_node_t *write_node = (pm_node_t *) pm_instance_variable_write_node_create(parser, (pm_instance_variable_read_node_t *) target, operator, value); pm_node_destroy(parser, target); @@ -13474,7 +13478,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b bool has_rest = PM_NODE_TYPE_P(first_target, PM_SPLAT_NODE); pm_multi_target_node_t *result = pm_multi_target_node_create(parser); - pm_multi_target_node_targets_append(parser, result, parse_target(parser, first_target, true)); + pm_multi_target_node_targets_append(parser, result, parse_target(parser, first_target, true, false)); while (accept1(parser, PM_TOKEN_COMMA)) { if (accept1(parser, PM_TOKEN_USTAR)) { @@ -13490,7 +13494,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b if (token_begins_expression_p(parser->current.type)) { name = parse_expression(parser, binding_power, false, PM_ERR_EXPECT_EXPRESSION_AFTER_STAR); - name = parse_target(parser, name, true); + name = parse_target(parser, name, true, true); } pm_node_t *splat = (pm_node_t *) pm_splat_node_create(parser, &star_operator, name); @@ -13498,7 +13502,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b has_rest = true; } else if (token_begins_expression_p(parser->current.type)) { pm_node_t *target = parse_expression(parser, binding_power, false, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA); - target = parse_target(parser, target, true); + target = parse_target(parser, target, true, false); pm_multi_target_node_targets_append(parser, result, target); } else if (!match1(parser, PM_TOKEN_EOF)) { @@ -13535,8 +13539,8 @@ parse_targets_validate(pm_parser_t *parser, pm_node_t *first_target, pm_binding_ */ static pm_statements_node_t * parse_statements(pm_parser_t *parser, pm_context_t context) { - // First, skip past any optional terminators that might be at the beginning of - // the statements. + // First, skip past any optional terminators that might be at the beginning + // of the statements. while (accept2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)); // If we have a terminator, then we can just return NULL. @@ -13552,20 +13556,20 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_STATEMENT, true, PM_ERR_CANNOT_PARSE_EXPRESSION); pm_statements_node_body_append(parser, statements, node); - // If we're recovering from a syntax error, then we need to stop parsing the - // statements now. + // If we're recovering from a syntax error, then we need to stop parsing + // the statements now. if (parser->recovering) { - // If this is the level of context where the recovery has happened, then - // we can mark the parser as done recovering. + // If this is the level of context where the recovery has happened, + // then we can mark the parser as done recovering. if (context_terminator(context, &parser->current)) parser->recovering = false; break; } - // If we have a terminator, then we will parse all consecutive terminators - // and then continue parsing the statements list. + // If we have a terminator, then we will parse all consecutive + // terminators and then continue parsing the statements list. if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - // If we have a terminator, then we will continue parsing the statements - // list. + // If we have a terminator, then we will continue parsing the + // statements list. while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)); if (context_terminator(context, &parser->current)) break; @@ -13573,27 +13577,28 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { continue; } - // At this point we have a list of statements that are not terminated by a - // newline or semicolon. At this point we need to check if we're at the end - // of the statements list. If we are, then we should break out of the loop. + // At this point we have a list of statements that are not terminated by + // a newline or semicolon. At this point we need to check if we're at + // the end of the statements list. If we are, then we should break out + // of the loop. if (context_terminator(context, &parser->current)) break; // At this point, we have a syntax error, because the statement was not // terminated by a newline or semicolon, and we're not at the end of the - // statements list. Ideally we should scan forward to determine if we should - // insert a missing terminator or break out of parsing the statements list - // at this point. + // statements list. Ideally we should scan forward to determine if we + // should insert a missing terminator or break out of parsing the + // statements list at this point. // - // We don't have that yet, so instead we'll do a more naive approach. If we - // were unable to parse an expression, then we will skip past this token and - // continue parsing the statements list. Otherwise we'll add an error and - // continue parsing the statements list. + // We don't have that yet, so instead we'll do a more naive approach. If + // we were unable to parse an expression, then we will skip past this + // token and continue parsing the statements list. Otherwise we'll add + // an error and continue parsing the statements list. if (PM_NODE_TYPE_P(node, PM_MISSING_NODE)) { parser_lex(parser); while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)); if (context_terminator(context, &parser->current)) break; - } else if (!accept1(parser, PM_TOKEN_NEWLINE)) { + } else if (!accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_EOF)) { // This is an inlined version of accept1 because the error that we // want to add has varargs. If this happens again, we should // probably extract a helper function. @@ -13615,7 +13620,7 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { */ static void pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node); + const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, true); if (duplicated != NULL) { pm_buffer_t buffer = { 0 }; @@ -13641,13 +13646,16 @@ pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *liter */ static void pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node) != NULL) { + pm_node_t *previous; + + if ((previous = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, node->location.start, node->location.end, PM_WARN_DUPLICATED_WHEN_CLAUSE, - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line, + pm_newline_list_line_column(&parser->newline_list, previous->location.start, parser->start_line).line ); } } @@ -14101,31 +14109,37 @@ static pm_parameters_order_t parameters_ordering[PM_TOKEN_MAXIMUM] = { * Check if current parameter follows valid parameters ordering. If not it adds * an error to the list without stopping the parsing, otherwise sets the * parameters state to the one corresponding to the current parameter. + * + * It returns true if it was successful, and false otherwise. */ -static void +static bool update_parameter_state(pm_parser_t *parser, pm_token_t *token, pm_parameters_order_t *current) { pm_parameters_order_t state = parameters_ordering[token->type]; - if (state == PM_PARAMETERS_NO_CHANGE) return; + if (state == PM_PARAMETERS_NO_CHANGE) return true; // If we see another ordered argument after a optional argument // we only continue parsing ordered arguments until we stop seeing ordered arguments. if (*current == PM_PARAMETERS_ORDER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) { *current = PM_PARAMETERS_ORDER_AFTER_OPTIONAL; - return; + return true; } else if (*current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) { - return; + return true; } if (token->type == PM_TOKEN_USTAR && *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL) { pm_parser_err_token(parser, token, PM_ERR_PARAMETER_STAR); - } - - if (*current == PM_PARAMETERS_ORDER_NOTHING_AFTER || state > *current) { + return false; + } else if (token->type == PM_TOKEN_UDOT_DOT_DOT && (*current >= PM_PARAMETERS_ORDER_KEYWORDS_REST && *current <= PM_PARAMETERS_ORDER_AFTER_OPTIONAL)) { + pm_parser_err_token(parser, token, *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL ? PM_ERR_PARAMETER_FORWARDING_AFTER_REST : PM_ERR_PARAMETER_ORDER); + return false; + } else if (*current == PM_PARAMETERS_ORDER_NOTHING_AFTER || state > *current) { // We know what transition we failed on, so we can provide a better error here. pm_parser_err_token(parser, token, PM_ERR_PARAMETER_ORDER); - } else if (state < *current) { - *current = state; + return false; } + + if (state < *current) *current = state; + return true; } /** @@ -14194,27 +14208,22 @@ parse_parameters( pm_parser_err_current(parser, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES); } - if (order > PM_PARAMETERS_ORDER_NOTHING_AFTER) { - update_parameter_state(parser, &parser->current, &order); - parser_lex(parser); + bool succeeded = update_parameter_state(parser, &parser->current, &order); + parser_lex(parser); - parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL; + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL; + pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous); - pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous); - if (params->keyword_rest != NULL) { - // If we already have a keyword rest parameter, then we replace it with the - // forwarding parameter and move the keyword rest parameter to the posts list. - pm_node_t *keyword_rest = params->keyword_rest; - pm_parameters_node_posts_append(params, keyword_rest); - pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_FWD); - params->keyword_rest = NULL; - } - pm_parameters_node_keyword_rest_set(params, (pm_node_t *)param); - } else { - update_parameter_state(parser, &parser->current, &order); - parser_lex(parser); + if (params->keyword_rest != NULL) { + // If we already have a keyword rest parameter, then we replace it with the + // forwarding parameter and move the keyword rest parameter to the posts list. + pm_node_t *keyword_rest = params->keyword_rest; + pm_parameters_node_posts_append(params, keyword_rest); + if (succeeded) pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_FWD); + params->keyword_rest = NULL; } + pm_parameters_node_keyword_rest_set(params, (pm_node_t *) param); break; } case PM_TOKEN_CLASS_VARIABLE: @@ -14258,7 +14267,7 @@ parse_parameters( context_push(parser, PM_CONTEXT_DEFAULT_PARAMS); pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &name); - uint32_t reads = pm_locals_reads(&parser->current_scope->locals, name_id); + uint32_t reads = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? pm_locals_reads(&parser->current_scope->locals, name_id) : 0; pm_node_t *value = parse_value_expression(parser, binding_power, false, PM_ERR_PARAMETER_NO_DEFAULT); pm_optional_parameter_node_t *param = pm_optional_parameter_node_create(parser, &name, &operator, value); @@ -14271,7 +14280,7 @@ parse_parameters( // If the value of the parameter increased the number of // reads of that parameter, then we need to warn that we // have a circular definition. - if (pm_locals_reads(&parser->current_scope->locals, name_id) != reads) { + if ((parser->version == PM_OPTIONS_VERSION_CRUBY_3_3) && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, name, PM_ERR_PARAMETER_CIRCULAR); } @@ -14309,6 +14318,12 @@ parse_parameters( pm_token_t local = name; local.end -= 1; + if (parser->encoding_changed ? parser->encoding->isupper_char(local.start, local.end - local.start) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start)) { + pm_parser_err(parser, local.start, local.end, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + } else if (local.end[-1] == '!' || local.end[-1] == '?') { + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); + } + bool repeated = pm_parser_parameter_name_check(parser, &local); pm_parser_local_add_token(parser, &local, 1); @@ -14344,10 +14359,10 @@ parse_parameters( context_push(parser, PM_CONTEXT_DEFAULT_PARAMS); pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &local); - uint32_t reads = pm_locals_reads(&parser->current_scope->locals, name_id); + uint32_t reads = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? pm_locals_reads(&parser->current_scope->locals, name_id) : 0; pm_node_t *value = parse_value_expression(parser, binding_power, false, PM_ERR_PARAMETER_NO_DEFAULT_KW); - if (pm_locals_reads(&parser->current_scope->locals, name_id) != reads) { + if (parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_PARAMETER_CIRCULAR); } @@ -14519,7 +14534,7 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, pm_rescues_type pm_rescue_node_operator_set(rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, PM_ERR_RESCUE_VARIABLE); - reference = parse_target(parser, reference, false); + reference = parse_target(parser, reference, false, false); pm_rescue_node_reference_set(rescue, reference); break; @@ -14549,7 +14564,7 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, pm_rescues_type pm_rescue_node_operator_set(rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, PM_ERR_RESCUE_VARIABLE); - reference = parse_target(parser, reference, false); + reference = parse_target(parser, reference, false, false); pm_rescue_node_reference_set(rescue, reference); break; @@ -14755,37 +14770,107 @@ parse_block_parameters( } /** + * Return true if any of the visible scopes to the current context are using + * numbered parameters. + */ +static bool +outer_scope_using_numbered_parameters_p(pm_parser_t *parser) { + for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { + if (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) return true; + } + + return false; +} + +/** + * These are the names of the various numbered parameters. We have them here so + * that when we insert them into the constant pool we can use a constant string + * and not have to allocate. + */ +static const char * const pm_numbered_parameter_names[] = { + "_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9" +}; + +/** * Return the node that should be used in the parameters field of a block-like * (block or lambda) node, depending on the kind of parameters that were * declared in the current scope. */ static pm_node_t * parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_token_t *opening, const pm_token_t *closing) { - uint8_t masked = parser->current_scope->parameters & PM_SCOPE_PARAMETERS_TYPE_MASK; + pm_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters; + + // If we have ordinary parameters, then we will return them as the set of + // parameters. + if (parameters != NULL) { + // If we also have implicit parameters, then this is an error. + if (implicit_parameters->size > 0) { + pm_node_t *node = implicit_parameters->nodes[0]; + + if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_ORDINARY); + } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_ORDINARY); + } else { + assert(false && "unreachable"); + } + } - if (masked == PM_SCOPE_PARAMETERS_NONE) { - assert(parameters == NULL); - return NULL; - } else if (masked == PM_SCOPE_PARAMETERS_ORDINARY) { - assert(parameters != NULL); return parameters; - } else if (masked == PM_SCOPE_PARAMETERS_NUMBERED) { - assert(parameters == NULL); + } - int8_t maximum = parser->current_scope->numbered_parameters; - if (maximum > 0) { - const pm_location_t location = { .start = opening->start, .end = closing->end }; - return (pm_node_t *) pm_numbered_parameters_node_create(parser, &location, (uint8_t) maximum); + // If we don't have any implicit parameters, then the set of parameters is + // NULL. + if (implicit_parameters->size == 0) { + return NULL; + } + + // If we don't have ordinary parameters, then we now must validate our set + // of implicit parameters. We can only have numbered parameters or it, but + // they cannot be mixed. + uint8_t numbered_parameter = 0; + bool it_parameter = false; + + for (size_t index = 0; index < implicit_parameters->size; index++) { + pm_node_t *node = implicit_parameters->nodes[index]; + + if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) { + if (it_parameter) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_IT); + } else if (outer_scope_using_numbered_parameters_p(parser)) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK); + } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK); + } else if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + numbered_parameter = MAX(numbered_parameter, (uint8_t) (node->location.start[1] - '0')); + } else { + assert(false && "unreachable"); + } + } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + if (numbered_parameter > 0) { + pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_NUMBERED); + } else { + it_parameter = true; + } } + } - return NULL; - } else if (masked == PM_SCOPE_PARAMETERS_IT) { - assert(parameters == NULL); + if (numbered_parameter > 0) { + // Go through the parent scopes and mark them as being disallowed from + // using numbered parameters because this inner scope is using them. + for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { + scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_INNER; + } + + const pm_location_t location = { .start = opening->start, .end = closing->end }; + return (pm_node_t *) pm_numbered_parameters_node_create(parser, &location, numbered_parameter); + } + + if (it_parameter) { return (pm_node_t *) pm_it_parameters_node_create(parser, opening, closing); - } else { - assert(false && "unreachable"); - return NULL; } + + return NULL; } /** @@ -14802,9 +14887,6 @@ parse_block(pm_parser_t *parser) { pm_block_parameters_node_t *block_parameters = NULL; if (accept1(parser, PM_TOKEN_PIPE)) { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_token_t block_parameters_opening = parser->previous; if (match1(parser, PM_TOKEN_PIPE)) { block_parameters = pm_block_parameters_node_create(parser, NULL, &block_parameters_opening); @@ -14873,7 +14955,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); } else { pm_accepts_block_stack_push(parser, true); - parse_arguments(parser, arguments, true, PM_TOKEN_PARENTHESIS_RIGHT); + parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); @@ -14891,7 +14973,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept // If we get here, then the subsequent token cannot be used as an infix // operator. In this case we assume the subsequent token is part of an // argument to this method call. - parse_arguments(parser, arguments, true, PM_TOKEN_EOF); + parse_arguments(parser, arguments, accepts_block, PM_TOKEN_EOF); // If we have done with the arguments and still not consumed the comma, // then we have a trailing comma where we need to check whether it is @@ -14922,11 +15004,8 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (arguments->block == NULL && !arguments->has_forwarding) { arguments->block = (pm_node_t *) block; } else { - if (arguments->has_forwarding) { - pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_FORWARDING); - } else { - pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_MULTI); - } + pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_MULTI); + if (arguments->block != NULL) { if (arguments->arguments == NULL) { arguments->arguments = pm_arguments_node_create(parser); @@ -14964,6 +15043,7 @@ parse_block_exit(pm_parser_t *parser, pm_node_t *node, const char *type) { case PM_CONTEXT_LAMBDA_ELSE: case PM_CONTEXT_LAMBDA_ENSURE: case PM_CONTEXT_LAMBDA_RESCUE: + case PM_CONTEXT_LOOP_PREDICATE: case PM_CONTEXT_POSTEXE: case PM_CONTEXT_UNTIL: case PM_CONTEXT_WHILE: @@ -15305,7 +15385,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context) { #define PM_CASE_WRITABLE PM_CLASS_VARIABLE_READ_NODE: case PM_CONSTANT_PATH_NODE: \ case PM_CONSTANT_READ_NODE: case PM_GLOBAL_VARIABLE_READ_NODE: case PM_LOCAL_VARIABLE_READ_NODE: \ case PM_INSTANCE_VARIABLE_READ_NODE: case PM_MULTI_TARGET_NODE: case PM_BACK_REFERENCE_READ_NODE: \ - case PM_NUMBERED_REFERENCE_READ_NODE + case PM_NUMBERED_REFERENCE_READ_NODE: case PM_IT_LOCAL_VARIABLE_READ_NODE // Assert here that the flags are the same so that we can safely switch the type // of the node without having to move the flags. @@ -15363,6 +15443,10 @@ parse_string_part(pm_parser_t *parser) { // "aaa #{bbb} #@ccc ddd" // ^^^^^^ case PM_TOKEN_EMBEXPR_BEGIN: { + // Ruby disallows seeing encoding around interpolation in strings, + // even though it is known at parse time. + parser->explicit_encoding = NULL; + pm_lex_state_t state = parser->lex_state; int brace_nesting = parser->brace_nesting; @@ -15385,6 +15469,13 @@ parse_string_part(pm_parser_t *parser) { expect1(parser, PM_TOKEN_EMBEXPR_END, PM_ERR_EMBEXPR_END); pm_token_t closing = parser->previous; + // If this set of embedded statements only contains a single + // statement, then Ruby does not consider it as a possible statement + // that could emit a line event. + if (statements != NULL && statements->body.size == 1) { + pm_node_flag_unset(statements->body.nodes[0], PM_NODE_FLAG_NEWLINE); + } + return (pm_node_t *) pm_embedded_statements_node_create(parser, &opening, statements, &closing); } @@ -15395,6 +15486,10 @@ parse_string_part(pm_parser_t *parser) { // "aaa #{bbb} #@ccc ddd" // ^^^^^ case PM_TOKEN_EMBVAR: { + // Ruby disallows seeing encoding around interpolation in strings, + // even though it is known at parse time. + parser->explicit_encoding = NULL; + lex_state_set(parser, PM_LEX_STATE_BEG); parser_lex(parser); @@ -15710,74 +15805,43 @@ parse_alias_argument(pm_parser_t *parser, bool first) { } /** - * Return true if any of the visible scopes to the current context are using - * numbered parameters. - */ -static bool -outer_scope_using_numbered_parameters_p(pm_parser_t *parser) { - for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { - if (scope->numbered_parameters > 0) return true; - } - - return false; -} - -/** - * These are the names of the various numbered parameters. We have them here so - * that when we insert them into the constant pool we can use a constant string - * and not have to allocate. - */ -static const char * const pm_numbered_parameter_names[] = { - "_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9" -}; - -/** * Parse an identifier into either a local variable read. If the local variable * is not found, it returns NULL instead. */ -static pm_local_variable_read_node_t * +static pm_node_t * parse_variable(pm_parser_t *parser) { + pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; - if ((depth = pm_parser_local_depth(parser, &parser->previous)) != -1) { - return pm_local_variable_read_node_create(parser, &parser->previous, (uint32_t) depth); + + if ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1) { + return (pm_node_t *) pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false); } pm_scope_t *current_scope = parser->current_scope; - if (!current_scope->closed && current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED && pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end)) { - // Now that we know we have a numbered parameter, we need to check - // if it's allowed in this context. If it is, then we will create a - // local variable read. If it's not, then we'll create a normal call - // node but add an error. - if (current_scope->parameters & PM_SCOPE_PARAMETERS_ORDINARY) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_ORDINARY); - } else if (current_scope->parameters & PM_SCOPE_PARAMETERS_IT) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_IT); - } else if (outer_scope_using_numbered_parameters_p(parser)) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE); - } else { - // Indicate that this scope is using numbered params so that child - // scopes cannot. We subtract the value for the character '0' to get - // the actual integer value of the number (only _1 through _9 are - // valid). - int8_t numbered_parameters = (int8_t) (parser->previous.start[1] - '0'); - current_scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED; - - if (numbered_parameters > current_scope->numbered_parameters) { - current_scope->numbered_parameters = numbered_parameters; + if (!current_scope->closed && !(current_scope->parameters & PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED)) { + if (pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end)) { + // When you use a numbered parameter, it implies the existence of + // all of the locals that exist before it. For example, referencing + // _2 means that _1 must exist. Therefore here we loop through all + // of the possibilities and add them into the constant pool. + uint8_t maximum = (uint8_t) (parser->previous.start[1] - '0'); + for (uint8_t number = 1; number <= maximum; number++) { + pm_parser_local_add_constant(parser, pm_numbered_parameter_names[number - 1], 2); } - // When you use a numbered parameter, it implies the existence - // of all of the locals that exist before it. For example, - // referencing _2 means that _1 must exist. Therefore here we - // loop through all of the possibilities and add them into the - // constant pool. - for (int8_t numbered_param = 1; numbered_param <= numbered_parameters - 1; numbered_param++) { - pm_parser_local_add_constant(parser, pm_numbered_parameter_names[numbered_param - 1], 2); + if (!match1(parser, PM_TOKEN_EQUAL)) { + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_FOUND; } - // Finally we can create the local variable read node. - pm_constant_id_t name_id = pm_parser_local_add_constant(parser, pm_numbered_parameter_names[numbered_parameters - 1], 2); - return pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, 0, false); + pm_node_t *node = (pm_node_t *) pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, 0, false); + pm_node_list_append(¤t_scope->implicit_parameters, node); + + return node; + } else if ((parser->version != PM_OPTIONS_VERSION_CRUBY_3_3) && pm_token_is_it(parser->previous.start, parser->previous.end)) { + pm_node_t *node = (pm_node_t *) pm_it_local_variable_read_node_create(parser, &parser->previous); + pm_node_list_append(¤t_scope->implicit_parameters, node); + + return node; } } @@ -15792,8 +15856,8 @@ parse_variable_call(pm_parser_t *parser) { pm_node_flags_t flags = 0; if (!match1(parser, PM_TOKEN_PARENTHESIS_LEFT) && (parser->previous.end[-1] != '!') && (parser->previous.end[-1] != '?')) { - pm_local_variable_read_node_t *node = parse_variable(parser); - if (node != NULL) return (pm_node_t *) node; + pm_node_t *node = parse_variable(parser); + if (node != NULL) return node; flags |= PM_CALL_NODE_FLAGS_VARIABLE_CALL; } @@ -15911,8 +15975,236 @@ parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_w nodes->size = write_index; } +/** + * Return a string content token at a particular location that is empty. + */ +static pm_token_t +parse_strings_empty_content(const uint8_t *location) { + return (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = location, .end = location }; +} + +/** + * Parse a set of strings that could be concatenated together. + */ +static inline pm_node_t * +parse_strings(pm_parser_t *parser, pm_node_t *current) { + assert(parser->current.type == PM_TOKEN_STRING_BEGIN); + + bool concating = false; + bool state_is_arg_labeled = lex_state_arg_labeled_p(parser); + + while (match1(parser, PM_TOKEN_STRING_BEGIN)) { + pm_node_t *node = NULL; + + // Here we have found a string literal. We'll parse it and add it to + // the list of strings. + const pm_lex_mode_t *lex_mode = parser->lex_modes.current; + assert(lex_mode->mode == PM_LEX_STRING); + bool lex_interpolation = lex_mode->as.string.interpolation; + + pm_token_t opening = parser->current; + parser_lex(parser); + + if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { + expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); + // If we get here, then we have an end immediately after a + // start. In that case we'll create an empty content token and + // return an uninterpolated string. + pm_token_t content = parse_strings_empty_content(parser->previous.start); + pm_string_node_t *string = pm_string_node_create(parser, &opening, &content, &parser->previous); + + pm_string_shared_init(&string->unescaped, content.start, content.end); + node = (pm_node_t *) string; + } else if (accept1(parser, PM_TOKEN_LABEL_END)) { + // If we get here, then we have an end of a label immediately + // after a start. In that case we'll create an empty symbol + // node. + pm_token_t content = parse_strings_empty_content(parser->previous.start); + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &content, &parser->previous); + + pm_string_shared_init(&symbol->unescaped, content.start, content.end); + node = (pm_node_t *) symbol; + } else if (!lex_interpolation) { + // If we don't accept interpolation then we expect the string to + // start with a single string content node. + pm_string_t unescaped; + pm_token_t content; + + if (match1(parser, PM_TOKEN_EOF)) { + unescaped = PM_STRING_EMPTY; + content = not_provided(parser); + } else { + unescaped = parser->current_string; + expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_EXPECT_STRING_CONTENT); + content = parser->previous; + } + + // It is unfortunately possible to have multiple string content + // nodes in a row in the case that there's heredoc content in + // the middle of the string, like this cursed example: + // + // <<-END+'b + // a + // END + // c'+'d' + // + // In that case we need to switch to an interpolated string to + // be able to contain all of the parts. + if (match1(parser, PM_TOKEN_STRING_CONTENT)) { + pm_node_list_t parts = { 0 }; + + pm_token_t delimiters = not_provided(parser); + pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &delimiters, &content, &delimiters, &unescaped); + pm_node_list_append(&parts, part); + + do { + part = (pm_node_t *) pm_string_node_create_current_string(parser, &delimiters, &parser->current, &delimiters); + pm_node_list_append(&parts, part); + parser_lex(parser); + } while (match1(parser, PM_TOKEN_STRING_CONTENT)); + + expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); + node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); + + pm_node_list_free(&parts); + } else if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { + node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); + } else if (match1(parser, PM_TOKEN_EOF)) { + pm_parser_err_token(parser, &opening, PM_ERR_STRING_LITERAL_EOF); + node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); + } else if (accept1(parser, PM_TOKEN_STRING_END)) { + node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); + } else { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); + } + } else if (match1(parser, PM_TOKEN_STRING_CONTENT)) { + // In this case we've hit string content so we know the string + // at least has something in it. We'll need to check if the + // following token is the end (in which case we can return a + // plain string) or if it's not then it has interpolation. + pm_token_t content = parser->current; + pm_string_t unescaped = parser->current_string; + parser_lex(parser); + + if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { + node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); + pm_node_flag_set(node, parse_unescaped_encoding(parser)); + + // Kind of odd behavior, but basically if we have an + // unterminated string and it ends in a newline, we back up one + // character so that the error message is on the last line of + // content in the string. + if (!accept1(parser, PM_TOKEN_STRING_END)) { + const uint8_t *location = parser->previous.end; + if (location > parser->start && location[-1] == '\n') location--; + pm_parser_err(parser, location, location, PM_ERR_STRING_LITERAL_EOF); + + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + } + } else if (accept1(parser, PM_TOKEN_LABEL_END)) { + node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); + } else { + // If we get here, then we have interpolation so we'll need + // to create a string or symbol node with interpolation. + pm_node_list_t parts = { 0 }; + pm_token_t string_opening = not_provided(parser); + pm_token_t string_closing = not_provided(parser); + + pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &string_opening, &parser->previous, &string_closing, &unescaped); + pm_node_flag_set(part, parse_unescaped_encoding(parser)); + pm_node_list_append(&parts, part); + + while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) { + if ((part = parse_string_part(parser)) != NULL) { + pm_node_list_append(&parts, part); + } + } + + if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { + node = (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); + } else if (match1(parser, PM_TOKEN_EOF)) { + pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM); + node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current); + } else { + expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); + node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); + } + + pm_node_list_free(&parts); + } + } else { + // If we get here, then the first part of the string is not plain + // string content, in which case we need to parse the string as an + // interpolated string. + pm_node_list_t parts = { 0 }; + pm_node_t *part; + + while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) { + if ((part = parse_string_part(parser)) != NULL) { + pm_node_list_append(&parts, part); + } + } + + if (accept1(parser, PM_TOKEN_LABEL_END)) { + node = (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); + } else if (match1(parser, PM_TOKEN_EOF)) { + pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM); + node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current); + } else { + expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); + node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); + } + + pm_node_list_free(&parts); + } + + if (current == NULL) { + // If the node we just parsed is a symbol node, then we can't + // concatenate it with anything else, so we can now return that + // node. + if (PM_NODE_TYPE_P(node, PM_SYMBOL_NODE) || PM_NODE_TYPE_P(node, PM_INTERPOLATED_SYMBOL_NODE)) { + return node; + } + + // If we don't already have a node, then it's fine and we can just + // set the result to be the node we just parsed. + current = node; + } else { + // Otherwise we need to check the type of the node we just parsed. + // If it cannot be concatenated with the previous node, then we'll + // need to add a syntax error. + if (!PM_NODE_TYPE_P(node, PM_STRING_NODE) && !PM_NODE_TYPE_P(node, PM_INTERPOLATED_STRING_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_STRING_CONCATENATION); + } + + // If we haven't already created our container for concatenation, + // we'll do that now. + if (!concating) { + concating = true; + pm_token_t bounds = not_provided(parser); + + pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); + pm_interpolated_string_node_append(container, current); + current = (pm_node_t *) container; + } + + pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, node); + } + } + + return current; +} + +#define PM_PARSE_PATTERN_SINGLE 0 +#define PM_PARSE_PATTERN_TOP 1 +#define PM_PARSE_PATTERN_MULTI 2 + static pm_node_t * -parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id); +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id); /** * Add the newly created local to the list of captures for this pattern matching @@ -15960,7 +16252,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); } @@ -15972,7 +16264,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); } @@ -16121,20 +16413,51 @@ parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) } /** + * Check that the slice of the source given by the bounds parameters constitutes + * a valid local variable name. + */ +static bool +pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { + ptrdiff_t length = end - start; + if (length == 0) return false; + + // First ensure that it starts with a valid identifier starting character. + size_t width = char_is_identifier_start(parser, start); + if (width == 0) return false; + + // Next, ensure that it's not an uppercase character. + if (parser->encoding_changed) { + if (parser->encoding->isupper_char(start, length)) return false; + } else { + if (pm_encoding_utf_8_isupper_char(start, length)) return false; + } + + // Next, iterate through all of the bytes of the string to ensure that they + // are all valid identifier characters. + const uint8_t *cursor = start + width; + while ((cursor < end) && (width = char_is_identifier(parser, cursor))) cursor += width; + return cursor == end; +} + +/** * Create an implicit node for the value of a hash pattern that has omitted the * value. This will use an implicit local variable target. */ static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); int depth = -1; - if (value_loc->end[-1] == '!' || value_loc->end[-1] == '?') { - pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); - PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); - } else { + + if (pm_slice_is_valid_local(parser, value_loc->start, value_loc->end)) { depth = pm_parser_local_depth_constant_id(parser, constant_id); + } else { + pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); + + if ((value_loc->end > value_loc->start) && ((value_loc->end[-1] == '!') || (value_loc->end[-1] == '?'))) { + PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); + } } if (depth == -1) { @@ -16158,7 +16481,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca */ static void parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node) != NULL) { + if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node, true) != NULL) { pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); } } @@ -16189,7 +16512,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node } else { // Here we have a value for the first assoc in the list, so // we will parse it now. - value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -16204,7 +16527,8 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node // If we get anything else, then this is an error. For this we'll // create a missing node for the value and create an assoc node for // the first node in the list. - pm_parser_err_node(parser, first_node, PM_ERR_PATTERN_HASH_KEY_LABEL); + pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL; + pm_parser_err_node(parser, first_node, diag_id); pm_token_t operator = not_provided(parser); pm_node_t *value = (pm_node_t *) pm_missing_node_create(parser, first_node->location.start, first_node->location.end); @@ -16232,8 +16556,20 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node pm_node_list_append(&assocs, assoc); } } else { - expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); - pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); + pm_node_t *key; + + if (match1(parser, PM_TOKEN_STRING_BEGIN)) { + key = parse_strings(parser, NULL); + + if (PM_NODE_TYPE_P(key, PM_INTERPOLATED_SYMBOL_NODE)) { + pm_parser_err_node(parser, key, PM_ERR_PATTERN_HASH_KEY_INTERPOLATED); + } else if (!pm_symbol_node_label_p(key)) { + pm_parser_err_node(parser, key, PM_ERR_PATTERN_LABEL_AFTER_COMMA); + } + } else { + expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); + key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); + } parse_pattern_hash_key(parser, &keys, key); pm_node_t *value = NULL; @@ -16241,7 +16577,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -16298,7 +16634,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm // Otherwise, we'll parse the inner pattern, then deal with it depending // on the type it returns. - pm_node_t *inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + pm_node_t *inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); @@ -16365,11 +16701,11 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm first_node = parse_pattern_keyword_rest(parser, captures); break; case PM_TOKEN_STRING_BEGIN: - first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY); + first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY_LABEL); break; default: { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); parser_lex(parser); - pm_parser_err_previous(parser, PM_ERR_PATTERN_HASH_KEY); first_node = (pm_node_t *) pm_missing_node_create(parser, parser->previous.start, parser->previous.end); break; @@ -16445,19 +16781,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *variable = (pm_node_t *) parse_variable(parser); if (variable == NULL) { - if ( - (parser->version != PM_OPTIONS_VERSION_CRUBY_3_3) && - !parser->current_scope->closed && - (parser->current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED) && - pm_token_is_it(parser->previous.start, parser->previous.end) - ) { - pm_local_variable_read_node_t *read = pm_local_variable_read_node_create_it(parser, &parser->previous); - if (read == NULL) read = pm_local_variable_read_node_create(parser, &parser->previous, 0); - variable = (pm_node_t *) read; - } else { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); - variable = (pm_node_t *) pm_local_variable_read_node_missing_create(parser, &parser->previous, 0); - } + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); + variable = (pm_node_t *) pm_local_variable_read_node_missing_create(parser, &parser->previous, 0); } return (pm_node_t *) pm_pinned_variable_node_create(parser, &operator, variable); @@ -16571,7 +16896,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p pm_token_t opening = parser->current; parser_lex(parser); - pm_node_t *body = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + pm_node_t *body = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); pm_node_t *right = (pm_node_t *) pm_parentheses_node_create(parser, &opening, body, &parser->previous); @@ -16630,7 +16955,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p * Parse a pattern matching expression. */ static pm_node_t * -parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id) { +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; bool leading_rest = false; @@ -16640,14 +16965,26 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pat case PM_TOKEN_LABEL: { parser_lex(parser); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); - return (pm_node_t *) parse_pattern_hash(parser, captures, key); + node = (pm_node_t *) parse_pattern_hash(parser, captures, key); + + if (!(flags & PM_PARSE_PATTERN_TOP)) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT); + } + + return node; } case PM_TOKEN_USTAR_STAR: { node = parse_pattern_keyword_rest(parser, captures); - return (pm_node_t *) parse_pattern_hash(parser, captures, node); + node = (pm_node_t *) parse_pattern_hash(parser, captures, node); + + if (!(flags & PM_PARSE_PATTERN_TOP)) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT); + } + + return node; } case PM_TOKEN_USTAR: { - if (top_pattern) { + if (flags & (PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI)) { parser_lex(parser); node = (pm_node_t *) parse_pattern_rest(parser, captures); leading_rest = true; @@ -16666,7 +17003,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pat return (pm_node_t *) parse_pattern_hash(parser, captures, node); } - if (top_pattern && match1(parser, PM_TOKEN_COMMA)) { + if ((flags & PM_PARSE_PATTERN_MULTI) && match1(parser, PM_TOKEN_COMMA)) { // If we have a comma, then we are now parsing either an array pattern or a // find pattern. We need to parse all of the patterns, put them into a big // list, and then determine which type of node we have. @@ -16740,10 +17077,12 @@ parse_negative_numeric(pm_node_t *node) { cast->value = -cast->value; break; } - case PM_RATIONAL_NODE: - node->location.start--; - parse_negative_numeric(((pm_rational_node_t *) node)->numeric); + case PM_RATIONAL_NODE: { + pm_rational_node_t *cast = (pm_rational_node_t *) node; + cast->base.location.start--; + cast->numerator.negative = true; break; + } case PM_IMAGINARY_NODE: node->location.start--; parse_negative_numeric(((pm_imaginary_node_t *) node)->numeric); @@ -16755,217 +17094,6 @@ parse_negative_numeric(pm_node_t *node) { } /** - * Return a string content token at a particular location that is empty. - */ -static pm_token_t -parse_strings_empty_content(const uint8_t *location) { - return (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = location, .end = location }; -} - -/** - * Parse a set of strings that could be concatenated together. - */ -static inline pm_node_t * -parse_strings(pm_parser_t *parser, pm_node_t *current) { - assert(parser->current.type == PM_TOKEN_STRING_BEGIN); - - bool concating = false; - bool state_is_arg_labeled = lex_state_arg_labeled_p(parser); - - while (match1(parser, PM_TOKEN_STRING_BEGIN)) { - pm_node_t *node = NULL; - - // Here we have found a string literal. We'll parse it and add it to - // the list of strings. - const pm_lex_mode_t *lex_mode = parser->lex_modes.current; - assert(lex_mode->mode == PM_LEX_STRING); - bool lex_interpolation = lex_mode->as.string.interpolation; - - pm_token_t opening = parser->current; - parser_lex(parser); - - if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); - // If we get here, then we have an end immediately after a - // start. In that case we'll create an empty content token and - // return an uninterpolated string. - pm_token_t content = parse_strings_empty_content(parser->previous.start); - pm_string_node_t *string = pm_string_node_create(parser, &opening, &content, &parser->previous); - - pm_string_shared_init(&string->unescaped, content.start, content.end); - node = (pm_node_t *) string; - } else if (accept1(parser, PM_TOKEN_LABEL_END)) { - // If we get here, then we have an end of a label immediately - // after a start. In that case we'll create an empty symbol - // node. - pm_token_t content = parse_strings_empty_content(parser->previous.start); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &content, &parser->previous); - - pm_string_shared_init(&symbol->unescaped, content.start, content.end); - node = (pm_node_t *) symbol; - } else if (!lex_interpolation) { - // If we don't accept interpolation then we expect the string to - // start with a single string content node. - pm_string_t unescaped; - pm_token_t content; - if (match1(parser, PM_TOKEN_EOF)) { - unescaped = PM_STRING_EMPTY; - content = not_provided(parser); - } else { - unescaped = parser->current_string; - expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_EXPECT_STRING_CONTENT); - content = parser->previous; - } - - // It is unfortunately possible to have multiple string content - // nodes in a row in the case that there's heredoc content in - // the middle of the string, like this cursed example: - // - // <<-END+'b - // a - // END - // c'+'d' - // - // In that case we need to switch to an interpolated string to - // be able to contain all of the parts. - if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_node_list_t parts = { 0 }; - - pm_token_t delimiters = not_provided(parser); - pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &delimiters, &content, &delimiters, &unescaped); - pm_node_list_append(&parts, part); - - do { - part = (pm_node_t *) pm_string_node_create_current_string(parser, &delimiters, &parser->current, &delimiters); - pm_node_list_append(&parts, part); - parser_lex(parser); - } while (match1(parser, PM_TOKEN_STRING_CONTENT)); - - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); - node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); - - pm_node_list_free(&parts); - } else if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { - node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); - } else if (match1(parser, PM_TOKEN_EOF)) { - pm_parser_err_token(parser, &opening, PM_ERR_STRING_LITERAL_EOF); - node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); - } else if (accept1(parser, PM_TOKEN_STRING_END)) { - node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); - } else { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); - parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; - node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); - } - } else if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - // In this case we've hit string content so we know the string - // at least has something in it. We'll need to check if the - // following token is the end (in which case we can return a - // plain string) or if it's not then it has interpolation. - pm_token_t content = parser->current; - pm_string_t unescaped = parser->current_string; - parser_lex(parser); - - if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { - node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); - pm_node_flag_set(node, parse_unescaped_encoding(parser)); - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); - } else if (accept1(parser, PM_TOKEN_LABEL_END)) { - node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); - } else { - // If we get here, then we have interpolation so we'll need - // to create a string or symbol node with interpolation. - pm_node_list_t parts = { 0 }; - pm_token_t string_opening = not_provided(parser); - pm_token_t string_closing = not_provided(parser); - - pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &string_opening, &parser->previous, &string_closing, &unescaped); - pm_node_flag_set(part, parse_unescaped_encoding(parser)); - pm_node_list_append(&parts, part); - - while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) { - if ((part = parse_string_part(parser)) != NULL) { - pm_node_list_append(&parts, part); - } - } - - if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { - node = (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); - } else if (match1(parser, PM_TOKEN_EOF)) { - pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM); - node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current); - } else { - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); - node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); - } - - pm_node_list_free(&parts); - } - } else { - // If we get here, then the first part of the string is not plain - // string content, in which case we need to parse the string as an - // interpolated string. - pm_node_list_t parts = { 0 }; - pm_node_t *part; - - while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) { - if ((part = parse_string_part(parser)) != NULL) { - pm_node_list_append(&parts, part); - } - } - - if (accept1(parser, PM_TOKEN_LABEL_END)) { - node = (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); - } else if (match1(parser, PM_TOKEN_EOF)) { - pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM); - node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current); - } else { - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); - node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); - } - - pm_node_list_free(&parts); - } - - if (current == NULL) { - // If the node we just parsed is a symbol node, then we can't - // concatenate it with anything else, so we can now return that - // node. - if (PM_NODE_TYPE_P(node, PM_SYMBOL_NODE) || PM_NODE_TYPE_P(node, PM_INTERPOLATED_SYMBOL_NODE)) { - return node; - } - - // If we don't already have a node, then it's fine and we can just - // set the result to be the node we just parsed. - current = node; - } else { - // Otherwise we need to check the type of the node we just parsed. - // If it cannot be concatenated with the previous node, then we'll - // need to add a syntax error. - if (!PM_NODE_TYPE_P(node, PM_STRING_NODE) && !PM_NODE_TYPE_P(node, PM_INTERPOLATED_STRING_NODE)) { - pm_parser_err_node(parser, node, PM_ERR_STRING_CONCATENATION); - } - - // If we haven't already created our container for concatenation, - // we'll do that now. - if (!concating) { - concating = true; - pm_token_t bounds = not_provided(parser); - - pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); - pm_interpolated_string_node_append(container, current); - current = (pm_node_t *) container; - } - - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, node); - } - } - - return current; -} - -/** * Append an error to the error list on the parser using the given diagnostic * ID. This function is a specialization that handles formatting the specific * kind of error that is being appended. @@ -16977,6 +17105,11 @@ pm_parser_err_prefix(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, pm_token_type_human(parser->previous.type)); break; } + case PM_ERR_HASH_VALUE: + case PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR: { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + break; + } case PM_ERR_UNARY_RECEIVER: { const char *human = (parser->current.type == PM_TOKEN_EOF ? "end-of-input" : pm_token_type_human(parser->current.type)); PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, human, parser->previous.start[0]); @@ -17059,6 +17192,7 @@ parse_retry(pm_parser_t *parser, const pm_node_t *node) { case PM_CONTEXT_IF: case PM_CONTEXT_LAMBDA_BRACES: case PM_CONTEXT_LAMBDA_DO_END: + case PM_CONTEXT_LOOP_PREDICATE: case PM_CONTEXT_PARENS: case PM_CONTEXT_POSTEXE: case PM_CONTEXT_PREDICATE: @@ -17137,6 +17271,7 @@ parse_yield(pm_parser_t *parser, const pm_node_t *node) { case PM_CONTEXT_LAMBDA_ELSE: case PM_CONTEXT_LAMBDA_ENSURE: case PM_CONTEXT_LAMBDA_RESCUE: + case PM_CONTEXT_LOOP_PREDICATE: case PM_CONTEXT_PARENS: case PM_CONTEXT_POSTEXE: case PM_CONTEXT_PREDICATE: @@ -17156,6 +17291,63 @@ parse_yield(pm_parser_t *parser, const pm_node_t *node) { } /** + * This struct is used to pass information between the regular expression parser + * and the error callback. + */ +typedef struct { + /** The parser that we are parsing the regular expression for. */ + pm_parser_t *parser; + + /** The start of the regular expression. */ + const uint8_t *start; + + /** The end of the regular expression. */ + const uint8_t *end; + + /** + * Whether or not the source of the regular expression is shared. This + * impacts the location of error messages, because if it is shared then we + * can use the location directly and if it is not, then we use the bounds of + * the regular expression itself. + */ + bool shared; +} parse_regular_expression_error_data_t; + +/** + * This callback is called when the regular expression parser encounters a + * syntax error. + */ +static void +parse_regular_expression_error(const uint8_t *start, const uint8_t *end, const char *message, void *data) { + parse_regular_expression_error_data_t *callback_data = (parse_regular_expression_error_data_t *) data; + pm_location_t location; + + if (callback_data->shared) { + location = (pm_location_t) { .start = start, .end = end }; + } else { + location = (pm_location_t) { .start = callback_data->start, .end = callback_data->end }; + } + + PM_PARSER_ERR_FORMAT(callback_data->parser, location.start, location.end, PM_ERR_REGEXP_PARSE_ERROR, message); +} + +/** + * Parse the errors for the regular expression and add them to the parser. + */ +static void +parse_regular_expression_errors(pm_parser_t *parser, pm_regular_expression_node_t *node) { + const pm_string_t *unescaped = &node->unescaped; + parse_regular_expression_error_data_t error_data = { + .parser = parser, + .start = node->base.location.start, + .end = node->base.location.end, + .shared = unescaped->type == PM_STRING_SHARED + }; + + pm_regexp_parse(parser, pm_string_source(unescaped), pm_string_length(unescaped), NULL, NULL, parse_regular_expression_error, &error_data); +} + +/** * Parse an expression that begins with the previous node that we just lexed. */ static inline pm_node_t * @@ -17175,8 +17367,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } - if (pm_array_node_size(array) != 0) { - expect1(parser, PM_TOKEN_COMMA, PM_ERR_ARRAY_SEPARATOR); + // Ensure that we have a comma between elements in the array. + if ((pm_array_node_size(array) != 0) && !accept1(parser, PM_TOKEN_COMMA)) { + const uint8_t *location = parser->previous.end; + PM_PARSER_ERR_FORMAT(parser, location, location, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); + + parser->previous.start = location; + parser->previous.type = PM_TOKEN_MISSING; } // If we have a right bracket immediately following a comma, @@ -17354,7 +17551,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we didn't find a terminator and we didn't find a right // parenthesis, then this is a syntax error. - if (!terminator_found) { + if (!terminator_found && !match1(parser, PM_TOKEN_EOF)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } @@ -17383,7 +17580,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) break; } else if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { break; - } else { + } else if (!match1(parser, PM_TOKEN_EOF)) { + // If we're at the end of the file, then we're going to add + // an error after this for the ) anyway. PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } } @@ -17602,8 +17801,28 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b ) { pm_arguments_t arguments = { 0 }; parse_arguments_list(parser, &arguments, true, accepts_command_call); - pm_call_node_t *fcall = pm_call_node_fcall_create(parser, &identifier, &arguments); + + if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + // If we're about to convert an 'it' implicit local + // variable read into a method call, we need to remove + // it from the list of implicit local variables. + parse_target_implicit_parameter(parser, node); + } else { + // Otherwise, we're about to convert a regular local + // variable read into a method call, in which case we + // need to indicate that this was not a read for the + // purposes of warnings. + assert(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)); + + if (pm_token_is_numbered_parameter(identifier.start, identifier.end)) { + parse_target_implicit_parameter(parser, node); + } else { + pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node; + pm_locals_unread(&pm_parser_scope_find(parser, cast->depth)->locals, cast->name); + } + } + pm_node_destroy(parser, node); return (pm_node_t *) fcall; } @@ -17611,31 +17830,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if ((binding_power == PM_BINDING_POWER_STATEMENT) && match1(parser, PM_TOKEN_COMMA)) { node = parse_targets_validate(parser, node, PM_BINDING_POWER_INDEX); - } else { - // Check if `it` is not going to be assigned. - switch (parser->current.type) { - case PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL: - case PM_TOKEN_AMPERSAND_EQUAL: - case PM_TOKEN_CARET_EQUAL: - case PM_TOKEN_EQUAL: - case PM_TOKEN_GREATER_GREATER_EQUAL: - case PM_TOKEN_LESS_LESS_EQUAL: - case PM_TOKEN_MINUS_EQUAL: - case PM_TOKEN_PARENTHESIS_RIGHT: - case PM_TOKEN_PERCENT_EQUAL: - case PM_TOKEN_PIPE_EQUAL: - case PM_TOKEN_PIPE_PIPE_EQUAL: - case PM_TOKEN_PLUS_EQUAL: - case PM_TOKEN_SLASH_EQUAL: - case PM_TOKEN_STAR_EQUAL: - case PM_TOKEN_STAR_STAR_EQUAL: - break; - default: - // Once we know it's neither a method call nor an - // assignment, we can finally create `it` default - // parameter. - node = pm_node_check_it(parser, node); - } } return node; @@ -17896,6 +18090,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // as frozen because when clause strings are frozen. if (PM_NODE_TYPE_P(condition, PM_STRING_NODE)) { pm_node_flag_set(condition, PM_STRING_FLAGS_FROZEN | PM_NODE_FLAG_STATIC_LITERAL); + } else if (PM_NODE_TYPE_P(condition, PM_SOURCE_FILE_NODE)) { + pm_node_flag_set(condition, PM_NODE_FLAG_STATIC_LITERAL); } pm_when_clause_static_literals_add(parser, &literals, condition); @@ -17952,7 +18148,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t in_keyword = parser->previous; pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -17981,7 +18177,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b then_keyword = not_provided(parser); } } else { - expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_WHEN_DELIMITER); + expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_IN_DELIMITER); then_keyword = parser->previous; } @@ -18301,7 +18497,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match2(parser, PM_TOKEN_DOT, PM_TOKEN_COLON_COLON)) { receiver = parse_variable_call(parser); - receiver = pm_node_check_it(parser, receiver); pm_parser_scope_push(parser, true); lex_state_set(parser, PM_LEX_STATE_FNAME); @@ -18435,7 +18630,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; - expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_DEF_PARAMS_TERM_PAREN); + if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + } + rparen = parser->previous; break; } @@ -18633,7 +18833,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_COMMA)) { index = parse_targets(parser, index, PM_BINDING_POWER_INDEX); } else { - index = parse_target(parser, index, false); + index = parse_target(parser, index, false, false); } context_pop(parser); @@ -18822,12 +19022,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_lex(parser); return (pm_node_t *) pm_true_node_create(parser, &parser->previous); case PM_TOKEN_KEYWORD_UNTIL: { + context_push(parser, PM_CONTEXT_LOOP_PREDICATE); pm_do_loop_stack_push(parser, true); + parser_lex(parser); pm_token_t keyword = parser->previous; - pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, PM_ERR_CONDITIONAL_UNTIL_PREDICATE); + pm_do_loop_stack_pop(parser); + context_pop(parser); expect3(parser, PM_TOKEN_KEYWORD_DO_LOOP, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_UNTIL_PREDICATE); pm_statements_node_t *statements = NULL; @@ -18843,12 +19046,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b return (pm_node_t *) pm_until_node_create(parser, &keyword, &parser->previous, predicate, statements, 0); } case PM_TOKEN_KEYWORD_WHILE: { + context_push(parser, PM_CONTEXT_LOOP_PREDICATE); pm_do_loop_stack_push(parser, true); + parser_lex(parser); pm_token_t keyword = parser->previous; - pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, PM_ERR_CONDITIONAL_WHILE_PREDICATE); + pm_do_loop_stack_pop(parser); + context_pop(parser); expect3(parser, PM_TOKEN_KEYWORD_DO_LOOP, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_WHILE_PREDICATE); pm_statements_node_t *statements = NULL; @@ -19268,13 +19474,22 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b bool ascii_only = parser->current_regular_expression_ascii_only; parser_lex(parser); - // If we hit an end, then we can create a regular expression node - // without interpolation, which can be represented more succinctly and - // more easily compiled. + // If we hit an end, then we can create a regular expression + // node without interpolation, which can be represented more + // succinctly and more easily compiled. if (accept1(parser, PM_TOKEN_REGEXP_END)) { - pm_node_t *node = (pm_node_t *) pm_regular_expression_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); - pm_node_flag_set(node, parse_and_validate_regular_expression_encoding(parser, &unescaped, ascii_only, node->flags)); - return node; + pm_regular_expression_node_t *node = (pm_regular_expression_node_t *) pm_regular_expression_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped); + + // If we're not immediately followed by a =~, then we want + // to parse all of the errors at this point. If it is + // followed by a =~, then it will get parsed higher up while + // parsing the named captures as well. + if (!match1(parser, PM_TOKEN_EQUAL_TILDE)) { + parse_regular_expression_errors(parser, node); + } + + pm_node_flag_set((pm_node_t *) node, parse_and_validate_regular_expression_encoding(parser, &unescaped, ascii_only, node->base.flags)); + return (pm_node_t *) node; } // If we get here, then we have interpolation so we'll need to create @@ -19492,9 +19707,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b switch (parser->current.type) { case PM_TOKEN_PARENTHESIS_LEFT: { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_token_t opening = parser->current; parser_lex(parser); @@ -19511,9 +19723,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_CASE_PARAMETER: { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_accepts_block_stack_push(parser, false); pm_token_t opening = not_provided(parser); block_parameters = parse_block_parameters(parser, false, &opening, true); @@ -19766,122 +19975,126 @@ parse_call_operator_write(pm_parser_t *parser, pm_call_node_t *call_node, const } /** - * Returns true if the name of the capture group is a valid local variable that - * can be written to. + * This struct is used to pass information between the regular expression parser + * and the named capture callback. */ -static bool -parse_regular_expression_named_capture(pm_parser_t *parser, const uint8_t *source, size_t length) { - if (length == 0) { - return false; - } +typedef struct { + /** The parser that is parsing the regular expression. */ + pm_parser_t *parser; - // First ensure that it starts with a valid identifier starting character. - size_t width = char_is_identifier_start(parser, source); - if (!width) { - return false; - } + /** The call node wrapping the regular expression node. */ + pm_call_node_t *call; - // Next, ensure that it's not an uppercase character. - if (parser->encoding_changed) { - if (parser->encoding->isupper_char(source, (ptrdiff_t) length)) return false; - } else { - if (pm_encoding_utf_8_isupper_char(source, (ptrdiff_t) length)) return false; - } + /** The match write node that is being created. */ + pm_match_write_node_t *match; - // Next, iterate through all of the bytes of the string to ensure that they - // are all valid identifier characters. - const uint8_t *cursor = source + width; - while (cursor < source + length && (width = char_is_identifier(parser, cursor))) { - cursor += width; - } + /** The list of names that have been parsed. */ + pm_constant_id_list_t names; - return cursor == source + length; -} + /** + * Whether the content of the regular expression is shared. This impacts + * whether or not we used owned constants or shared constants in the + * constant pool for the names of the captures. + */ + bool shared; +} parse_regular_expression_named_capture_data_t; /** - * Potentially change a =~ with a regular expression with named captures into a - * match write node. + * This callback is called when the regular expression parser encounters a named + * capture group. */ -static pm_node_t * -parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t *content, pm_call_node_t *call) { - pm_string_list_t named_captures = { 0 }; - pm_node_t *result; +static void +parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { + parse_regular_expression_named_capture_data_t *callback_data = (parse_regular_expression_named_capture_data_t *) data; - if (pm_regexp_named_capture_group_names(pm_string_source(content), pm_string_length(content), &named_captures, parser->encoding_changed, parser->encoding) && (named_captures.length > 0)) { - // Since we should not create a MatchWriteNode when all capture names - // are invalid, creating a MatchWriteNode is delaid here. - pm_match_write_node_t *match = NULL; - pm_constant_id_list_t names = { 0 }; + pm_parser_t *parser = callback_data->parser; + pm_call_node_t *call = callback_data->call; + pm_constant_id_list_t *names = &callback_data->names; - for (size_t index = 0; index < named_captures.length; index++) { - pm_string_t *string = &named_captures.strings[index]; + const uint8_t *source = pm_string_source(capture); + size_t length = pm_string_length(capture); - const uint8_t *source = pm_string_source(string); - size_t length = pm_string_length(string); + pm_location_t location; + pm_constant_id_t name; - pm_location_t location; - pm_constant_id_t name; + // If the name of the capture group isn't a valid identifier, we do + // not add it to the local table. + if (!pm_slice_is_valid_local(parser, source, source + length)) return; - // If the name of the capture group isn't a valid identifier, we do - // not add it to the local table. - if (!parse_regular_expression_named_capture(parser, source, length)) continue; + if (callback_data->shared) { + // If the unescaped string is a slice of the source, then we can + // copy the names directly. The pointers will line up. + location = (pm_location_t) { .start = source, .end = source + length }; + name = pm_parser_constant_id_location(parser, location.start, location.end); + } else { + // Otherwise, the name is a slice of the malloc-ed owned string, + // in which case we need to copy it out into a new string. + location = (pm_location_t) { .start = call->receiver->location.start, .end = call->receiver->location.end }; - if (content->type == PM_STRING_SHARED) { - // If the unescaped string is a slice of the source, then we can - // copy the names directly. The pointers will line up. - location = (pm_location_t) { .start = source, .end = source + length }; - name = pm_parser_constant_id_location(parser, location.start, location.end); - } else { - // Otherwise, the name is a slice of the malloc-ed owned string, - // in which case we need to copy it out into a new string. - location = call->receiver->location; + void *memory = xmalloc(length); + if (memory == NULL) abort(); - void *memory = xmalloc(length); - if (memory == NULL) abort(); + memcpy(memory, source, length); + name = pm_parser_constant_id_owned(parser, (uint8_t *) memory, length); + } - memcpy(memory, source, length); - name = pm_parser_constant_id_owned(parser, (uint8_t *) memory, length); - } + // Add this name to the list of constants if it is valid, not duplicated, + // and not a keyword. + if (name != 0 && !pm_constant_id_list_includes(names, name)) { + pm_constant_id_list_append(names, name); - if (name != 0) { - // We dont want to create duplicate targets if the capture name - // is duplicated. - if (pm_constant_id_list_includes(&names, name)) continue; - pm_constant_id_list_append(&names, name); + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, name)) == -1) { + // If the local is not already a local but it is a keyword, then we + // do not want to add a capture for this. + if (pm_local_is_keyword((const char *) source, length)) return; - int depth; - if ((depth = pm_parser_local_depth_constant_id(parser, name)) == -1) { - // If the identifier is not already a local, then we'll add - // it to the local table unless it's a keyword. - if (pm_local_is_keyword((const char *) source, length)) continue; + // If the identifier is not already a local, then we will add it to + // the local table. + pm_parser_local_add(parser, name, location.start, location.end, 0); + } - pm_parser_local_add(parser, name, location.start, location.end, 0); - } + // Here we lazily create the MatchWriteNode since we know we're + // about to add a target. + if (callback_data->match == NULL) { + callback_data->match = pm_match_write_node_create(parser, call); + } - // Here we lazily create the MatchWriteNode since we know we're - // about to add a target. - if (match == NULL) match = pm_match_write_node_create(parser, call); + // Next, create the local variable target and add it to the list of + // targets for the match. + pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); + pm_node_list_append(&callback_data->match->targets, target); + } +} - // Next, create the local variable target and add it to the - // list of targets for the match. - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); - pm_node_list_append(&match->targets, target); - } - } +/** + * Potentially change a =~ with a regular expression with named captures into a + * match write node. + */ +static pm_node_t * +parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t *content, pm_call_node_t *call) { + parse_regular_expression_named_capture_data_t callback_data = { + .parser = parser, + .call = call, + .names = { 0 }, + .shared = content->type == PM_STRING_SHARED + }; - if (match != NULL) { - result = (pm_node_t *) match; - } else { - result = (pm_node_t *) call; - } + parse_regular_expression_error_data_t error_data = { + .parser = parser, + .start = call->receiver->location.start, + .end = call->receiver->location.end, + .shared = content->type == PM_STRING_SHARED + }; - pm_constant_id_list_free(&names); + pm_regexp_parse(parser, pm_string_source(content), pm_string_length(content), parse_regular_expression_named_capture, &callback_data, parse_regular_expression_error, &error_data); + pm_constant_id_list_free(&callback_data.names); + + if (callback_data.match != NULL) { + return (pm_node_t *) callback_data.match; } else { - result = (pm_node_t *) call; + return (pm_node_t *) call; } - - pm_string_list_free(&named_captures); - return result; } static inline pm_node_t * @@ -19998,7 +20211,6 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_CALL_NODE: { - parser_lex(parser); pm_call_node_t *cast = (pm_call_node_t *) node; // If we have a vcall (a method with no arguments and no @@ -20009,6 +20221,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + parser_lex(parser); + pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ); pm_node_t *result = (pm_node_t *) pm_local_variable_and_write_node_create(parser, (pm_node_t *) cast, &token, value, constant_id, 0); @@ -20016,6 +20230,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } + // Move past the token here so that we have already added + // the local variable by this point. + parser_lex(parser); + // If there is no call operator and the message is "[]" then // this is an aref expression, and we can transform it into // an aset expression. @@ -20111,7 +20329,6 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_CALL_NODE: { - parser_lex(parser); pm_call_node_t *cast = (pm_call_node_t *) node; // If we have a vcall (a method with no arguments and no @@ -20122,6 +20339,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + parser_lex(parser); + pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ); pm_node_t *result = (pm_node_t *) pm_local_variable_or_write_node_create(parser, (pm_node_t *) cast, &token, value, constant_id, 0); @@ -20129,6 +20348,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } + // Move past the token here so that we have already added + // the local variable by this point. + parser_lex(parser); + // If there is no call operator and the message is "[]" then // this is an aref expression, and we can transform it into // an aset expression. @@ -20282,7 +20505,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // In this case we have an operator but we don't know what it's for. // We need to treat it as an error. For now, we'll mark it as an error // and just skip right past it. - pm_parser_err_previous(parser, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR); + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); return node; } } @@ -20538,7 +20761,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if ( (parser->current.type == PM_TOKEN_PARENTHESIS_LEFT) || - (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR)) + (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR))) ) { // If we have a constant immediately following a '::' operator, then // this can either be a constant path or a method call, depending on @@ -20664,7 +20887,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -20681,7 +20904,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -20694,6 +20917,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t } } +#undef PM_PARSE_PATTERN_SINGLE +#undef PM_PARSE_PATTERN_TOP +#undef PM_PARSE_PATTERN_MULTI + /** * Parse an expression at the given point of the parser using the given binding * power to parse subsequent chains. If this function finds a syntax error, it @@ -21077,7 +21304,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // Scopes given from the outside are not allowed to have numbered // parameters. - parser->current_scope->numbered_parameters = PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED; + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED; for (size_t local_index = 0; local_index < scope->locals_count; local_index++) { const pm_string_t *local = pm_options_scope_local_get(scope, local_index); @@ -21465,331 +21692,3 @@ pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t s } #endif - -/** An error that is going to be formatted into the output. */ -typedef struct { - /** A pointer to the diagnostic that was generated during parsing. */ - pm_diagnostic_t *error; - - /** The start line of the diagnostic message. */ - int32_t line; - - /** The column start of the diagnostic message. */ - uint32_t column_start; - - /** The column end of the diagnostic message. */ - uint32_t column_end; -} pm_error_t; - -/** The format that will be used to format the errors into the output. */ -typedef struct { - /** The prefix that will be used for line numbers. */ - const char *number_prefix; - - /** The prefix that will be used for blank lines. */ - const char *blank_prefix; - - /** The divider that will be used between sections of source code. */ - const char *divider; - - /** The length of the blank prefix. */ - size_t blank_prefix_length; - - /** The length of the divider. */ - size_t divider_length; -} pm_error_format_t; - -#define PM_COLOR_GRAY "\033[38;5;102m" -#define PM_COLOR_RED "\033[1;31m" -#define PM_COLOR_RESET "\033[m" - -static inline pm_error_t * -pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { - pm_error_t *errors = xcalloc(error_list->size, sizeof(pm_error_t)); - if (errors == NULL) return NULL; - - int32_t start_line = parser->start_line; - for (pm_diagnostic_t *error = (pm_diagnostic_t *) error_list->head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { - pm_line_column_t start = pm_newline_list_line_column(newline_list, error->location.start, start_line); - pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.end, start_line); - - // We're going to insert this error into the array in sorted order. We - // do this by finding the first error that has a line number greater - // than the current error and then inserting the current error before - // that one. - size_t index = 0; - while ( - (index < error_list->size) && - (errors[index].error != NULL) && - ( - (errors[index].line < start.line) || - ((errors[index].line == start.line) && (errors[index].column_start < start.column)) - ) - ) index++; - - // Now we're going to shift all of the errors after this one down one - // index to make room for the new error. - if (index + 1 < error_list->size) { - memmove(&errors[index + 1], &errors[index], sizeof(pm_error_t) * (error_list->size - index - 1)); - } - - // Finally, we'll insert the error into the array. - uint32_t column_end; - if (start.line == end.line) { - column_end = end.column; - } else { - column_end = (uint32_t) (newline_list->offsets[start.line - start_line + 1] - newline_list->offsets[start.line - start_line] - 1); - } - - // Ensure we have at least one column of error. - if (start.column == column_end) column_end++; - - errors[index] = (pm_error_t) { - .error = error, - .line = start.line, - .column_start = start.column, - .column_end = column_end - }; - } - - return errors; -} - -static inline void -pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, pm_buffer_t *buffer) { - int32_t line_delta = line - parser->start_line; - assert(line_delta >= 0); - - size_t index = (size_t) line_delta; - assert(index < newline_list->size); - - const uint8_t *start = &parser->start[newline_list->offsets[index]]; - const uint8_t *end; - - if (index >= newline_list->size - 1) { - end = parser->end; - } else { - end = &parser->start[newline_list->offsets[index + 1]]; - } - - pm_buffer_append_format(buffer, number_prefix, line); - pm_buffer_append_string(buffer, (const char *) start, (size_t) (end - start)); - - if (end == parser->end && end[-1] != '\n') { - pm_buffer_append_string(buffer, "\n", 1); - } -} - -/** - * Format the errors on the parser into the given buffer. - */ -PRISM_EXPORTED_FUNCTION void -pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { - assert(error_list->size != 0); - - // First, we're going to sort all of the errors by line number using an - // insertion sort into a newly allocated array. - const int32_t start_line = parser->start_line; - const pm_newline_list_t *newline_list = &parser->newline_list; - - pm_error_t *errors = pm_parser_errors_format_sort(parser, error_list, newline_list); - if (errors == NULL) return; - - // Now we're going to determine how we're going to format line numbers and - // blank lines based on the maximum number of digits in the line numbers - // that are going to be displaid. - pm_error_format_t error_format; - int32_t first_line_number = errors[0].line; - int32_t last_line_number = errors[error_list->size - 1].line; - - // If we have a maximum line number that is negative, then we're going to - // use the absolute value for comparison but multiple by 10 to additionally - // have a column for the negative sign. - if (first_line_number < 0) first_line_number = (-first_line_number) * 10; - if (last_line_number < 0) last_line_number = (-last_line_number) * 10; - int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; - - if (max_line_number < 10) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%1" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%1" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~\n" - }; - } - } else if (max_line_number < 100) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%2" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%2" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~\n" - }; - } - } else if (max_line_number < 1000) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%3" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%3" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~\n" - }; - } - } else if (max_line_number < 10000) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%4" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%4" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~~\n" - }; - } - } else { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%5" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%5" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~~\n" - }; - } - } - - error_format.blank_prefix_length = strlen(error_format.blank_prefix); - error_format.divider_length = strlen(error_format.divider); - - // Now we're going to iterate through every error in our error list and - // display it. While we're iterating, we will display some padding lines of - // the source before the error to give some context. We'll be careful not to - // display the same line twice in case the errors are close enough in the - // source. - int32_t last_line = parser->start_line - 1; - const pm_encoding_t *encoding = parser->encoding; - - for (size_t index = 0; index < error_list->size; index++) { - pm_error_t *error = &errors[index]; - - // Here we determine how many lines of padding of the source to display, - // based on the difference from the last line that was displaid. - if (error->line - last_line > 1) { - if (error->line - last_line > 2) { - if ((index != 0) && (error->line - last_line > 3)) { - pm_buffer_append_string(buffer, error_format.divider, error_format.divider_length); - } - - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 2, buffer); - } - - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 1, buffer); - } - - // If this is the first error or we're on a new line, then we'll display - // the line that has the error in it. - if ((index == 0) || (error->line != last_line)) { - if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); - } else { - pm_buffer_append_string(buffer, "> ", 2); - } - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, buffer); - } - - const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; - if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); - - // Now we'll display the actual error message. We'll do this by first - // putting the prefix to the line, then a bunch of blank spaces - // depending on the column, then as many carets as we need to display - // the width of the error, then the error message itself. - // - // Note that this doesn't take into account the width of the actual - // character when displaid in the terminal. For some east-asian - // languages or emoji, this means it can be thrown off pretty badly. We - // will need to solve this eventually. - pm_buffer_append_string(buffer, " ", 2); - pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); - - size_t column = 0; - while (column < error->column_start) { - pm_buffer_append_byte(buffer, ' '); - - size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); - column += (char_width == 0 ? 1 : char_width); - } - - if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RED, 7); - pm_buffer_append_byte(buffer, '^'); - - size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); - column += (char_width == 0 ? 1 : char_width); - - while (column < error->column_end) { - pm_buffer_append_byte(buffer, '~'); - - size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); - column += (char_width == 0 ? 1 : char_width); - } - - if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RESET, 3); - - if (inline_messages) { - pm_buffer_append_byte(buffer, ' '); - assert(error->error != NULL); - - const char *message = error->error->message; - pm_buffer_append_string(buffer, message, strlen(message)); - } - - pm_buffer_append_byte(buffer, '\n'); - - // Here we determine how many lines of padding to display after the - // error, depending on where the next error is in source. - last_line = error->line; - int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; - - if (next_line - last_line > 1) { - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, buffer); - } - - if (next_line - last_line > 1) { - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, buffer); - } - } - - // Finally, we'll free the array of errors that we allocated. - xfree(errors); -} - -#undef PM_COLOR_GRAY -#undef PM_COLOR_RED -#undef PM_COLOR_RESET diff --git a/prism/prism.h b/prism/prism.h index 59067c3021..755c38fca2 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -219,17 +219,6 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_ty */ const char * pm_token_type_human(pm_token_type_t token_type); -/** - * Format the errors on the parser into the given buffer. - * - * @param parser The parser to format the errors for. - * @param error_list The list of errors to format. - * @param buffer The buffer to format the errors into. - * @param colorize Whether or not to colorize the errors with ANSI escape sequences. - * @param inline_messages Whether or not to inline the messages with the source. - */ -PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages); - // We optionally support dumping to JSON. For systems that don't want or need // this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define. #ifndef PRISM_EXCLUDE_JSON diff --git a/prism/regexp.c b/prism/regexp.c index 6e0fdd295c..9eea90e12f 100644 --- a/prism/regexp.c +++ b/prism/regexp.c @@ -1,9 +1,14 @@ #include "prism/regexp.h" +#define PM_REGEXP_PARSE_DEPTH_MAX 4096 + /** * This is the parser that is going to handle parsing regular expressions. */ typedef struct { + /** The parser that is currently being used. */ + pm_parser_t *parser; + /** A pointer to the start of the source that we are parsing. */ const uint8_t *start; @@ -13,39 +18,42 @@ typedef struct { /** A pointer to the end of the source that we are parsing. */ const uint8_t *end; - /** A list of named captures that we've found. */ - pm_string_list_t *named_captures; - /** Whether the encoding has changed from the default. */ bool encoding_changed; /** The encoding of the source. */ const pm_encoding_t *encoding; + + /** The callback to call when a named capture group is found. */ + pm_regexp_name_callback_t name_callback; + + /** The data to pass to the name callback. */ + void *name_data; + + /** The callback to call when a parse error is found. */ + pm_regexp_error_callback_t error_callback; + + /** The data to pass to the error callback. */ + void *error_data; } pm_regexp_parser_t; /** - * This initializes a new parser with the given source. + * Append an error to the parser. */ -static void -pm_regexp_parser_init(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_string_list_t *named_captures, bool encoding_changed, const pm_encoding_t *encoding) { - *parser = (pm_regexp_parser_t) { - .start = start, - .cursor = start, - .end = end, - .named_captures = named_captures, - .encoding_changed = encoding_changed, - .encoding = encoding - }; +static inline void +pm_regexp_parse_error(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, const char *message) { + parser->error_callback(start, end, message, parser->error_data); } /** - * This appends a new string to the list of named captures. + * This appends a new string to the list of named captures. This function + * assumes the caller has already checked the validity of the name callback. */ static void pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) { pm_string_t string; pm_string_shared_init(&string, start, end); - pm_string_list_append(parser->named_captures, &string); + parser->name_callback(&string, parser->name_data); pm_string_free(&string); } @@ -217,21 +225,24 @@ pm_regexp_parse_range_quantifier(pm_regexp_parser_t *parser) { */ static bool pm_regexp_parse_quantifier(pm_regexp_parser_t *parser) { - if (pm_regexp_char_is_eof(parser)) return true; - - switch (*parser->cursor) { - case '*': - case '+': - case '?': - parser->cursor++; - return true; - case '{': - parser->cursor++; - return pm_regexp_parse_range_quantifier(parser); - default: - // In this case there is no quantifier. - return true; + while (!pm_regexp_char_is_eof(parser)) { + switch (*parser->cursor) { + case '*': + case '+': + case '?': + parser->cursor++; + break; + case '{': + parser->cursor++; + if (!pm_regexp_parse_range_quantifier(parser)) return false; + break; + default: + // In this case there is no quantifier. + return true; + } } + + return true; } /** @@ -255,20 +266,20 @@ pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) { // Forward declaration because character sets can be nested. static bool -pm_regexp_parse_lbracket(pm_regexp_parser_t *parser); +pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth); /** * match-char-set : '[' '^'? (match-range | match-char)* ']' * ; */ static bool -pm_regexp_parse_character_set(pm_regexp_parser_t *parser) { +pm_regexp_parse_character_set(pm_regexp_parser_t *parser, uint16_t depth) { pm_regexp_char_accept(parser, '^'); while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ']') { switch (*parser->cursor++) { case '[': - pm_regexp_parse_lbracket(parser); + pm_regexp_parse_lbracket(parser, (uint16_t) (depth + 1)); break; case '\\': if (!pm_regexp_char_is_eof(parser)) { @@ -288,7 +299,18 @@ pm_regexp_parse_character_set(pm_regexp_parser_t *parser) { * A left bracket can either mean a POSIX class or a character set. */ static bool -pm_regexp_parse_lbracket(pm_regexp_parser_t *parser) { +pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth) { + if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) { + pm_regexp_parse_error(parser, parser->start, parser->end, "parse depth limit over"); + return false; + } + + if ((parser->cursor < parser->end) && parser->cursor[0] == ']') { + parser->cursor++; + pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "empty char-class"); + return true; + } + const uint8_t *reset = parser->cursor; if ((parser->cursor + 2 < parser->end) && parser->cursor[0] == '[' && parser->cursor[1] == ':') { @@ -298,13 +320,13 @@ pm_regexp_parse_lbracket(pm_regexp_parser_t *parser) { parser->cursor = reset; } - return pm_regexp_parse_character_set(parser); + return pm_regexp_parse_character_set(parser, depth); } // Forward declaration here since parsing groups needs to go back up the grammar // to parse expressions within them. static bool -pm_regexp_parse_expression(pm_regexp_parser_t *parser); +pm_regexp_parse_expression(pm_regexp_parser_t *parser, uint16_t depth); /** * These are the states of the options that are configurable on the regular @@ -418,17 +440,27 @@ pm_regexp_options_remove(pm_regexp_options_t *options, uint8_t key) { * * (?imxdau-imx:subexp) - turn on and off configuration for an expression */ static bool -pm_regexp_parse_group(pm_regexp_parser_t *parser) { +pm_regexp_parse_group(pm_regexp_parser_t *parser, uint16_t depth) { + const uint8_t *group_start = parser->cursor; + // First, parse any options for the group. if (pm_regexp_char_accept(parser, '?')) { if (pm_regexp_char_is_eof(parser)) { + pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern in group"); return false; } + pm_regexp_options_t options; pm_regexp_options_init(&options); switch (*parser->cursor) { case '#': { // inline comments + parser->cursor++; + if (pm_regexp_char_is_eof(parser)) { + pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern in group"); + return false; + } + if (parser->encoding_changed && parser->encoding->multibyte) { bool escaped = false; @@ -472,6 +504,7 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) { case '<': parser->cursor++; if (pm_regexp_char_is_eof(parser)) { + pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern with unmatched parenthesis"); return false; } @@ -485,7 +518,15 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) { if (!pm_regexp_char_find(parser, '>')) { return false; } - pm_regexp_parser_named_capture(parser, start, parser->cursor - 1); + + if (parser->cursor - start == 1) { + pm_regexp_parse_error(parser, start, parser->cursor, "group name is empty"); + } + + if (parser->name_callback != NULL) { + pm_regexp_parser_named_capture(parser, start, parser->cursor - 1); + } + break; } } @@ -496,7 +537,10 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) { return false; } - pm_regexp_parser_named_capture(parser, start, parser->cursor - 1); + if (parser->name_callback != NULL) { + pm_regexp_parser_named_capture(parser, start, parser->cursor - 1); + } + break; } case '(': // conditional expression @@ -535,20 +579,25 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) { } break; default: - return false; + parser->cursor++; + pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "undefined group option"); + break; } } // Now, parse the expressions within this group. while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ')') { - if (!pm_regexp_parse_expression(parser)) { + if (!pm_regexp_parse_expression(parser, (uint16_t) (depth + 1))) { return false; } pm_regexp_char_accept(parser, '|'); } // Finally, make sure we have a closing parenthesis. - return pm_regexp_char_expect(parser, ')'); + if (pm_regexp_char_expect(parser, ')')) return true; + + pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern with unmatched parenthesis"); + return false; } /** @@ -564,12 +613,12 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) { * ; */ static bool -pm_regexp_parse_item(pm_regexp_parser_t *parser) { +pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) { switch (*parser->cursor) { case '^': case '$': parser->cursor++; - return true; + return pm_regexp_parse_quantifier(parser); case '\\': parser->cursor++; if (!pm_regexp_char_is_eof(parser)) { @@ -578,10 +627,20 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser) { return pm_regexp_parse_quantifier(parser); case '(': parser->cursor++; - return pm_regexp_parse_group(parser) && pm_regexp_parse_quantifier(parser); + return pm_regexp_parse_group(parser, depth) && pm_regexp_parse_quantifier(parser); case '[': parser->cursor++; - return pm_regexp_parse_lbracket(parser) && pm_regexp_parse_quantifier(parser); + return pm_regexp_parse_lbracket(parser, depth) && pm_regexp_parse_quantifier(parser); + case '*': + case '?': + case '+': + parser->cursor++; + pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "target of repeat operator is not specified"); + return true; + case ')': + parser->cursor++; + pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "unmatched close parenthesis"); + return true; default: { size_t width; if (!parser->encoding_changed) { @@ -603,13 +662,18 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser) { * ; */ static bool -pm_regexp_parse_expression(pm_regexp_parser_t *parser) { - if (!pm_regexp_parse_item(parser)) { +pm_regexp_parse_expression(pm_regexp_parser_t *parser, uint16_t depth) { + if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) { + pm_regexp_parse_error(parser, parser->start, parser->end, "parse depth limit over"); + return false; + } + + if (!pm_regexp_parse_item(parser, depth)) { return false; } while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ')' && *parser->cursor != '|') { - if (!pm_regexp_parse_item(parser)) { + if (!pm_regexp_parse_item(parser, depth)) { return false; } } @@ -625,29 +689,30 @@ pm_regexp_parse_expression(pm_regexp_parser_t *parser) { */ static bool pm_regexp_parse_pattern(pm_regexp_parser_t *parser) { - return ( - ( - // Exit early if the pattern is empty. - pm_regexp_char_is_eof(parser) || - // Parse the first expression in the pattern. - pm_regexp_parse_expression(parser) - ) && - ( - // Return now if we've parsed the entire pattern. - pm_regexp_char_is_eof(parser) || - // Otherwise, we should have a pipe character. - (pm_regexp_char_expect(parser, '|') && pm_regexp_parse_pattern(parser)) - ) - ); + do { + if (pm_regexp_char_is_eof(parser)) return true; + if (!pm_regexp_parse_expression(parser, 0)) return false; + } while (pm_regexp_char_accept(parser, '|')); + + return pm_regexp_char_is_eof(parser); } /** * Parse a regular expression and extract the names of all of the named capture * groups. */ -PRISM_EXPORTED_FUNCTION bool -pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, const pm_encoding_t *encoding) { - pm_regexp_parser_t parser; - pm_regexp_parser_init(&parser, source, source + size, named_captures, encoding_changed, encoding); - return pm_regexp_parse_pattern(&parser); +PRISM_EXPORTED_FUNCTION void +pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, pm_regexp_name_callback_t name_callback, void *name_data, pm_regexp_error_callback_t error_callback, void *error_data) { + pm_regexp_parse_pattern(&(pm_regexp_parser_t) { + .parser = parser, + .start = source, + .cursor = source, + .end = source + size, + .encoding_changed = parser->encoding_changed, + .encoding = parser->encoding, + .name_callback = name_callback, + .name_data = name_data, + .error_callback = error_callback, + .error_data = error_data + }); } diff --git a/prism/regexp.h b/prism/regexp.h index c5ceab11f9..42bc504107 100644 --- a/prism/regexp.h +++ b/prism/regexp.h @@ -10,7 +10,6 @@ #include "prism/parser.h" #include "prism/encoding.h" #include "prism/util/pm_memchr.h" -#include "prism/util/pm_string_list.h" #include "prism/util/pm_string.h" #include <stdbool.h> @@ -18,16 +17,26 @@ #include <string.h> /** - * Parse a regular expression and extract the names of all of the named capture - * groups. + * This callback is called when a named capture group is found. + */ +typedef void (*pm_regexp_name_callback_t)(const pm_string_t *name, void *data); + +/** + * This callback is called when a parse error is found. + */ +typedef void (*pm_regexp_error_callback_t)(const uint8_t *start, const uint8_t *end, const char *message, void *data); + +/** + * Parse a regular expression. * + * @param parser The parser that is currently being used. * @param source The source code to parse. * @param size The size of the source code. - * @param named_captures The list to add the names of the named capture groups. - * @param encoding_changed Whether or not the encoding changed from the default. - * @param encoding The encoding of the source code. - * @return Whether or not the parsing was successful. + * @param name_callback The optional callback to call when a named capture group is found. + * @param name_data The optional data to pass to the name callback. + * @param error_callback The callback to call when a parse error is found. + * @param error_data The data to pass to the error callback. */ -PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, const pm_encoding_t *encoding); +PRISM_EXPORTED_FUNCTION void pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, pm_regexp_name_callback_t name_callback, void *name_data, pm_regexp_error_callback_t error_callback, void *error_data); #endif diff --git a/prism/static_literals.c b/prism/static_literals.c index 08f61c81fa..b8604321c9 100644 --- a/prism/static_literals.c +++ b/prism/static_literals.c @@ -59,6 +59,25 @@ murmur_hash(const uint8_t *key, size_t length) { } /** + * Hash the value of an integer and return it. + */ +static uint32_t +integer_hash(const pm_integer_t *integer) { + uint32_t hash; + if (integer->values) { + hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length); + } else { + hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t)); + } + + if (integer->negative) { + hash ^= murmur_scramble((uint32_t) 1); + } + + return hash; +} + +/** * Return the hash of the given node. It is important that nodes that have * equivalent static literal values have the same hash. This is because we use * these hashes to look for duplicates. @@ -68,19 +87,8 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: { // Integers hash their value. - const pm_integer_t *integer = &((const pm_integer_node_t *) node)->value; - uint32_t hash; - if (integer->values) { - hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length); - } else { - hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t)); - } - - if (integer->negative) { - hash ^= murmur_scramble((uint32_t) 1); - } - - return hash; + const pm_integer_node_t *cast = (const pm_integer_node_t *) node; + return integer_hash(&cast->value); } case PM_SOURCE_LINE_NODE: { // Source lines hash their line number. @@ -94,11 +102,9 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) return murmur_hash((const uint8_t *) value, sizeof(double)); } case PM_RATIONAL_NODE: { - // Rationals hash their numeric value. Because their numeric value - // is stored as a subnode, we hash that node and then mix in the - // fact that this is a rational node. - const pm_node_t *numeric = ((const pm_rational_node_t *) node)->numeric; - return node_hash(metadata, numeric) ^ murmur_scramble((uint32_t) node->type); + // Rationals hash their numerator and denominator. + const pm_rational_node_t *cast = (const pm_rational_node_t *) node; + return integer_hash(&cast->numerator) ^ integer_hash(&cast->denominator) ^ murmur_scramble((uint32_t) cast->base.type); } case PM_IMAGINARY_NODE: { // Imaginaries hash their numeric value. Because their numeric value @@ -148,7 +154,7 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) * and must be able to compare all node types that will be stored in this hash. */ static pm_node_t * -pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) { +pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, bool replace, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) { // If we are out of space, we need to resize the hash. This will cause all // of the nodes to be rehashed and reinserted into the new hash. if (hash->size * 2 >= hash->capacity) { @@ -196,9 +202,14 @@ pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *m // already in the hash. Otherwise, we can just increment the size and insert // the new node. pm_node_t *result = hash->nodes[index]; - if (result == NULL) hash->size++; - hash->nodes[index] = node; + if (result == NULL) { + hash->size++; + hash->nodes[index] = node; + } else if (replace) { + hash->nodes[index] = node; + } + return result; } @@ -275,8 +286,15 @@ pm_compare_number_nodes(const pm_static_literals_metadata_t *metadata, const pm_ switch (PM_NODE_TYPE(left)) { case PM_IMAGINARY_NODE: return pm_compare_number_nodes(metadata, ((const pm_imaginary_node_t *) left)->numeric, ((const pm_imaginary_node_t *) right)->numeric); - case PM_RATIONAL_NODE: - return pm_compare_number_nodes(metadata, ((const pm_rational_node_t *) left)->numeric, ((const pm_rational_node_t *) right)->numeric); + case PM_RATIONAL_NODE: { + const pm_rational_node_t *left_rational = (const pm_rational_node_t *) left; + const pm_rational_node_t *right_rational = (const pm_rational_node_t *) right; + + int result = pm_integer_compare(&left_rational->denominator, &right_rational->denominator); + if (result != 0) return result; + + return pm_integer_compare(&left_rational->numerator, &right_rational->numerator); + } case PM_INTEGER_NODE: return pm_compare_integer_nodes(metadata, left, right); case PM_FLOAT_NODE: @@ -335,7 +353,7 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_static_liter * Add a node to the set of static literals. */ pm_node_t * -pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node) { +pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -347,6 +365,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_integer_nodes ); case PM_FLOAT_NODE: @@ -358,6 +377,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_float_nodes ); case PM_RATIONAL_NODE: @@ -370,6 +390,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_number_nodes ); case PM_STRING_NODE: @@ -382,6 +403,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_string_nodes ); case PM_REGULAR_EXPRESSION_NODE: @@ -393,6 +415,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_regular_expression_nodes ); case PM_SYMBOL_NODE: @@ -404,26 +427,27 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_string_nodes ); case PM_TRUE_NODE: { pm_node_t *duplicated = literals->true_node; - literals->true_node = node; + if ((duplicated == NULL) || replace) literals->true_node = node; return duplicated; } case PM_FALSE_NODE: { pm_node_t *duplicated = literals->false_node; - literals->false_node = node; + if ((duplicated == NULL) || replace) literals->false_node = node; return duplicated; } case PM_NIL_NODE: { pm_node_t *duplicated = literals->nil_node; - literals->nil_node = node; + if ((duplicated == NULL) || replace) literals->nil_node = node; return duplicated; } case PM_SOURCE_ENCODING_NODE: { pm_node_t *duplicated = literals->source_encoding_node; - literals->source_encoding_node = node; + if ((duplicated == NULL) || replace) literals->source_encoding_node = node; return duplicated; } default: @@ -456,7 +480,7 @@ pm_static_literal_positive_p(const pm_node_t *node) { case PM_INTEGER_NODE: return !((const pm_integer_node_t *) node)->value.negative; case PM_RATIONAL_NODE: - return pm_static_literal_positive_p(((const pm_rational_node_t *) node)->numeric); + return !((const pm_rational_node_t *) node)->numerator.negative; case PM_IMAGINARY_NODE: return pm_static_literal_positive_p(((const pm_imaginary_node_t *) node)->numeric); default: @@ -466,43 +490,6 @@ pm_static_literal_positive_p(const pm_node_t *node) { } /** - * Inspect a rational node that wraps a float node. This is going to be a - * poor-man's version of the Ruby `Rational#to_s` method, because we're not - * going to try to reduce the rational by finding the GCD. We'll leave that for - * a future improvement. - */ -static void -pm_rational_inspect(pm_buffer_t *buffer, pm_rational_node_t *node) { - const uint8_t *start = node->base.location.start; - const uint8_t *end = node->base.location.end - 1; // r - - while (start < end && *start == '0') start++; // 0.1 -> .1 - while (end > start && end[-1] == '0') end--; // 1.0 -> 1. - size_t length = (size_t) (end - start); - - const uint8_t *point = memchr(start, '.', length); - assert(point && "should have a decimal point"); - - uint8_t *digits = malloc(length - 1); - if (digits == NULL) return; - - memcpy(digits, start, (unsigned long) (point - start)); - memcpy(digits + (point - start), point + 1, (unsigned long) (end - point - 1)); - - pm_integer_t numerator = { 0 }; - pm_integer_parse(&numerator, PM_INTEGER_BASE_DECIMAL, digits, digits + length - 1); - - pm_buffer_append_byte(buffer, '('); - pm_integer_string(buffer, &numerator); - pm_buffer_append_string(buffer, "/1", 2); - for (size_t index = 0; index < (size_t) (end - point - 1); index++) pm_buffer_append_byte(buffer, '0'); - pm_buffer_append_byte(buffer, ')'); - - pm_integer_free(&numerator); - free(digits); -} - -/** * Create a string-based representation of the given static literal. */ static inline void @@ -544,7 +531,9 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met pm_buffer_append_string(buffer, "(0", 2); if (pm_static_literal_positive_p(numeric)) pm_buffer_append_byte(buffer, '+'); pm_static_literal_inspect_node(buffer, metadata, numeric); - if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE)) pm_buffer_append_byte(buffer, '*'); + if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE)) { + pm_buffer_append_byte(buffer, '*'); + } pm_buffer_append_string(buffer, "i)", 2); break; } @@ -555,22 +544,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met pm_buffer_append_string(buffer, "nil", 3); break; case PM_RATIONAL_NODE: { - const pm_node_t *numeric = ((const pm_rational_node_t *) node)->numeric; - - switch (PM_NODE_TYPE(numeric)) { - case PM_INTEGER_NODE: - pm_buffer_append_byte(buffer, '('); - pm_static_literal_inspect_node(buffer, metadata, numeric); - pm_buffer_append_string(buffer, "/1)", 3); - break; - case PM_FLOAT_NODE: - pm_rational_inspect(buffer, (pm_rational_node_t *) node); - break; - default: - assert(false && "unreachable"); - break; - } - + const pm_rational_node_t *rational = (const pm_rational_node_t *) node; + pm_buffer_append_byte(buffer, '('); + pm_integer_string(buffer, &rational->numerator); + pm_buffer_append_byte(buffer, '/'); + pm_integer_string(buffer, &rational->denominator); + pm_buffer_append_byte(buffer, ')'); break; } case PM_REGULAR_EXPRESSION_NODE: { @@ -624,7 +603,7 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met /** * Create a string-based representation of the given static literal. */ -PRISM_EXPORTED_FUNCTION void +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node) { pm_static_literal_inspect_node( buffer, diff --git a/prism/static_literals.h b/prism/static_literals.h index 72706054c1..bd29761899 100644 --- a/prism/static_literals.h +++ b/prism/static_literals.h @@ -95,9 +95,10 @@ typedef struct { * @param start_line The line number that the parser starts on. * @param literals The set of static literals to add the node to. * @param node The node to add to the set. + * @param replace Whether to replace the previous node if one already exists. * @return A pointer to the node that is being overwritten, if there is one. */ -pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node); +pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); /** * Free the internal memory associated with the given static literals set. @@ -115,6 +116,6 @@ void pm_static_literals_free(pm_static_literals_t *literals); * @param encoding_name The name of the encoding of the source being parsed. * @param node The node to create a string representation of. */ -PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); #endif diff --git a/prism/templates/lib/prism/inspect_visitor.rb.erb b/prism/templates/lib/prism/inspect_visitor.rb.erb index 8e7902f0f1..9328da636b 100644 --- a/prism/templates/lib/prism/inspect_visitor.rb.erb +++ b/prism/templates/lib/prism/inspect_visitor.rb.erb @@ -116,13 +116,8 @@ module Prism # Compose a header for the given node. def inspect_node(name, node) - result = +"@ #{name} (" - location = node.location - result << "location: (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})" - result << ", newline: true" if node.newline? - - result << ")\n" + "@ #{name} (location: (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column}))\n" end # Compose a string representing the given inner location field. diff --git a/prism/templates/lib/prism/node.rb.erb b/prism/templates/lib/prism/node.rb.erb index 50e622a8d1..f0ce226def 100644 --- a/prism/templates/lib/prism/node.rb.erb +++ b/prism/templates/lib/prism/node.rb.erb @@ -28,18 +28,6 @@ module Prism location.is_a?(Location) ? location.end_offset : ((location >> 32) + (location & 0xFFFFFFFF)) end - def newline? # :nodoc: - @newline ? true : false - end - - def set_newline_flag(newline_marked) # :nodoc: - line = location.start_line - unless newline_marked[line] - newline_marked[line] = true - @newline = true - end - end - # Returns all of the lines of the source code associated with this node. def source_lines location.source_lines @@ -181,7 +169,6 @@ module Prism # def initialize: (<%= (node.fields.map { |field| "#{field.rbs_class} #{field.name}" } + ["Location location"]).join(", ") %>) -> void def initialize(source, <%= (node.fields.map(&:name) + ["location"]).join(", ") %>) @source = source - @newline = false @location = location <%- node.fields.each do |field| -%> <%- if Prism::Template::CHECK_FIELD_KIND && field.respond_to?(:check_field_kind) -%> @@ -195,25 +182,6 @@ module Prism def accept(visitor) visitor.visit_<%= node.human %>(self) end - <%- if node.newline == false -%> - - def set_newline_flag(newline_marked) # :nodoc: - # Never mark <%= node.name %> with a newline flag, mark children instead - end - <%- elsif node.newline.is_a?(String) -%> - - def set_newline_flag(newline_marked) # :nodoc: - <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> - <%- case field -%> - <%- when Prism::Template::NodeField -%> - <%= field.name %>.set_newline_flag(newline_marked) - <%- when Prism::Template::NodeListField -%> - first = <%= field.name %>.first - first.set_newline_flag(newline_marked) if first - <%- else raise field.class.name -%> - <%- end -%> - end - <%- end -%> # def child_nodes: () -> Array[nil | Node] def child_nodes diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 578e7d2e70..19116b9e6b 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -10,7 +10,7 @@ module Prism # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 27 + MINOR_VERSION = 30 # The patch version of prism that we are expecting to find in the serialized # strings. diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 04b8af858b..e972d6aace 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -91,7 +91,6 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "both block arg and actual block given; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_CONFLICT_AMPERSAND] = { "unexpected `&`; anonymous block parameter is also used within block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_CONFLICT_STAR] = { "unexpected `*`; anonymous rest parameter is also used within block", PM_ERROR_LEVEL_SYNTAX }, @@ -113,7 +112,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_SEPARATOR] = { "unexpected %s; expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -149,7 +148,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_NAME] = { "unexpected %s; expected a method name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "unexpected %s; expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -159,16 +158,16 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hex escape sequence", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; Multiple codepoints at single character literal are disallowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "unterminated Unicode escape", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_SYNTAX }, @@ -177,11 +176,12 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "unexpected %s; expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_IN_DELIMITER] = { "expected a delimiter after the patterns of an `in` clause", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_MESSAGE] = { "unexpected %s; expecting a message to send to the receiver", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_SYNTAX }, @@ -197,6 +197,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EXPRESSION_NOT_WRITABLE_FILE] = { "Can't assign to __FILE__", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_LINE] = { "Can't assign to __LINE__", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_NIL] = { "Can't assign to nil", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED] = { "Can't assign to numbered parameter %.2s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_SELF] = { "Can't change the value of self", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE] = { "Can't assign to true", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_SYNTAX }, @@ -209,9 +210,9 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_VALUE] = { "unexpected %s; expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HEREDOC_IDENTIFIER] = { "unterminated here document identifier", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_HEREDOC_TERM] = { "unterminated heredoc; can't find string \"%.*s\"", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HEREDOC_TERM] = { "unterminated heredoc; can't find string \"%.*s\" anywhere before EOF", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, @@ -219,6 +220,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INSTANCE_VARIABLE_BARE] = { "'@' without identifiers is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_BLOCK_EXIT] = { "Invalid %s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_ESCAPE_CHARACTER] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_LOCAL_VARIABLE_READ] = { "identifier %.*s is not valid to get", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_LOCAL_VARIABLE_WRITE] = { "identifier %.*s is not valid to set", PM_ERROR_LEVEL_SYNTAX }, @@ -229,12 +231,13 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number; numeric literal without digits", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER] = { "invalid underscore placement in number", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING] = { "trailing '_' in number", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_CHARACTER] = { "Invalid char '\\x%02X' in expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_SYNTAX }, // TODO WHAT? + [PM_ERR_INVALID_PERCENT] = { "unknown type of %string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PERCENT_EOF] = { "unterminated quoted string meets end of file", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_AFTER_ELSE] = { "Invalid retry after else", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_AFTER_ENSURE] = { "Invalid retry after ensure", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_WITHOUT_RESCUE] = { "Invalid retry without rescue", PM_ERROR_LEVEL_SYNTAX }, @@ -242,7 +245,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_YIELD] = { "Invalid yield", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when a numbered parameter is already used", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, @@ -265,15 +268,17 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK] = { "numbered parameter is already used in inner block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when 'it' is already used", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK] = { "numbered parameter is already used in outer block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_CIRCULAR] = { "circular argument reference - %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_FORWARDING_AFTER_REST] = { "... after rest argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NAME_DUPLICATED] = { "duplicated argument name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_SYNTAX }, @@ -296,9 +301,11 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_IMPLICIT] = { "unexpected implicit hash in pattern; use '{' to delineate", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY] = { "unexpected %s; expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY_DUPLICATE] = { "duplicated key name", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, // TODO // THIS // AND // ABOVE // IS WEIRD + [PM_ERR_PATTERN_HASH_KEY_INTERPOLATED] = { "symbol literal with interpolation is not allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY_LOCALS] = { "key must be valid as local variables", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, @@ -311,6 +318,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_PARSE_ERROR] = { "%s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_TERM] = { "unterminated regexp meets end of file; expected a closing delimiter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_SYNTAX }, @@ -325,20 +333,20 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_INTERPOLATED_TERM] = { "unterminated string; expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, // TODO expected symbol? prism.c ~9719 [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "unterminated quoted string; expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "unterminated symbol; expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_BLOCK_ARGUMENT] = { "block argument should not be given", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_UNEXPECTED_INDEX_BLOCK] = { "unexpected block arg given in index; blocks are not allowed in index expressions", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_UNEXPECTED_INDEX_KEYWORDS] = { "unexpected keyword arg given in index; keywords are not allowed in index expressions", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_INDEX_BLOCK] = { "unexpected block arg given in index assignment; blocks are not allowed in index assignment expressions", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_INDEX_KEYWORDS] = { "unexpected keyword arg given in index assignment; keywords are not allowed in index assignment expressions", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_SAFE_NAVIGATION] = { "&. inside multiple assignment destination", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_SYNTAX }, @@ -351,6 +359,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_SYNTAX }, // Warnings + [PM_WARN_AMBIGUOUS_BINARY_OPERATOR] = { "'%s' after local variable or literal is interpreted as binary operator even though it seems like %s", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = { "ambiguous first argument; put parentheses or a space even after `-` operator", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS] = { "ambiguous first argument; put parentheses or a space even after `+` operator", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_PREFIX_AMPERSAND] = { "ambiguous `&` has been interpreted as an argument prefix", PM_WARNING_LEVEL_VERBOSE }, @@ -360,7 +369,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_COMPARISON_AFTER_COMPARISON] = { "comparison '%.*s' after comparison", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_DOT_DOT_DOT_EOL] = { "... at EOL, should be parenthesized?", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_DUPLICATED_HASH_KEY] = { "key %.*s is duplicated and overwritten on line %" PRIi32, PM_WARNING_LEVEL_DEFAULT }, - [PM_WARN_DUPLICATED_WHEN_CLAUSE] = { "duplicated 'when' clause with line %" PRIi32 " is ignored", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_DUPLICATED_WHEN_CLAUSE] = { "'when' clause on line %" PRIi32 " duplicates 'when' clause on line %" PRIi32 " and is ignored", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_EQUAL_IN_CONDITIONAL_3_3] = { "found `= literal' in conditional, should be ==", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_EQUAL_IN_CONDITIONAL] = { "found '= literal' in conditional, should be ==", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_END_IN_METHOD] = { "END in method; use at_exit", PM_WARNING_LEVEL_DEFAULT }, @@ -373,6 +382,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_KEYWORD_EOL] = { "`%.*s` at the end of line without an expression", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_LITERAL_IN_CONDITION_DEFAULT] = { "%sliteral in %s", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_LITERAL_IN_CONDITION_VERBOSE] = { "%sliteral in %s", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_SHAREABLE_CONSTANT_VALUE_LINE] = { "'shareable_constant_value' is ignored unless in comment-only line", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_SHEBANG_CARRIAGE_RETURN] = { "shebang line ending with \\r may cause problems", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNEXPECTED_CARRIAGE_RETURN] = { "encountered \\r in middle of line, treated as a mere space", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNREACHABLE_STATEMENT] = { "statement not reached", PM_WARNING_LEVEL_VERBOSE }, diff --git a/prism/templates/src/node.c.erb b/prism/templates/src/node.c.erb index e1c35f5a45..deb2ea49fc 100644 --- a/prism/templates/src/node.c.erb +++ b/prism/templates/src/node.c.erb @@ -1,19 +1,6 @@ #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" #include "prism/node.h" -static void -pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize); - -/** - * Calculate the size of the node list in bytes. - */ -static size_t -pm_node_list_memsize(pm_node_list_t *node_list, pm_memsize_t *memsize) { - pm_node_t *node; - PM_NODE_LIST_FOREACH(node_list, index, node) pm_node_memsize_node(node, memsize); - return sizeof(pm_node_list_t) + (node_list->capacity * sizeof(pm_node_t *)); -} - /** * Attempts to grow the node list to the next size. If there is already * capacity in the list, this function does nothing. Otherwise it reallocates @@ -156,57 +143,6 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { xfree(node); } -static void -pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize) { - memsize->node_count++; - - switch (PM_NODE_TYPE(node)) { - // We do not calculate memsize of a ScopeNode - // as it should never be generated - case PM_SCOPE_NODE: - return; - <%- nodes.each do |node| -%> -#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - case <%= node.type %>: { - pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; - memsize->memsize += sizeof(*cast); - <%- node.fields.each do |field| -%> - <%- case field -%> - <%- when Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::FlagsField, Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::DoubleField -%> - <%- when Prism::Template::NodeField -%> - pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize); - <%- when Prism::Template::OptionalNodeField -%> - if (cast-><%= field.name %> != NULL) { - pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize); - } - <%- when Prism::Template::StringField -%> - memsize->memsize += (pm_string_memsize(&cast-><%= field.name %>) - sizeof(pm_string_t)); - <%- when Prism::Template::NodeListField -%> - memsize->memsize += (pm_node_list_memsize(&cast-><%= field.name %>, memsize) - sizeof(pm_node_list_t)); - <%- when Prism::Template::ConstantListField -%> - memsize->memsize += (pm_constant_id_list_memsize(&cast-><%= field.name %>) - sizeof(pm_constant_id_list_t)); - <%- when Prism::Template::IntegerField -%> - memsize->memsize += (pm_integer_memsize(&cast-><%= field.name %>) - sizeof(pm_integer_t)); - <%- else -%> - <%- raise -%> - <%- end -%> - <%- end -%> - break; - } - <%- end -%> -#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - } -} - -/** - * Calculates the memory footprint of a given node. - */ -PRISM_EXPORTED_FUNCTION void -pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize) { - *memsize = (pm_memsize_t) { .memsize = 0, .node_count = 0 }; - pm_node_memsize_node(node, memsize); -} - /** * Returns a string representation of the given node type. */ diff --git a/prism/templates/src/token_type.c.erb b/prism/templates/src/token_type.c.erb index 0587fc6cdf..f196393ee1 100644 --- a/prism/templates/src/token_type.c.erb +++ b/prism/templates/src/token_type.c.erb @@ -30,7 +30,7 @@ const char * pm_token_type_human(pm_token_type_t token_type) { switch (token_type) { case PM_TOKEN_EOF: - return "end of file"; + return "end-of-input"; case PM_TOKEN_MISSING: return "missing token"; case PM_TOKEN_NOT_PROVIDED: @@ -90,9 +90,9 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_DOT: return "'.'"; case PM_TOKEN_DOT_DOT: - return "'..'"; + return ".."; case PM_TOKEN_DOT_DOT_DOT: - return "'...'"; + return "..."; case PM_TOKEN_EMBDOC_BEGIN: return "'=begin'"; case PM_TOKEN_EMBDOC_END: @@ -352,7 +352,7 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_USTAR: return "*"; case PM_TOKEN_USTAR_STAR: - return "'**'"; + return "**"; case PM_TOKEN_WORDS_SEP: return "string separator"; case PM_TOKEN___END__: diff --git a/prism/util/pm_char.h b/prism/util/pm_char.h index 32f698a42b..deeafd6321 100644 --- a/prism/util/pm_char.h +++ b/prism/util/pm_char.h @@ -34,8 +34,7 @@ size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length); * @return The number of characters at the start of the string that are * whitespace. */ -size_t -pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); +size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); /** * Returns the number of characters at the start of the string that are inline diff --git a/prism/util/pm_constant_pool.c b/prism/util/pm_constant_pool.c index 2a3203f4c4..624002cec9 100644 --- a/prism/util/pm_constant_pool.c +++ b/prism/util/pm_constant_pool.c @@ -62,14 +62,6 @@ pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id) { } /** - * Get the memory size of a list of constant ids. - */ -size_t -pm_constant_id_list_memsize(pm_constant_id_list_t *list) { - return sizeof(pm_constant_id_list_t) + (list->capacity * sizeof(pm_constant_id_t)); -} - -/** * Free the memory associated with a list of constant ids. */ void diff --git a/prism/util/pm_constant_pool.h b/prism/util/pm_constant_pool.h index 0fe16858a0..6df23f8f50 100644 --- a/prism/util/pm_constant_pool.h +++ b/prism/util/pm_constant_pool.h @@ -88,14 +88,6 @@ void pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_co bool pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id); /** - * Get the memory size of a list of constant ids. - * - * @param list The list to get the memory size of. - * @return The memory size of the list. - */ -size_t pm_constant_id_list_memsize(pm_constant_id_list_t *list); - -/** * Free the memory associated with a list of constant ids. * * @param list The list to free. diff --git a/prism/util/pm_integer.c b/prism/util/pm_integer.c index e523bae90b..5b0f34465c 100644 --- a/prism/util/pm_integer.c +++ b/prism/util/pm_integer.c @@ -48,7 +48,7 @@ big_add(pm_integer_t *destination, pm_integer_t *left, pm_integer_t *right, uint /** * Internal use for karatsuba_multiply. Calculates `a - b - c` with the given - * base. Assume a, b, c, a - b - c all to be poitive. + * base. Assume a, b, c, a - b - c all to be positive. * Return pm_integer_t with values allocated. Not normalized. */ static void @@ -471,15 +471,18 @@ pm_integer_parse_big(pm_integer_t *integer, uint32_t multiplier, const uint8_t * * has already been validated, as internal validation checks are not performed * here. */ -PRISM_EXPORTED_FUNCTION void +void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end) { - // Ignore unary +. Unary + is parsed differently and will not end up here. + // Ignore unary +. Unary - is parsed differently and will not end up here. // Instead, it will modify the parsed integer later. if (*start == '+') start++; // Determine the multiplier from the base, and skip past any prefixes. uint32_t multiplier = 10; switch (base) { + case PM_INTEGER_BASE_DEFAULT: + while (*start == '0') start++; // 01 -> 1 + break; case PM_INTEGER_BASE_BINARY: start += 2; // 0b multiplier = 2; @@ -534,14 +537,6 @@ pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *s } /** - * Return the memory size of the integer. - */ -size_t -pm_integer_memsize(const pm_integer_t *integer) { - return sizeof(pm_integer_t) + integer->length * sizeof(uint32_t); -} - -/** * Compare two integers. This function returns -1 if the left integer is less * than the right integer, 0 if they are equal, and 1 if the left integer is * greater than the right integer. @@ -573,6 +568,39 @@ pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right) { } /** + * Reduce a ratio of integers to its simplest form. + */ +void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator) { + // If either the numerator or denominator do not fit into a 32-bit integer, + // then this function is a no-op. In the future, we may consider reducing + // even the larger numbers, but for now we're going to keep it simple. + if ( + // If the numerator doesn't fit into a 32-bit integer, return early. + numerator->length != 0 || + // If the denominator doesn't fit into a 32-bit integer, return early. + denominator->length != 0 || + // If the numerator is 0, then return early. + numerator->value == 0 || + // If the denominator is 1, then return early. + denominator->value == 1 + ) return; + + // Find the greatest common divisor of the numerator and denominator. + uint32_t divisor = numerator->value; + uint32_t remainder = denominator->value; + + while (remainder != 0) { + uint32_t temporary = remainder; + remainder = divisor % remainder; + divisor = temporary; + } + + // Divide the numerator and denominator by the greatest common divisor. + numerator->value /= divisor; + denominator->value /= divisor; +} + +/** * Convert an integer to a decimal string. */ PRISM_EXPORTED_FUNCTION void diff --git a/prism/util/pm_integer.h b/prism/util/pm_integer.h index 7f172988b3..91b28ad2f3 100644 --- a/prism/util/pm_integer.h +++ b/prism/util/pm_integer.h @@ -48,6 +48,9 @@ typedef struct { * from the string itself. */ typedef enum { + /** The default decimal base, with no prefix. Leading 0s will be ignored. */ + PM_INTEGER_BASE_DEFAULT, + /** The binary base, indicated by a 0b or 0B prefix. */ PM_INTEGER_BASE_BINARY, @@ -79,15 +82,7 @@ typedef enum { * @param start The start of the string. * @param end The end of the string. */ -PRISM_EXPORTED_FUNCTION void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end); - -/** - * Return the memory size of the integer. - * - * @param integer The integer to get the memory size of. - * @return The size of the memory associated with the integer. - */ -size_t pm_integer_memsize(const pm_integer_t *integer); +void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end); /** * Compare two integers. This function returns -1 if the left integer is less @@ -101,6 +96,18 @@ size_t pm_integer_memsize(const pm_integer_t *integer); int pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right); /** + * Reduce a ratio of integers to its simplest form. + * + * If either the numerator or denominator do not fit into a 32-bit integer, then + * this function is a no-op. In the future, we may consider reducing even the + * larger numbers, but for now we're going to keep it simple. + * + * @param numerator The numerator of the ratio. + * @param denominator The denominator of the ratio. + */ +void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator); + +/** * Convert an integer to a decimal string. * * @param buffer The buffer to append the string to. diff --git a/prism/util/pm_string.c b/prism/util/pm_string.c index 8342edc34e..dfc121b6a2 100644 --- a/prism/util/pm_string.c +++ b/prism/util/pm_string.c @@ -246,18 +246,6 @@ pm_string_file_init(pm_string_t *string, const char *filepath) { } /** - * Returns the memory size associated with the string. - */ -size_t -pm_string_memsize(const pm_string_t *string) { - size_t size = sizeof(pm_string_t); - if (string->type == PM_STRING_OWNED) { - size += string->length; - } - return size; -} - -/** * Ensure the string is owned. If it is not, then reinitialize it as owned and * copy over the previous source. */ diff --git a/prism/util/pm_string.h b/prism/util/pm_string.h index a68e2a7c91..d23792c0ba 100644 --- a/prism/util/pm_string.h +++ b/prism/util/pm_string.h @@ -121,14 +121,6 @@ PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const ch PRISM_EXPORTED_FUNCTION bool pm_string_file_init(pm_string_t *string, const char *filepath); /** - * Returns the memory size associated with the string. - * - * @param string The string to get the memory size of. - * @return The size of the memory associated with the string. - */ -size_t pm_string_memsize(const pm_string_t *string); - -/** * Ensure the string is owned. If it is not, then reinitialize it as owned and * copy over the previous source. * diff --git a/prism/util/pm_string_list.c b/prism/util/pm_string_list.c deleted file mode 100644 index f6c2145987..0000000000 --- a/prism/util/pm_string_list.c +++ /dev/null @@ -1,28 +0,0 @@ -#include "prism/util/pm_string_list.h" - -/** - * Append a pm_string_t to the given string list. - */ -void -pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) { - if (string_list->length + 1 > string_list->capacity) { - if (string_list->capacity == 0) { - string_list->capacity = 1; - } else { - string_list->capacity *= 2; - } - - string_list->strings = xrealloc(string_list->strings, string_list->capacity * sizeof(pm_string_t)); - if (string_list->strings == NULL) abort(); - } - - string_list->strings[string_list->length++] = *string; -} - -/** - * Free the memory associated with the string list - */ -void -pm_string_list_free(pm_string_list_t *string_list) { - xfree(string_list->strings); -} diff --git a/prism/util/pm_string_list.h b/prism/util/pm_string_list.h deleted file mode 100644 index 0d406cc5d8..0000000000 --- a/prism/util/pm_string_list.h +++ /dev/null @@ -1,44 +0,0 @@ -/** - * @file pm_string_list.h - * - * A list of strings. - */ -#ifndef PRISM_STRING_LIST_H -#define PRISM_STRING_LIST_H - -#include "prism/defines.h" -#include "prism/util/pm_string.h" - -#include <stddef.h> -#include <stdlib.h> - -/** - * A list of strings. - */ -typedef struct { - /** The length of the string list. */ - size_t length; - - /** The capacity of the string list that has been allocated. */ - size_t capacity; - - /** A pointer to the start of the string list. */ - pm_string_t *strings; -} pm_string_list_t; - -/** - * Append a pm_string_t to the given string list. - * - * @param string_list The string list to append to. - * @param string The string to append. - */ -void pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string); - -/** - * Free the memory associated with the string list. - * - * @param string_list The string list to free. - */ -PRISM_EXPORTED_FUNCTION void pm_string_list_free(pm_string_list_t *string_list); - -#endif diff --git a/prism/util/pm_strpbrk.c b/prism/util/pm_strpbrk.c index 6c8dea1836..916a4cc3fd 100644 --- a/prism/util/pm_strpbrk.c +++ b/prism/util/pm_strpbrk.c @@ -9,6 +9,27 @@ pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start } /** + * Set the explicit encoding for the parser to the current encoding. + */ +static inline void +pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) { + if (parser->explicit_encoding != NULL) { + if (parser->explicit_encoding == parser->encoding) { + // Okay, we already locked to this encoding. + } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { + // Not okay, we already found a Unicode escape sequence and this + // conflicts. + pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name); + } else { + // Should not be anything else. + assert(false && "unreachable"); + } + } + + parser->explicit_encoding = parser->encoding; +} + +/** * This is the default path. */ static inline const uint8_t * @@ -52,7 +73,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars * This is the path when the encoding is ASCII-8BIT. */ static inline const uint8_t * -pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) { +pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; while (index < maximum) { @@ -60,6 +81,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi return source + index; } + if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1); index++; } @@ -72,6 +94,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi static inline const uint8_t * pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; + const pm_encoding_t *encoding = parser->encoding; while (index < maximum) { if (strchr((const char *) charset, source[index]) != NULL) { @@ -81,7 +104,8 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t if (source[index] < 0x80) { index++; } else { - size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width); if (width > 0) { index += width; @@ -96,7 +120,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t do { index++; - } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); + } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); } @@ -113,6 +137,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t static inline const uint8_t * pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; + const pm_encoding_t *encoding = parser->encoding; while (index < maximum) { if (strchr((const char *) charset, source[index]) != NULL) { @@ -122,7 +147,8 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t if (source[index] < 0x80 || !validate) { index++; } else { - size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + pm_strpbrk_explicit_encoding_set(parser, source, width); if (width > 0) { index += width; @@ -135,7 +161,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t do { index++; - } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); + } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); } @@ -171,7 +197,7 @@ pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, p } else if (!parser->encoding_changed) { return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate); } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) { - return pm_strpbrk_ascii_8bit(source, charset, (size_t) length); + return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate); } else if (parser->encoding->multibyte) { return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate); } else { diff --git a/prism/version.h b/prism/version.h index 0d25b2883c..d5b8a86018 100644 --- a/prism/version.h +++ b/prism/version.h @@ -14,7 +14,7 @@ /** * The minor version of the Prism library as an int. */ -#define PRISM_VERSION_MINOR 27 +#define PRISM_VERSION_MINOR 30 /** * The patch version of the Prism library as an int. @@ -24,6 +24,6 @@ /** * The version of the Prism library as a constant string. */ -#define PRISM_VERSION "0.27.0" +#define PRISM_VERSION "0.30.0" #endif diff --git a/prism_compile.c b/prism_compile.c index f784e9d0ee..7fefe7c0e4 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -152,12 +152,11 @@ pm_location_line_number(const pm_parser_t *parser, const pm_location_t *location } /** - * Convert the value of an integer node into a Ruby Integer. + * Parse the value of a pm_integer_t into a Ruby Integer. */ static VALUE -parse_integer(const pm_integer_node_t *node) +parse_integer_value(const pm_integer_t *integer) { - const pm_integer_t *integer = &node->value; VALUE result; if (integer->values == NULL) { @@ -188,6 +187,15 @@ parse_integer(const pm_integer_node_t *node) } /** + * Convert the value of an integer node into a Ruby Integer. + */ +static inline VALUE +parse_integer(const pm_integer_node_t *node) +{ + return parse_integer_value(&node->value); +} + +/** * Convert the value of a float node into a Ruby Float. */ static VALUE @@ -205,36 +213,9 @@ parse_float(const pm_float_node_t *node) static VALUE parse_rational(const pm_rational_node_t *node) { - VALUE result; - - if (PM_NODE_TYPE_P(node->numeric, PM_FLOAT_NODE)) { - const uint8_t *start = node->base.location.start; - const uint8_t *end = node->base.location.end - 1; - size_t length = end - start; - - char *buffer = malloc(length + 1); - memcpy(buffer, start, length); - - buffer[length] = '\0'; - - char *decimal = memchr(buffer, '.', length); - RUBY_ASSERT(decimal); - size_t seen_decimal = decimal - buffer; - size_t fraclen = length - seen_decimal - 1; - memmove(decimal, decimal + 1, fraclen + 1); - - VALUE numerator = rb_cstr_to_inum(buffer, 10, false); - result = rb_rational_new(numerator, rb_int_positive_pow(10, fraclen)); - - free(buffer); - } - else { - RUBY_ASSERT(PM_NODE_TYPE_P(node->numeric, PM_INTEGER_NODE)); - VALUE numerator = parse_integer((const pm_integer_node_t *) node->numeric); - result = rb_rational_raw(numerator, INT2FIX(1)); - } - - return result; + VALUE numerator = parse_integer_value(&node->numerator); + VALUE denominator = parse_integer_value(&node->denominator); + return rb_rational_new(numerator, denominator); } /** @@ -301,17 +282,17 @@ parse_static_literal_string(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, { rb_encoding *encoding; - if (node->flags & PM_ENCODING_FLAGS_FORCED_BINARY_ENCODING) { + if (node->flags & PM_STRING_FLAGS_FORCED_BINARY_ENCODING) { encoding = rb_ascii8bit_encoding(); } - else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { + else if (node->flags & PM_STRING_FLAGS_FORCED_UTF8_ENCODING) { encoding = rb_utf8_encoding(); } else { encoding = scope_node->encoding; } - VALUE value = rb_enc_interned_str((const char *) pm_string_source(string), pm_string_length(string), encoding); + VALUE value = rb_enc_literal_str((const char *) pm_string_source(string), pm_string_length(string), encoding); rb_enc_str_coderange(value); if (ISEQ_COMPILE_DATA(iseq)->option->debug_frozen_string_literal || RTEST(ruby_debug)) { @@ -570,6 +551,7 @@ pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const if (parts_size > 0) { VALUE current_string = Qnil; + pm_line_column_t current_location = *node_location; for (size_t index = 0; index < parts_size; index++) { const pm_node_t *part = parts->nodes[index]; @@ -590,6 +572,7 @@ pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const } else { current_string = string_value; + if (index != 0) current_location = PM_NODE_END_LINE_COLUMN(scope_node->parser, part); } } else { @@ -616,6 +599,7 @@ pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const } else { current_string = string_value; + current_location = PM_NODE_START_LINE_COLUMN(scope_node->parser, part); } } else { @@ -640,11 +624,13 @@ pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const current_string = rb_enc_str_new(NULL, 0, encoding); } - PUSH_INSN1(ret, *node_location, putobject, rb_fstring(current_string)); + PUSH_INSN1(ret, current_location, putobject, rb_fstring(current_string)); PM_COMPILE_NOT_POPPED(part); - PUSH_INSN(ret, *node_location, dup); - PUSH_INSN1(ret, *node_location, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); - PUSH_INSN(ret, *node_location, anytostring); + + const pm_line_column_t current_location = PM_NODE_START_LINE_COLUMN(scope_node->parser, part); + PUSH_INSN(ret, current_location, dup); + PUSH_INSN1(ret, current_location, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE, NULL, FALSE)); + PUSH_INSN(ret, current_location, anytostring); current_string = Qnil; stack_size += 2; @@ -656,10 +642,10 @@ pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const current_string = rb_fstring(current_string); if (stack_size == 0 && interpolated) { - PUSH_INSN1(ret, *node_location, putstring, current_string); + PUSH_INSN1(ret, current_location, putstring, current_string); } else { - PUSH_INSN1(ret, *node_location, putobject, current_string); + PUSH_INSN1(ret, current_location, putobject, current_string); } current_string = Qnil; @@ -1001,9 +987,12 @@ pm_compile_conditional(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_ LABEL *else_label = NEW_LABEL(location.line); LABEL *end_label = NULL; - pm_compile_branch_condition(iseq, ret, predicate, then_label, else_label, false, scope_node); + DECL_ANCHOR(cond_seq); + INIT_ANCHOR(cond_seq); + pm_compile_branch_condition(iseq, cond_seq, predicate, then_label, else_label, false, scope_node); + PUSH_SEQ(ret, cond_seq); - rb_code_location_t conditional_location; + rb_code_location_t conditional_location = { 0 }; VALUE branches = Qfalse; if (then_label->refcnt && else_label->refcnt && PM_BRANCH_COVERAGE_P(iseq)) { @@ -1737,7 +1726,7 @@ pm_compile_index_operator_write_node(rb_iseq_t *iseq, const pm_index_operator_wr PUSH_SEND_R(ret, location, idAREF, INT2FIX(argc), NULL, INT2FIX(flag & ~(VM_CALL_ARGS_SPLAT_MUT | VM_CALL_KW_SPLAT_MUT)), keywords); PM_COMPILE_NOT_POPPED(node->value); - ID id_operator = pm_constant_id_lookup(scope_node, node->operator); + ID id_operator = pm_constant_id_lookup(scope_node, node->binary_operator); PUSH_SEND(ret, location, id_operator, INT2FIX(1)); if (!popped) { @@ -2656,7 +2645,7 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t const char *name = rb_id2name(id); if (name && strlen(name) > 0 && name[0] != '_') { - COMPILE_ERROR(ERROR_ARGS "illegal variable in alternative pattern (%"PRIsVALUE")", rb_id2str(id)); + COMPILE_ERROR(iseq, location.line, "illegal variable in alternative pattern (%"PRIsVALUE")", rb_id2str(id)); return COMPILE_NG; } } @@ -2863,6 +2852,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ scope->encoding = previous->encoding; scope->filepath_encoding = previous->filepath_encoding; scope->constants = previous->constants; + scope->coverage_enabled = previous->coverage_enabled; } switch (PM_NODE_TYPE(node)) { @@ -3003,6 +2993,264 @@ pm_compile_retry_end_label(rb_iseq_t *iseq, LINK_ANCHOR *const ret, LABEL *retry } } +static const char * +pm_iseq_builtin_function_name(const pm_scope_node_t *scope_node, const pm_node_t *receiver, ID method_id) +{ + const char *name = rb_id2name(method_id); + static const char prefix[] = "__builtin_"; + const size_t prefix_len = sizeof(prefix) - 1; + + if (receiver == NULL) { + if (UNLIKELY(strncmp(prefix, name, prefix_len) == 0)) { + // __builtin_foo + return &name[prefix_len]; + } + } + else if (PM_NODE_TYPE_P(receiver, PM_CALL_NODE)) { + if (PM_NODE_FLAG_P(receiver, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { + const pm_call_node_t *cast = (const pm_call_node_t *) receiver; + if (pm_constant_id_lookup(scope_node, cast->name) == rb_intern_const("__builtin")) { + // __builtin.foo + return name; + } + } + } + else if (PM_NODE_TYPE_P(receiver, PM_CONSTANT_READ_NODE)) { + const pm_constant_read_node_t *cast = (const pm_constant_read_node_t *) receiver; + if (pm_constant_id_lookup(scope_node, cast->name) == rb_intern_const("Primitive")) { + // Primitive.foo + return name; + } + } + + return NULL; +} + +// Compile Primitive.attr! :leaf, ... +static int +pm_compile_builtin_attr(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_arguments_node_t *arguments, const pm_line_column_t *node_location) +{ + if (arguments == NULL) { + COMPILE_ERROR(iseq, node_location->line, "attr!: no argument"); + return COMPILE_NG; + } + + const pm_node_t *argument; + PM_NODE_LIST_FOREACH(&arguments->arguments, index, argument) { + if (!PM_NODE_TYPE_P(argument, PM_SYMBOL_NODE)) { + COMPILE_ERROR(iseq, node_location->line, "non symbol argument to attr!: %s", pm_node_type_to_str(PM_NODE_TYPE(argument))); + return COMPILE_NG; + } + + VALUE symbol = pm_static_literal_value(iseq, argument, scope_node); + VALUE string = rb_sym_to_s(symbol); + + if (strcmp(RSTRING_PTR(string), "leaf") == 0) { + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_LEAF; + } + else if (strcmp(RSTRING_PTR(string), "inline_block") == 0) { + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_INLINE_BLOCK; + } + else if (strcmp(RSTRING_PTR(string), "use_block") == 0) { + iseq_set_use_block(iseq); + } + else { + COMPILE_ERROR(iseq, node_location->line, "unknown argument to attr!: %s", RSTRING_PTR(string)); + return COMPILE_NG; + } + } + + return COMPILE_OK; +} + +static int +pm_compile_builtin_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const pm_scope_node_t *scope_node, const pm_arguments_node_t *arguments, const pm_line_column_t *node_location, int popped) +{ + if (arguments == NULL) { + COMPILE_ERROR(iseq, node_location->line, "arg!: no argument"); + return COMPILE_NG; + } + + if (arguments->arguments.size != 1) { + COMPILE_ERROR(iseq, node_location->line, "arg!: too many argument"); + return COMPILE_NG; + } + + const pm_node_t *argument = arguments->arguments.nodes[0]; + if (!PM_NODE_TYPE_P(argument, PM_SYMBOL_NODE)) { + COMPILE_ERROR(iseq, node_location->line, "non symbol argument to arg!: %s", pm_node_type_to_str(PM_NODE_TYPE(argument))); + return COMPILE_NG; + } + + if (!popped) { + ID name = parse_string_symbol(scope_node, ((const pm_symbol_node_t *) argument)); + int index = ISEQ_BODY(ISEQ_BODY(iseq)->local_iseq)->local_table_size - get_local_var_idx(iseq, name); + + debugs("id: %s idx: %d\n", rb_id2name(name), index); + PUSH_GETLOCAL(ret, *node_location, index, get_lvar_level(iseq)); + } + + return COMPILE_OK; +} + +static int +pm_compile_builtin_mandatory_only_method(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_call_node_t *call_node, const pm_line_column_t *node_location) +{ + const pm_node_t *ast_node = scope_node->ast_node; + if (!PM_NODE_TYPE_P(ast_node, PM_DEF_NODE)) { + rb_bug("mandatory_only?: not in method definition"); + return COMPILE_NG; + } + + const pm_def_node_t *def_node = (const pm_def_node_t *) ast_node; + const pm_parameters_node_t *parameters_node = def_node->parameters; + if (parameters_node == NULL) { + rb_bug("mandatory_only?: in method definition with no parameters"); + return COMPILE_NG; + } + + const pm_node_t *body_node = def_node->body; + if (body_node == NULL || !PM_NODE_TYPE_P(body_node, PM_STATEMENTS_NODE) || (((const pm_statements_node_t *) body_node)->body.size != 1) || !PM_NODE_TYPE_P(((const pm_statements_node_t *) body_node)->body.nodes[0], PM_IF_NODE)) { + rb_bug("mandatory_only?: not in method definition with plain statements"); + return COMPILE_NG; + } + + const pm_if_node_t *if_node = (const pm_if_node_t *) ((const pm_statements_node_t *) body_node)->body.nodes[0]; + if (if_node->predicate != ((const pm_node_t *) call_node)) { + rb_bug("mandatory_only?: can't find mandatory node"); + return COMPILE_NG; + } + + pm_parameters_node_t parameters = { + .base = parameters_node->base, + .requireds = parameters_node->requireds + }; + + const pm_def_node_t def = { + .base = def_node->base, + .name = def_node->name, + .receiver = def_node->receiver, + .parameters = ¶meters, + .body = (pm_node_t *) if_node->statements, + .locals = { + .ids = def_node->locals.ids, + .size = parameters_node->requireds.size, + .capacity = def_node->locals.capacity + } + }; + + pm_scope_node_t next_scope_node; + pm_scope_node_init(&def.base, &next_scope_node, scope_node); + + ISEQ_BODY(iseq)->mandatory_only_iseq = pm_iseq_new_with_opt( + &next_scope_node, + rb_iseq_base_label(iseq), + rb_iseq_path(iseq), + rb_iseq_realpath(iseq), + node_location->line, + NULL, + 0, + ISEQ_TYPE_METHOD, + ISEQ_COMPILE_DATA(iseq)->option + ); + + pm_scope_node_destroy(&next_scope_node); + return COMPILE_OK; +} + +static int +pm_compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, pm_scope_node_t *scope_node, const pm_call_node_t *call_node, const pm_line_column_t *node_location, int popped, const rb_iseq_t *parent_block, const char *builtin_func) +{ + const pm_arguments_node_t *arguments = call_node->arguments; + + if (parent_block != NULL) { + COMPILE_ERROR(iseq, node_location->line, "should not call builtins here."); + return COMPILE_NG; + } + +#define BUILTIN_INLINE_PREFIX "_bi" + char inline_func[sizeof(BUILTIN_INLINE_PREFIX) + DECIMAL_SIZE_OF(int)]; + bool cconst = false; +retry:; + const struct rb_builtin_function *bf = iseq_builtin_function_lookup(iseq, builtin_func); + + if (bf == NULL) { + if (strcmp("cstmt!", builtin_func) == 0 || strcmp("cexpr!", builtin_func) == 0) { + // ok + } + else if (strcmp("cconst!", builtin_func) == 0) { + cconst = true; + } + else if (strcmp("cinit!", builtin_func) == 0) { + // ignore + return COMPILE_OK; + } + else if (strcmp("attr!", builtin_func) == 0) { + return pm_compile_builtin_attr(iseq, scope_node, arguments, node_location); + } + else if (strcmp("arg!", builtin_func) == 0) { + return pm_compile_builtin_arg(iseq, ret, scope_node, arguments, node_location, popped); + } + else if (strcmp("mandatory_only?", builtin_func) == 0) { + if (popped) { + rb_bug("mandatory_only? should be in if condition"); + } + else if (!LIST_INSN_SIZE_ZERO(ret)) { + rb_bug("mandatory_only? should be put on top"); + } + + PUSH_INSN1(ret, *node_location, putobject, Qfalse); + return pm_compile_builtin_mandatory_only_method(iseq, scope_node, call_node, node_location); + } + else if (1) { + rb_bug("can't find builtin function:%s", builtin_func); + } + else { + COMPILE_ERROR(iseq, node_location->line, "can't find builtin function:%s", builtin_func); + return COMPILE_NG; + } + + int inline_index = node_location->line; + snprintf(inline_func, sizeof(inline_func), BUILTIN_INLINE_PREFIX "%d", inline_index); + builtin_func = inline_func; + arguments = NULL; + goto retry; + } + + if (cconst) { + typedef VALUE(*builtin_func0)(void *, VALUE); + VALUE const_val = (*(builtin_func0)bf->func_ptr)(NULL, Qnil); + PUSH_INSN1(ret, *node_location, putobject, const_val); + return COMPILE_OK; + } + + // fprintf(stderr, "func_name:%s -> %p\n", builtin_func, bf->func_ptr); + + DECL_ANCHOR(args_seq); + INIT_ANCHOR(args_seq); + + int flags = 0; + struct rb_callinfo_kwarg *keywords = NULL; + int argc = pm_setup_args(arguments, call_node->block, &flags, &keywords, iseq, args_seq, scope_node, node_location); + + if (argc != bf->argc) { + COMPILE_ERROR(iseq, node_location->line, "argc is not match for builtin function:%s (expect %d but %d)", builtin_func, bf->argc, argc); + return COMPILE_NG; + } + + unsigned int start_index; + if (delegate_call_p(iseq, argc, args_seq, &start_index)) { + PUSH_INSN2(ret, *node_location, opt_invokebuiltin_delegate, bf, INT2FIX(start_index)); + } + else { + PUSH_SEQ(ret, args_seq); + PUSH_INSN1(ret, *node_location, invokebuiltin, bf); + } + + if (popped) PUSH_INSN(ret, *node_location, pop); + return COMPILE_OK; +} + /** * Compile a call node into the given iseq. */ @@ -3054,6 +3302,7 @@ pm_compile_call(rb_iseq_t *iseq, const pm_call_node_t *call_node, LINK_ANCHOR *c struct rb_callinfo_kwarg *kw_arg = NULL; int orig_argc = pm_setup_args(call_node->arguments, call_node->block, &flags, &kw_arg, iseq, ret, scope_node, &location); + const rb_iseq_t *previous_block = ISEQ_COMPILE_DATA(iseq)->current_block; const rb_iseq_t *block_iseq = NULL; if (call_node->block != NULL && PM_NODE_TYPE_P(call_node->block, PM_BLOCK_NODE)) { @@ -3124,6 +3373,7 @@ pm_compile_call(rb_iseq_t *iseq, const pm_call_node_t *call_node, LINK_ANCHOR *c } if (popped) PUSH_INSN(ret, location, pop); + ISEQ_COMPILE_DATA(iseq)->current_block = previous_block; } static void @@ -3809,7 +4059,7 @@ pm_multi_target_state_update(pm_multi_target_state_t *state) } } -static size_t +static void pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const parents, LINK_ANCHOR *const writes, LINK_ANCHOR *const cleanup, pm_scope_node_t *scope_node, pm_multi_target_state_t *state); /** @@ -3956,6 +4206,13 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons pm_compile_node(iseq, cast->receiver, parents, false, scope_node); + LABEL *safe_label = NULL; + if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { + safe_label = NEW_LABEL(location.line); + PUSH_INSN(parents, location, dup); + PUSH_INSNL(parents, location, branchnil, safe_label); + } + if (state != NULL) { PUSH_INSN1(writes, location, topn, INT2FIX(1)); pm_multi_target_state_push(state, (INSN *) LAST_ELEMENT(writes), 1); @@ -3966,7 +4223,9 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY)) flags |= VM_CALL_FCALL; PUSH_SEND_WITH_FLAG(writes, location, method_id, INT2FIX(1), INT2FIX(flags)); + if (safe_label != NULL && state == NULL) PUSH_LABEL(writes, safe_label); PUSH_INSN(writes, location, pop); + if (safe_label != NULL && state != NULL) PUSH_LABEL(writes, safe_label); if (state != NULL) { PUSH_INSN(cleanup, location, pop); @@ -4040,9 +4299,15 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons // // for i, j in []; end // - if (state != NULL) state->position--; + size_t before_position; + if (state != NULL) { + before_position = state->position; + state->position--; + } + pm_compile_multi_target_node(iseq, node, parents, writes, cleanup, scope_node, state); - if (state != NULL) state->position++; + if (state != NULL) state->position = before_position; + break; } default: @@ -4056,7 +4321,7 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons * on the stack that correspond to the parent expressions of the various * targets. */ -static size_t +static void pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const parents, LINK_ANCHOR *const writes, LINK_ANCHOR *const cleanup, pm_scope_node_t *scope_node, pm_multi_target_state_t *state) { const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); @@ -4096,26 +4361,28 @@ pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR // going through the targets because we will need to revisit them once // we know how many values are being pushed onto the stack. pm_multi_target_state_t target_state = { 0 }; - size_t base_position = state == NULL ? 0 : state->position; - size_t splat_position = has_rest ? 1 : 0; + if (state == NULL) state = &target_state; + + size_t base_position = state->position; + size_t splat_position = (has_rest || has_posts) ? 1 : 0; // Next, we'll iterate through all of the leading targets. for (size_t index = 0; index < lefts->size; index++) { const pm_node_t *target = lefts->nodes[index]; - target_state.position = lefts->size - index + splat_position + base_position; - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + state->position = lefts->size - index + splat_position + base_position; + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } // Next, we'll compile the rest target if there is one. if (has_rest) { const pm_node_t *target = ((const pm_splat_node_t *) rest)->expression; - target_state.position = 1 + rights->size + base_position; + state->position = 1 + rights->size + base_position; if (has_posts) { PUSH_INSN2(writes, location, expandarray, INT2FIX(rights->size), INT2FIX(3)); } - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } // Finally, we'll compile the trailing targets. @@ -4126,18 +4393,10 @@ pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR for (size_t index = 0; index < rights->size; index++) { const pm_node_t *target = rights->nodes[index]; - target_state.position = rights->size - index + base_position; - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + state->position = rights->size - index + base_position; + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } } - - // Now, we need to go back and modify the topn instructions in order to - // ensure they can correctly retrieve the parent expressions. - pm_multi_target_state_update(&target_state); - - if (state != NULL) state->stack_size += target_state.stack_size; - - return target_state.stack_size; } /** @@ -4267,7 +4526,7 @@ pm_compile_rescue(rb_iseq_t *iseq, const pm_begin_node_t *cast, const pm_line_co PM_COMPILE_NOT_POPPED((const pm_node_t *) cast->statements); } else { - PUSH_INSN(ret, *node_location, putnil); + PUSH_SYNTHETIC_PUTNIL(ret, iseq); } ISEQ_COMPILE_DATA(iseq)->in_rescue = prev_in_rescue; @@ -4333,7 +4592,6 @@ pm_compile_ensure(rb_iseq_t *iseq, const pm_begin_node_t *cast, const pm_line_co ); pm_scope_node_destroy(&next_scope_node); - ISEQ_COMPILE_DATA(iseq)->current_block = child_iseq; erange = ISEQ_COMPILE_DATA(iseq)->ensure_node_stack->erange; if (estart->link.next != &eend->link) { @@ -4563,7 +4821,496 @@ pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t * return dispatch; } -/* +/** + * Return the object that will be pushed onto the stack for the given node. + */ +static VALUE +pm_compile_shareable_constant_literal(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_node_t *scope_node) +{ + switch (PM_NODE_TYPE(node)) { + case PM_TRUE_NODE: + case PM_FALSE_NODE: + case PM_NIL_NODE: + case PM_SYMBOL_NODE: + case PM_REGULAR_EXPRESSION_NODE: + case PM_SOURCE_LINE_NODE: + case PM_INTEGER_NODE: + case PM_FLOAT_NODE: + case PM_RATIONAL_NODE: + case PM_IMAGINARY_NODE: + case PM_SOURCE_ENCODING_NODE: + return pm_static_literal_value(iseq, node, scope_node); + case PM_STRING_NODE: + return parse_static_literal_string(iseq, scope_node, node, &((const pm_string_node_t *) node)->unescaped); + case PM_SOURCE_FILE_NODE: + return pm_source_file_value((const pm_source_file_node_t *) node, scope_node); + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + VALUE result = rb_ary_new_capa(cast->elements.size); + + for (size_t index = 0; index < cast->elements.size; index++) { + VALUE element = pm_compile_shareable_constant_literal(iseq, cast->elements.nodes[index], scope_node); + if (element == Qundef) return Qundef; + + rb_ary_push(result, element); + } + + return rb_ractor_make_shareable(result); + } + case PM_HASH_NODE: { + const pm_hash_node_t *cast = (const pm_hash_node_t *) node; + VALUE result = rb_hash_new_capa(cast->elements.size); + + for (size_t index = 0; index < cast->elements.size; index++) { + const pm_node_t *element = cast->elements.nodes[index]; + if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE)) return Qundef; + + const pm_assoc_node_t *assoc = (const pm_assoc_node_t *) element; + + VALUE key = pm_compile_shareable_constant_literal(iseq, assoc->key, scope_node); + if (key == Qundef) return Qundef; + + VALUE value = pm_compile_shareable_constant_literal(iseq, assoc->value, scope_node); + if (value == Qundef) return Qundef; + + rb_hash_aset(result, key, value); + } + + return rb_ractor_make_shareable(result); + } + default: + return Qundef; + } +} + +/** + * Compile the instructions for pushing the value that will be written to a + * shared constant. + */ +static void +pm_compile_shareable_constant_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_flags_t shareability, VALUE path, LINK_ANCHOR *const ret, pm_scope_node_t *scope_node, bool top) +{ + VALUE literal = pm_compile_shareable_constant_literal(iseq, node, scope_node); + if (literal != Qundef) { + const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); + PUSH_INSN1(ret, location, putobject, literal); + return; + } + + const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); + switch (PM_NODE_TYPE(node)) { + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + + if (top) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + } + + for (size_t index = 0; index < cast->elements.size; index++) { + pm_compile_shareable_constant_value(iseq, cast->elements.nodes[index], shareability, path, ret, scope_node, false); + } + + PUSH_INSN1(ret, location, newarray, INT2FIX(cast->elements.size)); + + if (top) { + ID method_id = (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) ? rb_intern("make_shareable_copy") : rb_intern("make_shareable"); + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + return; + } + case PM_HASH_NODE: { + const pm_hash_node_t *cast = (const pm_hash_node_t *) node; + + if (top) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + } + + for (size_t index = 0; index < cast->elements.size; index++) { + const pm_node_t *element = cast->elements.nodes[index]; + + if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE)) { + COMPILE_ERROR(iseq, location.line, "Ractor constant writes do not support **"); + } + + const pm_assoc_node_t *assoc = (const pm_assoc_node_t *) element; + pm_compile_shareable_constant_value(iseq, assoc->key, shareability, path, ret, scope_node, false); + pm_compile_shareable_constant_value(iseq, assoc->value, shareability, path, ret, scope_node, false); + } + + PUSH_INSN1(ret, location, newhash, INT2FIX(cast->elements.size * 2)); + + if (top) { + ID method_id = (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) ? rb_intern("make_shareable_copy") : rb_intern("make_shareable"); + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + return; + } + default: { + DECL_ANCHOR(value_seq); + INIT_ANCHOR(value_seq); + + pm_compile_node(iseq, node, value_seq, false, scope_node); + if (PM_NODE_TYPE_P(node, PM_INTERPOLATED_STRING_NODE)) { + PUSH_SEND_WITH_FLAG(value_seq, location, idUMinus, INT2FIX(0), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + PUSH_INSN1(ret, location, putobject, path); + PUSH_SEND_WITH_FLAG(ret, location, rb_intern("ensure_shareable"), INT2FIX(2), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + else if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) { + if (top) PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + if (top) PUSH_SEND_WITH_FLAG(ret, location, rb_intern("make_shareable_copy"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + else if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING) { + if (top) PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + if (top) PUSH_SEND_WITH_FLAG(ret, location, rb_intern("make_shareable"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + break; + } + } +} + +/** + * Compile a constant write node, either in the context of a ractor pragma or + * not. + */ +static void +pm_compile_constant_write_node(rb_iseq_t *iseq, const pm_constant_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + ID name_id = pm_constant_id_lookup(scope_node, node->name); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, rb_id2str(name_id), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, ID2SYM(name_id)); +} + +/** + * Compile a constant and write node, either in the context of a ractor pragma + * or not. + */ +static void +pm_compile_constant_and_write_node(rb_iseq_t *iseq, const pm_constant_and_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + LABEL *end_label = NEW_LABEL(location.line); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSNL(ret, location, branchunless, end_label); + if (!popped) PUSH_INSN(ret, location, pop); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, end_label); +} + +/** + * Compile a constant or write node, either in the context of a ractor pragma or + * not. + */ +static void +pm_compile_constant_or_write_node(rb_iseq_t *iseq, const pm_constant_or_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + + LABEL *set_label = NEW_LABEL(location.line); + LABEL *end_label = NEW_LABEL(location.line); + + PUSH_INSN(ret, location, putnil); + PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST), name, Qtrue); + PUSH_INSNL(ret, location, branchunless, set_label); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSNL(ret, location, branchif, end_label); + if (!popped) PUSH_INSN(ret, location, pop); + PUSH_LABEL(ret, set_label); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, end_label); +} + +/** + * Compile a constant operator write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_operator_write_node(rb_iseq_t *iseq, const pm_constant_operator_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + ID method_id = pm_constant_id_lookup(scope_node, node->binary_operator); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); +} + +/** + * Creates a string that is used in ractor error messages to describe the + * constant path being written. + */ +static VALUE +pm_constant_path_path(const pm_constant_path_node_t *node, const pm_scope_node_t *scope_node) +{ + VALUE parts = rb_ary_new(); + rb_ary_push(parts, rb_id2str(pm_constant_id_lookup(scope_node, node->name))); + + const pm_node_t *current = node->parent; + while (current != NULL && PM_NODE_TYPE_P(current, PM_CONSTANT_PATH_NODE)) { + const pm_constant_path_node_t *cast = (const pm_constant_path_node_t *) current; + rb_ary_unshift(parts, rb_id2str(pm_constant_id_lookup(scope_node, cast->name))); + current = cast->parent; + } + + if (current == NULL) { + rb_ary_unshift(parts, rb_id2str(idNULL)); + } + else if (PM_NODE_TYPE_P(current, PM_CONSTANT_READ_NODE)) { + rb_ary_unshift(parts, rb_id2str(pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) current)->name))); + } + else { + rb_ary_unshift(parts, rb_str_new_cstr("...")); + } + + return rb_ary_join(parts, rb_str_new_cstr("::")); +} + +/** + * Compile a constant path write node, either in the context of a ractor pragma + * or not. + */ +static void +pm_compile_constant_path_write_node(rb_iseq_t *iseq, const pm_constant_path_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + + if (target->parent) { + PM_COMPILE_NOT_POPPED((const pm_node_t *) target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) { + PUSH_INSN(ret, location, swap); + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + + PUSH_INSN(ret, location, swap); + PUSH_INSN1(ret, location, setconstant, name); +} + +/** + * Compile a constant path and write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_path_and_write_node(rb_iseq_t *iseq, const pm_constant_path_and_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + LABEL *lfin = NEW_LABEL(location.line); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSNL(ret, location, branchunless, lfin); + + if (!popped) PUSH_INSN(ret, location, pop); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + else { + PUSH_INSN1(ret, location, dupn, INT2FIX(2)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, lfin); + + if (!popped) PUSH_INSN(ret, location, swap); + PUSH_INSN(ret, location, pop); +} + +/** + * Compile a constant path or write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_path_or_write_node(rb_iseq_t *iseq, const pm_constant_path_or_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + LABEL *lassign = NEW_LABEL(location.line); + LABEL *lfin = NEW_LABEL(location.line); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST_FROM), name, Qtrue); + PUSH_INSNL(ret, location, branchunless, lassign); + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSNL(ret, location, branchif, lfin); + + if (!popped) PUSH_INSN(ret, location, pop); + PUSH_LABEL(ret, lassign); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + else { + PUSH_INSN1(ret, location, dupn, INT2FIX(2)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, lfin); + + if (!popped) PUSH_INSN(ret, location, swap); + PUSH_INSN(ret, location, pop); +} + +/** + * Compile a constant path operator write node, either in the context of a + * ractor pragma or not. + */ +static void +pm_compile_constant_path_operator_write_node(rb_iseq_t *iseq, const pm_constant_path_operator_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + ID method_id = pm_constant_id_lookup(scope_node, node->binary_operator); + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + PUSH_CALL(ret, location, method_id, INT2FIX(1)); + PUSH_INSN(ret, location, swap); + + if (!popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); +} + +/** * Compiles a prism node into instruction sequences. * * iseq - The current instruction sequence object (used for locals) @@ -4580,8 +5327,24 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(parser, node); int lineno = (int) location.line; - if (!PM_NODE_TYPE_P(node, PM_RETURN_NODE) || !PM_NODE_FLAG_P(node, PM_RETURN_NODE_FLAGS_REDUNDANT) || ((const pm_return_node_t *) node)->arguments != NULL) { + if (PM_NODE_TYPE_P(node, PM_RETURN_NODE) && PM_NODE_FLAG_P(node, PM_RETURN_NODE_FLAGS_REDUNDANT) && ((const pm_return_node_t *) node)->arguments == NULL) { + // If the node that we're compiling is a return node that is redundant, + // then it cannot be considered a line node because the other parser + // eliminates it from the parse tree. In this case we must replicate + // this behavior. + } else { + if (PM_NODE_TYPE_P(node, PM_BEGIN_NODE) && (((const pm_begin_node_t *) node)->statements == NULL) && (((const pm_begin_node_t *) node)->rescue_clause != NULL)) { + // If this node is a begin node and it has empty statements and also + // has a rescue clause, then the other parser considers it as + // starting on the same line as the rescue, as opposed to the + // location of the begin keyword. We replicate that behavior here. + lineno = (int) PM_NODE_START_LINE_COLUMN(parser, ((const pm_begin_node_t *) node)->rescue_clause).line; + } + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_NEWLINE) && ISEQ_COMPILE_DATA(iseq)->last_line != lineno) { + // If this node has the newline flag set and it is on a new line + // from the previous nodes that have been compiled for this ISEQ, + // then we need to emit a newline event. int event = RUBY_EVENT_LINE; ISEQ_COMPILE_DATA(iseq)->last_line = lineno; @@ -4831,7 +5594,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE((const pm_node_t *) cast->statements); } else if (!popped) { - PUSH_INSN(ret, location, putnil); + PUSH_SYNTHETIC_PUTNIL(ret, iseq); } } return; @@ -4894,7 +5657,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, throw_flag = 0; } else if (ISEQ_BODY(ip)->type == ISEQ_TYPE_EVAL) { - COMPILE_ERROR(ERROR_ARGS "Can't escape from eval with break"); + COMPILE_ERROR(iseq, location.line, "Can't escape from eval with break"); return; } else { @@ -4916,8 +5679,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } - COMPILE_ERROR(ERROR_ARGS "Invalid break"); - rb_bug("Invalid break"); + COMPILE_ERROR(iseq, location.line, "Invalid break"); } return; } @@ -4931,13 +5693,21 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // foo.bar() {} // ^^^^^^^^^^^^ const pm_call_node_t *cast = (const pm_call_node_t *) node; - LABEL *start = NEW_LABEL(location.line); + ID method_id = pm_constant_id_lookup(scope_node, cast->name); - if (cast->block) { - PUSH_LABEL(ret, start); + const pm_location_t *message_loc = &cast->message_loc; + if (message_loc->start == NULL) message_loc = &cast->base.location; + + const pm_line_column_t location = PM_LOCATION_START_LINE_COLUMN(scope_node->parser, message_loc); + const char *builtin_func; + + if (UNLIKELY(iseq_has_builtin_function_table(iseq)) && (builtin_func = pm_iseq_builtin_function_name(scope_node, cast->receiver, method_id)) != NULL) { + pm_compile_builtin_function_call(iseq, ret, scope_node, cast, &location, popped, ISEQ_COMPILE_DATA(iseq)->current_block, builtin_func); + return; } - ID method_id = pm_constant_id_lookup(scope_node, cast->name); + LABEL *start = NEW_LABEL(location.line); + if (cast->block) PUSH_LABEL(ret, start); switch (method_id) { case idUMinus: { @@ -5053,7 +5823,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_SEND_WITH_FLAG(ret, location, id_read_name, INT2FIX(0), INT2FIX(flag)); PM_COMPILE_NOT_POPPED(cast->value); - ID id_operator = pm_constant_id_lookup(scope_node, cast->operator); + ID id_operator = pm_constant_id_lookup(scope_node, cast->binary_operator); PUSH_SEND(ret, location, id_operator, INT2FIX(1)); if (!popped) { @@ -5127,7 +5897,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_compile_node(iseq, (const pm_node_t *) clause->statements, body_seq, popped, scope_node); } else if (!popped) { - PUSH_INSN(body_seq, location, putnil); + PUSH_SYNTHETIC_PUTNIL(body_seq, iseq); } PUSH_INSNL(body_seq, location, jump, end_label); @@ -5276,7 +6046,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_compile_node(iseq, (const pm_node_t *) clause->statements, body_seq, popped, scope_node); } else if (!popped) { - PUSH_INSN(body_seq, clause_location, putnil); + PUSH_SYNTHETIC_PUTNIL(body_seq, iseq); } PUSH_INSNL(body_seq, clause_location, jump, end_label); @@ -5367,7 +6137,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // We're going to use this to uniquely identify each branch so that we // can track coverage information. - rb_code_location_t case_location; + rb_code_location_t case_location = { 0 }; VALUE branches = Qfalse; int branch_id = 0; @@ -5424,7 +6194,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE_INTO_ANCHOR(body_seq, (const pm_node_t *) in_node->statements); } else if (!popped) { - PUSH_INSN(body_seq, in_location, putnil); + PUSH_SYNTHETIC_PUTNIL(body_seq, iseq); } PUSH_INSNL(body_seq, in_location, jump, end_label); @@ -5557,7 +6327,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, getclassvariable, name, get_cvar_ic_value(iseq, name_id)); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -5651,147 +6421,28 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Foo::Bar &&= baz // ^^^^^^^^^^^^^^^^ const pm_constant_path_and_write_node_t *cast = (const pm_constant_path_and_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); - LABEL *lfin = NEW_LABEL(location.line); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - if (!popped) PUSH_INSN(ret, location, dup); - PUSH_INSNL(ret, location, branchunless, lfin); - - if (!popped) PUSH_INSN(ret, location, pop); - PM_COMPILE_NOT_POPPED(cast->value); - - if (popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - else { - PUSH_INSN1(ret, location, dupn, INT2FIX(2)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, lfin); - - if (!popped) PUSH_INSN(ret, location, swap); - PUSH_INSN(ret, location, pop); - + pm_compile_constant_path_and_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_OR_WRITE_NODE: { // Foo::Bar ||= baz // ^^^^^^^^^^^^^^^^ const pm_constant_path_or_write_node_t *cast = (const pm_constant_path_or_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); - LABEL *lassign = NEW_LABEL(location.line); - LABEL *lfin = NEW_LABEL(location.line); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST_FROM), name, Qtrue); - PUSH_INSNL(ret, location, branchunless, lassign); - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - if (!popped) PUSH_INSN(ret, location, dup); - PUSH_INSNL(ret, location, branchif, lfin); - - if (!popped) PUSH_INSN(ret, location, pop); - PUSH_LABEL(ret, lassign); - PM_COMPILE_NOT_POPPED(cast->value); - - if (popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - else { - PUSH_INSN1(ret, location, dupn, INT2FIX(2)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, lfin); - - if (!popped) PUSH_INSN(ret, location, swap); - PUSH_INSN(ret, location, pop); - + pm_compile_constant_path_or_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_OPERATOR_WRITE_NODE: { // Foo::Bar += baz // ^^^^^^^^^^^^^^^ const pm_constant_path_operator_write_node_t *cast = (const pm_constant_path_operator_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - PM_COMPILE_NOT_POPPED(cast->value); - PUSH_CALL(ret, location, method_id, INT2FIX(1)); - PUSH_INSN(ret, location, swap); - - if (!popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); + pm_compile_constant_path_operator_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_WRITE_NODE: { // Foo::Bar = 1 // ^^^^^^^^^^^^ const pm_constant_path_write_node_t *cast = (const pm_constant_path_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); - - if (target->parent) { - PM_COMPILE_NOT_POPPED((const pm_node_t *) target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PM_COMPILE_NOT_POPPED(cast->value); - - if (!popped) { - PUSH_INSN(ret, location, swap); - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - - PUSH_INSN(ret, location, swap); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_path_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_READ_NODE: { @@ -5809,82 +6460,28 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Foo &&= bar // ^^^^^^^^^^^ const pm_constant_and_write_node_t *cast = (const pm_constant_and_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - LABEL *end_label = NEW_LABEL(location.line); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSNL(ret, location, branchunless, end_label); - if (!popped) PUSH_INSN(ret, location, pop); - - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, end_label); - + pm_compile_constant_and_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_OR_WRITE_NODE: { // Foo ||= bar // ^^^^^^^^^^^ const pm_constant_or_write_node_t *cast = (const pm_constant_or_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - LABEL *set_label = NEW_LABEL(location.line); - LABEL *end_label = NEW_LABEL(location.line); - - PUSH_INSN(ret, location, putnil); - PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST), name, Qtrue); - PUSH_INSNL(ret, location, branchunless, set_label); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSNL(ret, location, branchif, end_label); - if (!popped) PUSH_INSN(ret, location, pop); - - PUSH_LABEL(ret, set_label); - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, end_label); - + pm_compile_constant_or_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_OPERATOR_WRITE_NODE: { // Foo += bar // ^^^^^^^^^^ const pm_constant_operator_write_node_t *cast = (const pm_constant_operator_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - PM_COMPILE_NOT_POPPED(cast->value); - - PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_operator_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_WRITE_NODE: { // Foo = 1 // ^^^^^^^ const pm_constant_write_node_t *cast = (const pm_constant_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_DEF_NODE: { @@ -5933,7 +6530,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE((const pm_node_t *) (cast->statements)); } else { - PUSH_INSN(ret, location, putnil); + PUSH_SYNTHETIC_PUTNIL(ret, iseq); } if (popped) PUSH_INSN(ret, location, pop); @@ -6221,7 +6818,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN1(ret, location, getglobal, name); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -6419,7 +7016,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, getinstancevariable, name, get_ivar_ic_value(iseq, name_id)); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -6604,6 +7201,15 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + // -> { it } + // ^^ + if (!popped) { + PUSH_GETLOCAL(ret, location, scope_node->local_table_for_iseq_size, 0); + } + + return; + } case PM_KEYWORD_HASH_NODE: { // foo(bar: baz) // ^^^^^^^^ @@ -6669,7 +7275,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); if (!popped) PUSH_INSN(ret, location, dup); @@ -6707,9 +7313,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_LOCAL_VARIABLE_READ_NODE: { // foo // ^^^ - const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) node; - if (!popped) { + const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) node; pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, cast->name, cast->depth); PUSH_GETLOCAL(ret, location, index.index, index.level); } @@ -6970,18 +7575,22 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_multi_target_state_t state = { 0 }; state.position = popped ? 0 : 1; - size_t stack_size = pm_compile_multi_target_node(iseq, node, ret, writes, cleanup, scope_node, &state); + pm_compile_multi_target_node(iseq, node, ret, writes, cleanup, scope_node, &state); PM_COMPILE_NOT_POPPED(cast->value); if (!popped) PUSH_INSN(ret, location, dup); PUSH_SEQ(ret, writes); - if (!popped && stack_size >= 1) { + if (!popped && state.stack_size >= 1) { // Make sure the value on the right-hand side of the = operator is // being returned before we pop the parent expressions. - PUSH_INSN1(ret, location, setn, INT2FIX(stack_size)); + PUSH_INSN1(ret, location, setn, INT2FIX(state.stack_size)); } + // Now, we need to go back and modify the topn instructions in order to + // ensure they can correctly retrieve the parent expressions. + pm_multi_target_state_update(&state); + PUSH_SEQ(ret, cleanup); return; } @@ -7050,7 +7659,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } else if (ISEQ_BODY(ip)->type == ISEQ_TYPE_EVAL) { - COMPILE_ERROR(ERROR_ARGS "Can't escape from eval with next"); + COMPILE_ERROR(iseq, location.line, "Can't escape from eval with next"); return; } @@ -7068,7 +7677,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (popped) PUSH_INSN(ret, location, pop); } else { - COMPILE_ERROR(ERROR_ARGS "Invalid next"); + COMPILE_ERROR(iseq, location.line, "Invalid next"); return; } } @@ -7303,7 +7912,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } else if (ISEQ_BODY(ip)->type == ISEQ_TYPE_EVAL) { - COMPILE_ERROR(ERROR_ARGS "Can't escape from eval with redo"); + COMPILE_ERROR(iseq, location.line, "Can't escape from eval with redo"); return; } @@ -7316,7 +7925,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (popped) PUSH_INSN(ret, location, pop); } else { - COMPILE_ERROR(ERROR_ARGS "Invalid redo"); + COMPILE_ERROR(iseq, location.line, "Invalid redo"); return; } } @@ -7544,7 +8153,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (popped) PUSH_INSN(ret, location, pop); } else { - COMPILE_ERROR(ERROR_ARGS "Invalid retry"); + COMPILE_ERROR(iseq, location.line, "Invalid retry"); return; } return; @@ -7651,6 +8260,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } } + // If we have the `it` implicit local variable, we need to account for + // it in the local table size. + if (scope_node->parameters != NULL && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { + table_size++; + } + // Ensure there is enough room in the local table for any // parameters that have been repeated // ex: def underscore_parameters(_, _ = 1, _ = 2); _; end @@ -7799,6 +8414,11 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, body->param.flags.has_lead = true; } + if (scope_node->parameters != NULL && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { + ID local = rb_make_temporary_id(local_index); + local_table_for_iseq->ids[local_index++] = local; + } + // def foo(a, (b, *c, d), e = 1, *f, g, (h, *i, j), k:, l: 1, **m, &n) // ^^^^^ if (optionals_list && optionals_list->size) { @@ -8159,18 +8779,6 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, body->param.flags.has_lead = true; } - // Fill in the it variable, if it exists - if (scope_node->parameters && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { - const uint8_t param_name[] = { '0', 'i', 't' }; - pm_constant_id_t constant_id = pm_constant_pool_find(&parser->constant_pool, param_name, 3); - RUBY_ASSERT(constant_id && "parser should have inserted 0it for 'it' local"); - - ID local = rb_make_temporary_id(local_index); - local_table_for_iseq->ids[local_index] = local; - st_insert(index_lookup_table, (st_data_t) constant_id, (st_data_t) local_index); - local_index++; - } - //********END OF STEP 3********** //********STEP 4********** @@ -8383,7 +8991,9 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } case ISEQ_TYPE_METHOD: { + ISEQ_COMPILE_DATA(iseq)->root_node = (const void *) scope_node->body; PUSH_TRACE(ret, RUBY_EVENT_CALL); + if (scope_node->body) { PM_COMPILE((const pm_node_t *) scope_node->body); } @@ -8391,9 +9001,10 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN(ret, location, putnil); } + ISEQ_COMPILE_DATA(iseq)->root_node = (const void *) scope_node->body; PUSH_TRACE(ret, RUBY_EVENT_RETURN); - ISEQ_COMPILE_DATA(iseq)->last_line = body->location.code_location.end_pos.lineno; + ISEQ_COMPILE_DATA(iseq)->last_line = body->location.code_location.end_pos.lineno; break; } case ISEQ_TYPE_RESCUE: { @@ -8429,7 +9040,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } - if (PM_NODE_TYPE_P(scope_node->ast_node, PM_CLASS_NODE)) { + if (PM_NODE_TYPE_P(scope_node->ast_node, PM_CLASS_NODE) || PM_NODE_TYPE_P(scope_node->ast_node, PM_MODULE_NODE)) { const pm_line_column_t end_location = PM_NODE_END_LINE_COLUMN(scope_node->parser, scope_node->ast_node); ADD_TRACE(ret, RUBY_EVENT_END); ISEQ_COMPILE_DATA(iseq)->last_line = end_location.line; @@ -8453,7 +9064,38 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_SHAREABLE_CONSTANT_NODE: { // A value that is being written to a constant that is being marked as // shared depending on the current lexical context. - PM_COMPILE(((const pm_shareable_constant_node_t *) node)->write); + const pm_shareable_constant_node_t *cast = (const pm_shareable_constant_node_t *) node; + + switch (PM_NODE_TYPE(cast->write)) { + case PM_CONSTANT_WRITE_NODE: + pm_compile_constant_write_node(iseq, (const pm_constant_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_AND_WRITE_NODE: + pm_compile_constant_and_write_node(iseq, (const pm_constant_and_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_OR_WRITE_NODE: + pm_compile_constant_or_write_node(iseq, (const pm_constant_or_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_OPERATOR_WRITE_NODE: + pm_compile_constant_operator_write_node(iseq, (const pm_constant_operator_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_WRITE_NODE: + pm_compile_constant_path_write_node(iseq, (const pm_constant_path_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_AND_WRITE_NODE: + pm_compile_constant_path_and_write_node(iseq, (const pm_constant_path_and_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_OR_WRITE_NODE: + pm_compile_constant_path_or_write_node(iseq, (const pm_constant_path_or_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_OPERATOR_WRITE_NODE: + pm_compile_constant_path_operator_write_node(iseq, (const pm_constant_path_operator_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + default: + rb_bug("Unexpected node type for shareable constant write: %s", pm_node_type_to_str(PM_NODE_TYPE(cast->write))); + break; + } + return; } case PM_SINGLETON_CLASS_NODE: { @@ -8706,7 +9348,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: case ISEQ_TYPE_CLASS: - COMPILE_ERROR(ERROR_ARGS "Invalid yield"); + COMPILE_ERROR(iseq, location.line, "Invalid yield"); return; default: /* valid */; } @@ -8812,6 +9454,368 @@ pm_parse_result_free(pm_parse_result_t *result) pm_options_free(&result->options); } +/** An error that is going to be formatted into the output. */ +typedef struct { + /** A pointer to the diagnostic that was generated during parsing. */ + pm_diagnostic_t *error; + + /** The start line of the diagnostic message. */ + int32_t line; + + /** The column start of the diagnostic message. */ + uint32_t column_start; + + /** The column end of the diagnostic message. */ + uint32_t column_end; +} pm_parse_error_t; + +/** The format that will be used to format the errors into the output. */ +typedef struct { + /** The prefix that will be used for line numbers. */ + const char *number_prefix; + + /** The prefix that will be used for blank lines. */ + const char *blank_prefix; + + /** The divider that will be used between sections of source code. */ + const char *divider; + + /** The length of the blank prefix. */ + size_t blank_prefix_length; + + /** The length of the divider. */ + size_t divider_length; +} pm_parse_error_format_t; + +#define PM_COLOR_GRAY "\033[38;5;102m" +#define PM_COLOR_RED "\033[1;31m" +#define PM_COLOR_RESET "\033[m" +#define PM_ERROR_TRUNCATE 30 + +static inline pm_parse_error_t * +pm_parse_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { + pm_parse_error_t *errors = xcalloc(error_list->size, sizeof(pm_parse_error_t)); + if (errors == NULL) return NULL; + + int32_t start_line = parser->start_line; + for (pm_diagnostic_t *error = (pm_diagnostic_t *) error_list->head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { + pm_line_column_t start = pm_newline_list_line_column(newline_list, error->location.start, start_line); + pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.end, start_line); + + // We're going to insert this error into the array in sorted order. We + // do this by finding the first error that has a line number greater + // than the current error and then inserting the current error before + // that one. + size_t index = 0; + while ( + (index < error_list->size) && + (errors[index].error != NULL) && + ( + (errors[index].line < start.line) || + ((errors[index].line == start.line) && (errors[index].column_start < start.column)) + ) + ) index++; + + // Now we're going to shift all of the errors after this one down one + // index to make room for the new error. + if (index + 1 < error_list->size) { + memmove(&errors[index + 1], &errors[index], sizeof(pm_parse_error_t) * (error_list->size - index - 1)); + } + + // Finally, we'll insert the error into the array. + uint32_t column_end; + if (start.line == end.line) { + column_end = end.column; + } else { + column_end = (uint32_t) (newline_list->offsets[start.line - start_line + 1] - newline_list->offsets[start.line - start_line] - 1); + } + + // Ensure we have at least one column of error. + if (start.column == column_end) column_end++; + + errors[index] = (pm_parse_error_t) { + .error = error, + .line = start.line, + .column_start = start.column, + .column_end = column_end + }; + } + + return errors; +} + +static inline void +pm_parse_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, uint32_t column_start, uint32_t column_end, pm_buffer_t *buffer) { + int32_t line_delta = line - parser->start_line; + assert(line_delta >= 0); + + size_t index = (size_t) line_delta; + assert(index < newline_list->size); + + const uint8_t *start = &parser->start[newline_list->offsets[index]]; + const uint8_t *end; + + if (index >= newline_list->size - 1) { + end = parser->end; + } else { + end = &parser->start[newline_list->offsets[index + 1]]; + } + + pm_buffer_append_format(buffer, number_prefix, line); + + // Here we determine if we should truncate the end of the line. + bool truncate_end = false; + if ((column_end != 0) && ((end - (start + column_end)) >= PM_ERROR_TRUNCATE)) { + end = start + column_end + PM_ERROR_TRUNCATE; + truncate_end = true; + } + + // Here we determine if we should truncate the start of the line. + if (column_start >= PM_ERROR_TRUNCATE) { + pm_buffer_append_string(buffer, "... ", 4); + start += column_start; + } + + pm_buffer_append_string(buffer, (const char *) start, (size_t) (end - start)); + + if (truncate_end) { + pm_buffer_append_string(buffer, " ...\n", 5); + } else if (end == parser->end && end[-1] != '\n') { + pm_buffer_append_string(buffer, "\n", 1); + } +} + +/** + * Format the errors on the parser into the given buffer. + */ +static void +pm_parse_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { + assert(error_list->size != 0); + + // First, we're going to sort all of the errors by line number using an + // insertion sort into a newly allocated array. + const int32_t start_line = parser->start_line; + const pm_newline_list_t *newline_list = &parser->newline_list; + + pm_parse_error_t *errors = pm_parse_errors_format_sort(parser, error_list, newline_list); + if (errors == NULL) return; + + // Now we're going to determine how we're going to format line numbers and + // blank lines based on the maximum number of digits in the line numbers + // that are going to be displaid. + pm_parse_error_format_t error_format; + int32_t first_line_number = errors[0].line; + int32_t last_line_number = errors[error_list->size - 1].line; + + // If we have a maximum line number that is negative, then we're going to + // use the absolute value for comparison but multiple by 10 to additionally + // have a column for the negative sign. + if (first_line_number < 0) first_line_number = (-first_line_number) * 10; + if (last_line_number < 0) last_line_number = (-last_line_number) * 10; + int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; + + if (max_line_number < 10) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%1" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%1" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~\n" + }; + } + } else if (max_line_number < 100) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%2" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%2" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~\n" + }; + } + } else if (max_line_number < 1000) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%3" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%3" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~\n" + }; + } + } else if (max_line_number < 10000) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%4" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%4" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~~\n" + }; + } + } else { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%5" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%5" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~~\n" + }; + } + } + + error_format.blank_prefix_length = strlen(error_format.blank_prefix); + error_format.divider_length = strlen(error_format.divider); + + // Now we're going to iterate through every error in our error list and + // display it. While we're iterating, we will display some padding lines of + // the source before the error to give some context. We'll be careful not to + // display the same line twice in case the errors are close enough in the + // source. + int32_t last_line = parser->start_line - 1; + uint32_t last_column_start = 0; + const pm_encoding_t *encoding = parser->encoding; + + for (size_t index = 0; index < error_list->size; index++) { + pm_parse_error_t *error = &errors[index]; + + // Here we determine how many lines of padding of the source to display, + // based on the difference from the last line that was displaid. + if (error->line - last_line > 1) { + if (error->line - last_line > 2) { + if ((index != 0) && (error->line - last_line > 3)) { + pm_buffer_append_string(buffer, error_format.divider, error_format.divider_length); + } + + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 2, 0, 0, buffer); + } + + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 1, 0, 0, buffer); + } + + // If this is the first error or we're on a new line, then we'll display + // the line that has the error in it. + if ((index == 0) || (error->line != last_line)) { + if (colorize) { + pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); + } else { + pm_buffer_append_string(buffer, "> ", 2); + } + + last_column_start = error->column_start; + + // Find the maximum column end of all the errors on this line. + uint32_t column_end = error->column_end; + for (size_t next_index = index + 1; next_index < error_list->size; next_index++) { + if (errors[next_index].line != error->line) break; + if (errors[next_index].column_end > column_end) column_end = errors[next_index].column_end; + } + + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, error->column_start, column_end, buffer); + } + + const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; + if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); + + // Now we'll display the actual error message. We'll do this by first + // putting the prefix to the line, then a bunch of blank spaces + // depending on the column, then as many carets as we need to display + // the width of the error, then the error message itself. + // + // Note that this doesn't take into account the width of the actual + // character when displaid in the terminal. For some east-asian + // languages or emoji, this means it can be thrown off pretty badly. We + // will need to solve this eventually. + pm_buffer_append_string(buffer, " ", 2); + pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); + + size_t column = 0; + if (last_column_start >= PM_ERROR_TRUNCATE) { + pm_buffer_append_string(buffer, " ", 4); + column = last_column_start; + } + + while (column < error->column_start) { + pm_buffer_append_byte(buffer, ' '); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + } + + if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RED, 7); + pm_buffer_append_byte(buffer, '^'); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + + while (column < error->column_end) { + pm_buffer_append_byte(buffer, '~'); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + } + + if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RESET, 3); + + if (inline_messages) { + pm_buffer_append_byte(buffer, ' '); + assert(error->error != NULL); + + const char *message = error->error->message; + pm_buffer_append_string(buffer, message, strlen(message)); + } + + pm_buffer_append_byte(buffer, '\n'); + + // Here we determine how many lines of padding to display after the + // error, depending on where the next error is in source. + last_line = error->line; + int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; + + if (next_line - last_line > 1) { + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, 0, 0, buffer); + } + + if (next_line - last_line > 1) { + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, 0, 0, buffer); + } + } + + // Finally, we'll free the array of errors that we allocated. + xfree(errors); +} + +#undef PM_ERROR_TRUNCATE +#undef PM_COLOR_GRAY +#undef PM_COLOR_RED +#undef PM_COLOR_RESET + /** * Check if the given source slice is valid UTF-8. The location represents the * location of the error, but the slice of the source will include the content @@ -8883,7 +9887,7 @@ pm_parse_process_error(const pm_parse_result_t *result) pm_list_node_t *list_node = (pm_list_node_t *) error; pm_list_t error_list = { .size = 1, .head = list_node, .tail = list_node }; - pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); + pm_parse_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); } VALUE value = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); @@ -8913,7 +9917,7 @@ pm_parse_process_error(const pm_parse_result_t *result) ); if (valid_utf8) { - pm_parser_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); + pm_parse_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); } else { for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { @@ -8922,7 +9926,8 @@ pm_parse_process_error(const pm_parse_result_t *result) } } - VALUE error = rb_exc_new(rb_eSyntaxError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + VALUE message = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), result->node.encoding); + VALUE error = rb_exc_new_str(rb_eSyntaxError, message); rb_encoding *filepath_encoding = result->node.filepath_encoding != NULL ? result->node.filepath_encoding : rb_utf8_encoding(); VALUE path = rb_enc_str_new((const char *) pm_string_source(filepath), pm_string_length(filepath), filepath_encoding); @@ -8947,10 +9952,16 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // freed regardless of whether or we return an error. pm_scope_node_t *scope_node = &result->node; rb_encoding *filepath_encoding = scope_node->filepath_encoding; + int coverage_enabled = scope_node->coverage_enabled; pm_scope_node_init(node, scope_node, NULL); scope_node->filepath_encoding = filepath_encoding; + scope_node->encoding = rb_enc_find(parser->encoding->name); + if (!scope_node->encoding) rb_bug("Encoding not found %s!", parser->encoding->name); + + scope_node->coverage_enabled = coverage_enabled; + // Emit all of the various warnings from the parse. const pm_diagnostic_t *warning; const char *warning_filepath = (const char *) pm_string_source(&parser->filepath); @@ -8959,10 +9970,10 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) int line = pm_location_line_number(parser, &warning->location); if (warning->level == PM_WARNING_LEVEL_VERBOSE) { - rb_compile_warning(warning_filepath, line, "%s", warning->message); + rb_enc_compile_warning(scope_node->encoding, warning_filepath, line, "%s", warning->message); } else { - rb_compile_warn(warning_filepath, line, "%s", warning->message); + rb_enc_compile_warn(scope_node->encoding, warning_filepath, line, "%s", warning->message); } } @@ -8977,9 +9988,6 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // Now set up the constant pool and intern all of the various constants into // their corresponding IDs. - scope_node->encoding = rb_enc_find(parser->encoding->name); - if (!scope_node->encoding) rb_bug("Encoding not found %s!", parser->encoding->name); - scope_node->parser = parser; scope_node->constants = calloc(parser->constant_pool.size, sizeof(ID)); @@ -9103,6 +10111,7 @@ pm_load_file(pm_parse_result_t *result, VALUE filepath, bool load_error) VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath) { + result->node.filepath_encoding = rb_enc_get(filepath); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); diff --git a/prism_compile.h b/prism_compile.h index 0f82782ec0..8df82c5c7c 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -36,14 +36,16 @@ typedef struct pm_scope_node { */ rb_encoding *filepath_encoding; - // The size of the local table - // on the iseq which includes - // locals and hidden variables + // The size of the local table on the iseq which includes locals and hidden + // variables. int local_table_for_iseq_size; ID *constants; st_table *index_lookup_table; + // The current coverage setting, passed down through the various scopes. + int coverage_enabled; + /** * This will only be set on the top-level scope node. It will contain all of * the instructions pertaining to BEGIN{} nodes. @@ -2984,10 +2984,7 @@ rb_obj_traverse(VALUE obj, static int frozen_shareable_p(VALUE obj, bool *made_shareable) { - if (CHILLED_STRING_P(obj)) { - return false; - } - else if (!RB_TYPE_P(obj, T_DATA)) { + if (!RB_TYPE_P(obj, T_DATA)) { return true; } else if (RTYPEDDATA_P(obj)) { @@ -3016,18 +3013,7 @@ make_shareable_check_shareable(VALUE obj) if (rb_ractor_shareable_p(obj)) { return traverse_skip; } - else if (CHILLED_STRING_P(obj)) { - rb_funcall(obj, idFreeze, 0); - - if (UNLIKELY(!RB_OBJ_FROZEN_RAW(obj))) { - rb_raise(rb_eRactorError, "#freeze does not freeze object correctly"); - } - - if (RB_OBJ_SHAREABLE_P(obj)) { - return traverse_skip; - } - } - else if (!frozen_shareable_p(obj, &made_shareable)) { + if (!frozen_shareable_p(obj, &made_shareable)) { if (made_shareable) { return traverse_skip; } @@ -131,8 +131,9 @@ typedef struct { static VALUE rand_init(const rb_random_interface_t *, rb_random_t *, VALUE); static VALUE random_seed(VALUE); -static void fill_random_seed(uint32_t *seed, size_t cnt); +static void fill_random_seed(uint32_t *seed, size_t cnt, bool try_bytes); static VALUE make_seed_value(uint32_t *ptr, size_t len); +#define fill_random_bytes ruby_fill_random_bytes RB_RANDOM_INTERFACE_DECLARE(rand_mt); static const rb_random_interface_t random_mt_if = { @@ -354,7 +355,7 @@ rand_init_default(const rb_random_interface_t *rng, rb_random_t *rnd) size_t len = roomof(rng->default_seed_bits, 32); uint32_t *buf = ALLOCV_N(uint32_t, buf0, len+1); - fill_random_seed(buf, len); + fill_random_seed(buf, len, true); rng->init(rnd, buf, len); seed = make_seed_value(buf, len); explicit_bzero(buf, len * sizeof(*buf)); @@ -566,8 +567,6 @@ fill_random_bytes_syscall(void *buf, size_t size, int unused) #endif # if defined(CRYPT_VERIFYCONTEXT) -STATIC_ASSERT(sizeof_HCRYPTPROV, sizeof(HCRYPTPROV) == sizeof(size_t)); - /* Although HCRYPTPROV is not a HANDLE, it looks like * INVALID_HANDLE_VALUE is not a valid value */ static const HCRYPTPROV INVALID_HCRYPTPROV = (HCRYPTPROV)INVALID_HANDLE_VALUE; @@ -576,27 +575,33 @@ static void release_crypt(void *p) { HCRYPTPROV *ptr = p; - HCRYPTPROV prov = (HCRYPTPROV)ATOMIC_SIZE_EXCHANGE(*ptr, INVALID_HCRYPTPROV); + HCRYPTPROV prov = (HCRYPTPROV)ATOMIC_PTR_EXCHANGE(*ptr, INVALID_HCRYPTPROV); if (prov && prov != INVALID_HCRYPTPROV) { CryptReleaseContext(prov, 0); } } +static const rb_data_type_t crypt_prov_type = { + "HCRYPTPROV", + {0, release_crypt,}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE +}; + static int fill_random_bytes_crypt(void *seed, size_t size) { static HCRYPTPROV perm_prov; HCRYPTPROV prov = perm_prov, old_prov; if (!prov) { + VALUE wrapper = TypedData_Wrap_Struct(0, &crypt_prov_type, 0); if (!CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) { prov = INVALID_HCRYPTPROV; } - old_prov = (HCRYPTPROV)ATOMIC_SIZE_CAS(perm_prov, 0, prov); + old_prov = (HCRYPTPROV)ATOMIC_PTR_CAS(perm_prov, 0, prov); if (LIKELY(!old_prov)) { /* no other threads acquired */ if (prov != INVALID_HCRYPTPROV) { -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - rb_vm_register_global_object(Data_Wrap_Struct(0, 0, release_crypt, &perm_prov)); + DATA_PTR(wrapper) = (void *)prov; + rb_vm_register_global_object(wrapper); } } else { /* another thread acquired */ @@ -673,11 +678,9 @@ ruby_fill_random_bytes(void *seed, size_t size, int need_secure) return fill_random_bytes_urandom(seed, size); } -#define fill_random_bytes ruby_fill_random_bytes - /* cnt must be 4 or more */ static void -fill_random_seed(uint32_t *seed, size_t cnt) +fill_random_seed(uint32_t *seed, size_t cnt, bool try_bytes) { static rb_atomic_t n = 0; #if defined HAVE_CLOCK_GETTIME @@ -687,10 +690,12 @@ fill_random_seed(uint32_t *seed, size_t cnt) #endif size_t len = cnt * sizeof(*seed); - memset(seed, 0, len); - - fill_random_bytes(seed, len, FALSE); + if (try_bytes) { + fill_random_bytes(seed, len, FALSE); + return; + } + memset(seed, 0, len); #if defined HAVE_CLOCK_GETTIME clock_gettime(CLOCK_REALTIME, &tv); seed[0] ^= tv.tv_nsec; @@ -725,8 +730,8 @@ make_seed_value(uint32_t *ptr, size_t len) return seed; } -#define with_random_seed(size, add) \ - for (uint32_t seedbuf[(size)+(add)], loop = (fill_random_seed(seedbuf, (size)), 1); \ +#define with_random_seed(size, add, try_bytes) \ + for (uint32_t seedbuf[(size)+(add)], loop = (fill_random_seed(seedbuf, (size), try_bytes), 1); \ loop; explicit_bzero(seedbuf, (size)*sizeof(seedbuf[0])), loop = 0) /* @@ -741,7 +746,7 @@ static VALUE random_seed(VALUE _) { VALUE v; - with_random_seed(DEFAULT_SEED_CNT, 1) { + with_random_seed(DEFAULT_SEED_CNT, 1, true) { v = make_seed_value(seedbuf, DEFAULT_SEED_CNT); } return v; @@ -1770,7 +1775,7 @@ Init_RandomSeedCore(void) */ struct MT mt; - with_random_seed(DEFAULT_SEED_CNT, 0) { + with_random_seed(DEFAULT_SEED_CNT, 0, false) { init_by_array(&mt, seedbuf, DEFAULT_SEED_CNT); } diff --git a/rational.c b/rational.c index 014cbb6c6a..1b162e7b56 100644 --- a/rational.c +++ b/rational.c @@ -22,9 +22,6 @@ # define USE_GMP 0 #endif #endif -#if USE_GMP -#include <gmp.h> -#endif #include "id.h" #include "internal.h" @@ -36,6 +33,15 @@ #include "internal/rational.h" #include "ruby_assert.h" +#if USE_GMP +RBIMPL_WARNING_PUSH() +# ifdef _MSC_VER +RBIMPL_WARNING_IGNORED(4146) /* for mpn_neg() */ +# endif +# include <gmp.h> +RBIMPL_WARNING_POP() +#endif + #define ZERO INT2FIX(0) #define ONE INT2FIX(1) #define TWO INT2FIX(2) @@ -519,6 +519,7 @@ extern VALUE rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_fram extern VALUE rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); extern VALUE rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); extern VALUE rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); +extern VALUE rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr, VALUE fmt); extern VALUE rb_vm_splat_array(VALUE flag, VALUE array); extern bool rb_simple_iseq_p(const rb_iseq_t *iseq); extern bool rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v); @@ -171,9 +171,9 @@ module RubyVM::RJIT # :nodoc: all me_addr == 0 ? nil : rb_method_entry_t.new(me_addr) end - def rb_shape_get_next(shape, obj, id) + def rb_shape_get_next_no_warnings(shape, obj, id) _shape = shape.to_i - shape_addr = Primitive.cexpr! 'SIZET2NUM((size_t)rb_shape_get_next((rb_shape_t *)NUM2SIZET(_shape), obj, (ID)NUM2SIZET(id)))' + shape_addr = Primitive.cexpr! 'SIZET2NUM((size_t)rb_shape_get_next_no_warnings((rb_shape_t *)NUM2SIZET(_shape), obj, (ID)NUM2SIZET(id)))' rb_shape_t.new(shape_addr) end @@ -691,6 +691,10 @@ module RubyVM::RJIT # :nodoc: all Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_opt_newarray_min) } end + def C.rb_vm_opt_newarray_pack + Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_opt_newarray_pack) } + end + def C.rb_vm_set_ivar_id Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_set_ivar_id) } end @@ -2099,27 +2099,6 @@ process_script(ruby_cmdline_options_t *opt) } /** - * Call ruby_opt_init to set up the global state based on the command line - * options, and then warn if prism is enabled and the experimental warning - * category is enabled. - */ -static void -prism_opt_init(ruby_cmdline_options_t *opt) -{ - ruby_opt_init(opt); - - if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) { - rb_category_warn( - RB_WARN_CATEGORY_EXPERIMENTAL, - "The compiler based on the Prism parser is currently experimental " - "and compatibility with the compiler based on parse.y is not yet " - "complete. Please report any issues you find on the `ruby/prism` " - "issue tracker." - ); - } -} - -/** * Process the command line options and parse the script into the given result. * Raise an error if the script cannot be parsed. */ @@ -2147,14 +2126,15 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) pm_options_command_line_set(options, command_line); pm_options_filepath_set(options, "-"); - prism_opt_init(opt); + ruby_opt_init(opt); error = pm_parse_stdin(result); } else if (opt->e_script) { command_line |= PM_OPTIONS_COMMAND_LINE_E; pm_options_command_line_set(options, command_line); - prism_opt_init(opt); + ruby_opt_init(opt); + result->node.coverage_enabled = 0; error = pm_parse_string(result, opt->e_script, rb_str_new2("-e")); } else { @@ -2165,7 +2145,7 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) // line options. We do it in this order so that if the main script fails // to load, it doesn't require files required by -r. if (NIL_P(error)) { - prism_opt_init(opt); + ruby_opt_init(opt); error = pm_parse_file(result, opt->script_name); } @@ -2301,7 +2281,8 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt) #endif #if USE_YJIT if (FEATURE_SET_P(opt->features, yjit)) { - opt->yjit = true; // set opt->yjit for Init_ruby_description() and calling rb_yjit_init() + bool rb_yjit_option_disable(void); + opt->yjit = !rb_yjit_option_disable(); // set opt->yjit for Init_ruby_description() and calling rb_yjit_init() } #endif diff --git a/ruby_parser.c b/ruby_parser.c index f9d4e6d59c..93901ea4d9 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -32,6 +32,8 @@ #include "vm_core.h" #include "symbol.h" +#define parser_encoding const void + static int is_ascii_string2(VALUE str) { @@ -41,9 +43,9 @@ is_ascii_string2(VALUE str) RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0) static VALUE syntax_error_append(VALUE exc, VALUE file, int line, int column, - void *enc, const char *fmt, va_list args) + parser_encoding *enc, const char *fmt, va_list args) { - return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args); + return rb_syntax_error_append(exc, file, line, column, enc, fmt, args); } static int @@ -59,9 +61,9 @@ dvar_defined(ID id, const void *p) } static int -is_usascii_enc(void *enc) +is_usascii_enc(parser_encoding *enc) { - return rb_is_usascii_enc((rb_encoding *)enc); + return rb_is_usascii_enc(enc); } static int @@ -83,21 +85,21 @@ is_notop_id2(ID id) } static VALUE -enc_str_new(const char *ptr, long len, void *enc) +enc_str_new(const char *ptr, long len, parser_encoding *enc) { - return rb_enc_str_new(ptr, len, (rb_encoding *)enc); + return rb_enc_str_new(ptr, len, enc); } static int -enc_isalnum(OnigCodePoint c, void *enc) +enc_isalnum(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isalnum(c, (rb_encoding *)enc); + return rb_enc_isalnum(c, enc); } static int -enc_precise_mbclen(const char *p, const char *e, void *enc) +enc_precise_mbclen(const char *p, const char *e, parser_encoding *enc) { - return rb_enc_precise_mbclen(p, e, (rb_encoding *)enc); + return rb_enc_precise_mbclen(p, e, enc); } static int @@ -113,93 +115,87 @@ mbclen_charfound_len(int len) } static const char * -enc_name(void *enc) +enc_name(parser_encoding *enc) { - return rb_enc_name((rb_encoding *)enc); + return rb_enc_name(enc); } static char * -enc_prev_char(const char *s, const char *p, const char *e, void *enc) +enc_prev_char(const char *s, const char *p, const char *e, parser_encoding *enc) { - return rb_enc_prev_char(s, p, e, (rb_encoding *)enc); + return rb_enc_prev_char(s, p, e, enc); } -static void * +static parser_encoding * enc_get(VALUE obj) { - return (void *)rb_enc_get(obj); + return rb_enc_get(obj); } static int -enc_asciicompat(void *enc) +enc_asciicompat(parser_encoding *enc) { - return rb_enc_asciicompat((rb_encoding *)enc); + return rb_enc_asciicompat(enc); } -static void * +static parser_encoding * utf8_encoding(void) { - return (void *)rb_utf8_encoding(); + return rb_utf8_encoding(); } static VALUE -enc_associate(VALUE obj, void *enc) +enc_associate(VALUE obj, parser_encoding *enc) { - return rb_enc_associate(obj, (rb_encoding *)enc); + return rb_enc_associate(obj, enc); } -static void * +static parser_encoding * ascii8bit_encoding(void) { - return (void *)rb_ascii8bit_encoding(); + return rb_ascii8bit_encoding(); } static int -enc_codelen(int c, void *enc) +enc_codelen(int c, parser_encoding *enc) { - return rb_enc_codelen(c, (rb_encoding *)enc); + return rb_enc_codelen(c, enc); } static int -enc_mbcput(unsigned int c, void *buf, void *enc) +enc_mbcput(unsigned int c, void *buf, parser_encoding *enc) { - return rb_enc_mbcput(c, buf, (rb_encoding *)enc); + return rb_enc_mbcput(c, buf, enc); } -static int -enc_mbclen(const char *p, const char *e, void *enc) -{ - return rb_enc_mbclen(p, e, (rb_encoding *)enc); -} - -static void * +static parser_encoding * enc_from_index(int idx) { - return (void *)rb_enc_from_index(idx); + return rb_enc_from_index(idx); } static int -enc_isspace(OnigCodePoint c, void *enc) +enc_isspace(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isspace(c, (rb_encoding *)enc); + return rb_enc_isspace(c, enc); } static ID -intern3(const char *name, long len, void *enc) +intern3(const char *name, long len, parser_encoding *enc) { - return rb_intern3(name, len, (rb_encoding *)enc); + return rb_intern3(name, len, enc); } -static void * +static parser_encoding * usascii_encoding(void) { - return (void *)rb_usascii_encoding(); + return rb_usascii_encoding(); } static int -enc_symname_type(const char *name, long len, void *enc, unsigned int allowed_attrset) +enc_symname_type(const char *name, long len, parser_encoding *enc, unsigned int allowed_attrset) { - return rb_enc_symname_type(name, len, (rb_encoding *)enc, allowed_attrset); + return rb_enc_symname_type(name, len, enc, allowed_attrset); } typedef struct { @@ -220,7 +216,7 @@ reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end, long len = name_end - name; const char *s = (const char *)name; - return rb_reg_named_capture_assign_iter_impl(p, s, len, (void *)enc, &arg->succ_block, loc); + return rb_reg_named_capture_assign_iter_impl(p, s, len, enc, &arg->succ_block, loc); } static NODE * @@ -305,25 +301,25 @@ static_id2sym(ID id) } static long -str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr) +str_coderange_scan_restartable(const char *s, const char *e, parser_encoding *enc, int *cr) { - return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr); + return rb_str_coderange_scan_restartable(s, e, enc, cr); } static int -enc_mbminlen(void *enc) +enc_mbminlen(parser_encoding *enc) { - return rb_enc_mbminlen((rb_encoding *)enc); + return rb_enc_mbminlen(enc); } static bool -enc_isascii(OnigCodePoint c, void *enc) +enc_isascii(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isascii(c, (rb_encoding *)enc); + return rb_enc_isascii(c, enc); } static OnigCodePoint -enc_mbc_to_codepoint(const char *p, const char *e, void *enc) +enc_mbc_to_codepoint(const char *p, const char *e, parser_encoding *enc) { const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p); const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e); @@ -385,7 +381,6 @@ static const rb_parser_config_t rb_global_parser_config = { .is_ascii_string = is_ascii_string2, .enc_str_new = enc_str_new, .str_vcatf = rb_str_vcatf, - .string_value_cstr = rb_string_value_cstr, .rb_sprintf = rb_sprintf, .rstring_ptr = RSTRING_PTR, .rstring_end = RSTRING_END, @@ -417,7 +412,6 @@ static const rb_parser_config_t rb_global_parser_config = { .ascii8bit_encoding = ascii8bit_encoding, .enc_codelen = enc_codelen, .enc_mbcput = enc_mbcput, - .enc_mbclen = enc_mbclen, .enc_find_index = rb_enc_find_index, .enc_from_index = enc_from_index, .enc_isspace = enc_isspace, @@ -662,19 +656,7 @@ static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines); static rb_ast_t* parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line) { - rb_ast_t *ast; - const char *ptr = 0; - long len = 0; - rb_encoding *enc = 0; - - if (!NIL_P(fname)) { - StringValueCStr(fname); - ptr = RSTRING_PTR(fname); - len = RSTRING_LEN(fname); - enc = rb_enc_get(fname); - } - - ast = rb_parser_compile(p, gets, ptr, len, enc, input, line); + rb_ast_t *ast = rb_parser_compile(p, gets, fname, input, line); parser_aset_script_lines_for(fname, ast->body.script_lines); return ast; } @@ -919,7 +901,7 @@ rb_parser_build_script_lines_from(rb_parser_ary_t *lines) VALUE rb_str_new_parser_string(rb_parser_string_t *str) { - VALUE string = rb_enc_interned_str(str->ptr, str->len, str->enc); + VALUE string = rb_enc_literal_str(str->ptr, str->len, str->enc); rb_enc_str_coderange(string); return string; } diff --git a/rubyparser.h b/rubyparser.h index cb90b847b6..c98f4a91a5 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -9,7 +9,7 @@ #ifdef UNIVERSAL_PARSER -#define rb_encoding void +#define rb_encoding const void #define OnigCodePoint unsigned int #include "parser_st.h" #ifndef RUBY_RUBY_H @@ -731,31 +731,22 @@ typedef struct RNode_STR { struct rb_parser_string *string; } rb_node_str_t; -/* RNode_DSTR, RNode_DXSTR and RNode_DSYM should be same structure */ +/* RNode_DSTR, RNode_DXSTR, RNode_DREGX and RNode_DSYM should be same structure */ typedef struct RNode_DSTR { NODE node; struct rb_parser_string *string; union { long nd_alen; + long nd_cflag; struct RNode *nd_end; /* Second dstr node has this structure. See also RNode_LIST */ } as; struct RNode_LIST *nd_next; } rb_node_dstr_t; -typedef struct RNode_XSTR { - NODE node; - - struct rb_parser_string *string; -} rb_node_xstr_t; +typedef rb_node_str_t rb_node_xstr_t; -typedef struct RNode_DXSTR { - NODE node; - - struct rb_parser_string *string; - long nd_alen; - struct RNode_LIST *nd_next; -} rb_node_dxstr_t; +typedef rb_node_dstr_t rb_node_dxstr_t; typedef struct RNode_EVSTR { NODE node; @@ -770,13 +761,7 @@ typedef struct RNode_REGX { int options; } rb_node_regx_t; -typedef struct RNode_DREGX { - NODE node; - - struct rb_parser_string *string; - ID nd_cflag; - struct RNode_LIST *nd_next; -} rb_node_dregx_t; +typedef rb_node_dstr_t rb_node_dregx_t; typedef struct RNode_ONCE { NODE node; @@ -952,19 +937,8 @@ typedef struct RNode_DOT3 { struct RNode *nd_end; } rb_node_dot3_t; -typedef struct RNode_FLIP2 { - NODE node; - - struct RNode *nd_beg; - struct RNode *nd_end; -} rb_node_flip2_t; - -typedef struct RNode_FLIP3 { - NODE node; - - struct RNode *nd_beg; - struct RNode *nd_end; -} rb_node_flip3_t; +typedef rb_node_dot2_t rb_node_flip2_t; +typedef rb_node_dot3_t rb_node_flip3_t; typedef struct RNode_SELF { NODE node; @@ -1006,13 +980,7 @@ typedef struct RNode_SYM { struct rb_parser_string *string; } rb_node_sym_t; -typedef struct RNode_DSYM { - NODE node; - - struct rb_parser_string *string; - long nd_alen; - struct RNode_LIST *nd_next; -} rb_node_dsym_t; +typedef rb_node_dstr_t rb_node_dsym_t; typedef struct RNode_ATTRASGN { NODE node; @@ -1073,119 +1041,119 @@ typedef struct RNode_ERROR { NODE node; } rb_node_error_t; -#define RNODE(obj) ((struct RNode *)(obj)) - -#define RNODE_SCOPE(node) ((struct RNode_SCOPE *)(node)) -#define RNODE_BLOCK(node) ((struct RNode_BLOCK *)(node)) -#define RNODE_IF(node) ((struct RNode_IF *)(node)) -#define RNODE_UNLESS(node) ((struct RNode_UNLESS *)(node)) -#define RNODE_CASE(node) ((struct RNode_CASE *)(node)) -#define RNODE_CASE2(node) ((struct RNode_CASE2 *)(node)) -#define RNODE_CASE3(node) ((struct RNode_CASE3 *)(node)) -#define RNODE_WHEN(node) ((struct RNode_WHEN *)(node)) -#define RNODE_IN(node) ((struct RNode_IN *)(node)) -#define RNODE_WHILE(node) ((struct RNode_WHILE *)(node)) -#define RNODE_UNTIL(node) ((struct RNode_UNTIL *)(node)) -#define RNODE_ITER(node) ((struct RNode_ITER *)(node)) -#define RNODE_FOR(node) ((struct RNode_FOR *)(node)) -#define RNODE_FOR_MASGN(node) ((struct RNode_FOR_MASGN *)(node)) -#define RNODE_BREAK(node) ((struct RNode_BREAK *)(node)) -#define RNODE_NEXT(node) ((struct RNode_NEXT *)(node)) -#define RNODE_REDO(node) ((struct RNode_REDO *)(node)) -#define RNODE_RETRY(node) ((struct RNode_RETRY *)(node)) -#define RNODE_BEGIN(node) ((struct RNode_BEGIN *)(node)) -#define RNODE_RESCUE(node) ((struct RNode_RESCUE *)(node)) -#define RNODE_RESBODY(node) ((struct RNode_RESBODY *)(node)) -#define RNODE_ENSURE(node) ((struct RNode_ENSURE *)(node)) -#define RNODE_AND(node) ((struct RNode_AND *)(node)) -#define RNODE_OR(node) ((struct RNode_OR *)(node)) -#define RNODE_MASGN(node) ((struct RNode_MASGN *)(node)) -#define RNODE_LASGN(node) ((struct RNode_LASGN *)(node)) -#define RNODE_DASGN(node) ((struct RNode_DASGN *)(node)) -#define RNODE_GASGN(node) ((struct RNode_GASGN *)(node)) -#define RNODE_IASGN(node) ((struct RNode_IASGN *)(node)) -#define RNODE_CDECL(node) ((struct RNode_CDECL *)(node)) -#define RNODE_CVASGN(node) ((struct RNode_CVASGN *)(node)) -#define RNODE_OP_ASGN1(node) ((struct RNode_OP_ASGN1 *)(node)) -#define RNODE_OP_ASGN2(node) ((struct RNode_OP_ASGN2 *)(node)) -#define RNODE_OP_ASGN_AND(node) ((struct RNode_OP_ASGN_AND *)(node)) -#define RNODE_OP_ASGN_OR(node) ((struct RNode_OP_ASGN_OR *)(node)) -#define RNODE_OP_CDECL(node) ((struct RNode_OP_CDECL *)(node)) -#define RNODE_CALL(node) ((struct RNode_CALL *)(node)) -#define RNODE_OPCALL(node) ((struct RNode_OPCALL *)(node)) -#define RNODE_FCALL(node) ((struct RNode_FCALL *)(node)) -#define RNODE_VCALL(node) ((struct RNode_VCALL *)(node)) -#define RNODE_QCALL(node) ((struct RNode_QCALL *)(node)) -#define RNODE_SUPER(node) ((struct RNode_SUPER *)(node)) -#define RNODE_ZSUPER(node) ((struct RNode_ZSUPER *)(node)) -#define RNODE_LIST(node) ((struct RNode_LIST *)(node)) -#define RNODE_ZLIST(node) ((struct RNode_ZLIST *)(node)) -#define RNODE_HASH(node) ((struct RNode_HASH *)(node)) -#define RNODE_RETURN(node) ((struct RNode_RETURN *)(node)) -#define RNODE_YIELD(node) ((struct RNode_YIELD *)(node)) -#define RNODE_LVAR(node) ((struct RNode_LVAR *)(node)) -#define RNODE_DVAR(node) ((struct RNode_DVAR *)(node)) -#define RNODE_GVAR(node) ((struct RNode_GVAR *)(node)) -#define RNODE_IVAR(node) ((struct RNode_IVAR *)(node)) -#define RNODE_CONST(node) ((struct RNode_CONST *)(node)) -#define RNODE_CVAR(node) ((struct RNode_CVAR *)(node)) -#define RNODE_NTH_REF(node) ((struct RNode_NTH_REF *)(node)) -#define RNODE_BACK_REF(node) ((struct RNode_BACK_REF *)(node)) -#define RNODE_MATCH(node) ((struct RNode_MATCH *)(node)) -#define RNODE_MATCH2(node) ((struct RNode_MATCH2 *)(node)) -#define RNODE_MATCH3(node) ((struct RNode_MATCH3 *)(node)) -#define RNODE_INTEGER(node) ((struct RNode_INTEGER *)(node)) -#define RNODE_FLOAT(node) ((struct RNode_FLOAT *)(node)) -#define RNODE_RATIONAL(node) ((struct RNode_RATIONAL *)(node)) -#define RNODE_IMAGINARY(node) ((struct RNode_IMAGINARY *)(node)) -#define RNODE_STR(node) ((struct RNode_STR *)(node)) -#define RNODE_DSTR(node) ((struct RNode_DSTR *)(node)) -#define RNODE_XSTR(node) ((struct RNode_XSTR *)(node)) -#define RNODE_DXSTR(node) ((struct RNode_DXSTR *)(node)) -#define RNODE_EVSTR(node) ((struct RNode_EVSTR *)(node)) -#define RNODE_REGX(node) ((struct RNode_REGX *)(node)) -#define RNODE_DREGX(node) ((struct RNode_DREGX *)(node)) -#define RNODE_ONCE(node) ((struct RNode_ONCE *)(node)) -#define RNODE_ARGS(node) ((struct RNode_ARGS *)(node)) -#define RNODE_ARGS_AUX(node) ((struct RNode_ARGS_AUX *)(node)) -#define RNODE_OPT_ARG(node) ((struct RNode_OPT_ARG *)(node)) -#define RNODE_KW_ARG(node) ((struct RNode_KW_ARG *)(node)) -#define RNODE_POSTARG(node) ((struct RNode_POSTARG *)(node)) -#define RNODE_ARGSCAT(node) ((struct RNode_ARGSCAT *)(node)) -#define RNODE_ARGSPUSH(node) ((struct RNode_ARGSPUSH *)(node)) -#define RNODE_SPLAT(node) ((struct RNode_SPLAT *)(node)) -#define RNODE_BLOCK_PASS(node) ((struct RNode_BLOCK_PASS *)(node)) -#define RNODE_DEFN(node) ((struct RNode_DEFN *)(node)) -#define RNODE_DEFS(node) ((struct RNode_DEFS *)(node)) -#define RNODE_ALIAS(node) ((struct RNode_ALIAS *)(node)) -#define RNODE_VALIAS(node) ((struct RNode_VALIAS *)(node)) -#define RNODE_UNDEF(node) ((struct RNode_UNDEF *)(node)) -#define RNODE_CLASS(node) ((struct RNode_CLASS *)(node)) -#define RNODE_MODULE(node) ((struct RNode_MODULE *)(node)) -#define RNODE_SCLASS(node) ((struct RNode_SCLASS *)(node)) -#define RNODE_COLON2(node) ((struct RNode_COLON2 *)(node)) -#define RNODE_COLON3(node) ((struct RNode_COLON3 *)(node)) -#define RNODE_DOT2(node) ((struct RNode_DOT2 *)(node)) -#define RNODE_DOT3(node) ((struct RNode_DOT3 *)(node)) -#define RNODE_FLIP2(node) ((struct RNode_FLIP2 *)(node)) -#define RNODE_FLIP3(node) ((struct RNode_FLIP3 *)(node)) -#define RNODE_SELF(node) ((struct RNode_SELF *)(node)) -#define RNODE_NIL(node) ((struct RNode_NIL *)(node)) -#define RNODE_TRUE(node) ((struct RNode_TRUE *)(node)) -#define RNODE_FALSE(node) ((struct RNode_FALSE *)(node)) -#define RNODE_ERRINFO(node) ((struct RNode_ERRINFO *)(node)) -#define RNODE_DEFINED(node) ((struct RNode_DEFINED *)(node)) -#define RNODE_POSTEXE(node) ((struct RNode_POSTEXE *)(node)) -#define RNODE_SYM(node) ((struct RNode_SYM *)(node)) -#define RNODE_DSYM(node) ((struct RNode_DSYM *)(node)) -#define RNODE_ATTRASGN(node) ((struct RNode_ATTRASGN *)(node)) -#define RNODE_LAMBDA(node) ((struct RNode_LAMBDA *)(node)) -#define RNODE_ARYPTN(node) ((struct RNode_ARYPTN *)(node)) -#define RNODE_HSHPTN(node) ((struct RNode_HSHPTN *)(node)) -#define RNODE_FNDPTN(node) ((struct RNode_FNDPTN *)(node)) -#define RNODE_LINE(node) ((struct RNode_LINE *)(node)) -#define RNODE_FILE(node) ((struct RNode_FILE *)(node)) -#define RNODE_ENCODING(node) ((struct RNode_ENCODING *)(node)) +#define RNODE(obj) ((NODE *)(obj)) + +#define RNODE_SCOPE(node) ((rb_node_scope_t *)(node)) +#define RNODE_BLOCK(node) ((rb_node_block_t *)(node)) +#define RNODE_IF(node) ((rb_node_if_t *)(node)) +#define RNODE_UNLESS(node) ((rb_node_unless_t *)(node)) +#define RNODE_CASE(node) ((rb_node_case_t *)(node)) +#define RNODE_CASE2(node) ((rb_node_case2_t *)(node)) +#define RNODE_CASE3(node) ((rb_node_case3_t *)(node)) +#define RNODE_WHEN(node) ((rb_node_when_t *)(node)) +#define RNODE_IN(node) ((rb_node_in_t *)(node)) +#define RNODE_WHILE(node) ((rb_node_while_t *)(node)) +#define RNODE_UNTIL(node) ((rb_node_until_t *)(node)) +#define RNODE_ITER(node) ((rb_node_iter_t *)(node)) +#define RNODE_FOR(node) ((rb_node_for_t *)(node)) +#define RNODE_FOR_MASGN(node) ((rb_node_for_masgn_t *)(node)) +#define RNODE_BREAK(node) ((rb_node_break_t *)(node)) +#define RNODE_NEXT(node) ((rb_node_next_t *)(node)) +#define RNODE_REDO(node) ((rb_node_redo_t *)(node)) +#define RNODE_RETRY(node) ((rb_node_retry_t *)(node)) +#define RNODE_BEGIN(node) ((rb_node_begin_t *)(node)) +#define RNODE_RESCUE(node) ((rb_node_rescue_t *)(node)) +#define RNODE_RESBODY(node) ((rb_node_resbody_t *)(node)) +#define RNODE_ENSURE(node) ((rb_node_ensure_t *)(node)) +#define RNODE_AND(node) ((rb_node_and_t *)(node)) +#define RNODE_OR(node) ((rb_node_or_t *)(node)) +#define RNODE_MASGN(node) ((rb_node_masgn_t *)(node)) +#define RNODE_LASGN(node) ((rb_node_lasgn_t *)(node)) +#define RNODE_DASGN(node) ((rb_node_dasgn_t *)(node)) +#define RNODE_GASGN(node) ((rb_node_gasgn_t *)(node)) +#define RNODE_IASGN(node) ((rb_node_iasgn_t *)(node)) +#define RNODE_CDECL(node) ((rb_node_cdecl_t *)(node)) +#define RNODE_CVASGN(node) ((rb_node_cvasgn_t *)(node)) +#define RNODE_OP_ASGN1(node) ((rb_node_op_asgn1_t *)(node)) +#define RNODE_OP_ASGN2(node) ((rb_node_op_asgn2_t *)(node)) +#define RNODE_OP_ASGN_AND(node) ((rb_node_op_asgn_and_t *)(node)) +#define RNODE_OP_ASGN_OR(node) ((rb_node_op_asgn_or_t *)(node)) +#define RNODE_OP_CDECL(node) ((rb_node_op_cdecl_t *)(node)) +#define RNODE_CALL(node) ((rb_node_call_t *)(node)) +#define RNODE_OPCALL(node) ((rb_node_opcall_t *)(node)) +#define RNODE_FCALL(node) ((rb_node_fcall_t *)(node)) +#define RNODE_VCALL(node) ((rb_node_vcall_t *)(node)) +#define RNODE_QCALL(node) ((rb_node_qcall_t *)(node)) +#define RNODE_SUPER(node) ((rb_node_super_t *)(node)) +#define RNODE_ZSUPER(node) ((rb_node_zsuper_t *)(node)) +#define RNODE_LIST(node) ((rb_node_list_t *)(node)) +#define RNODE_ZLIST(node) ((rb_node_zlist_t *)(node)) +#define RNODE_HASH(node) ((rb_node_hash_t *)(node)) +#define RNODE_RETURN(node) ((rb_node_return_t *)(node)) +#define RNODE_YIELD(node) ((rb_node_yield_t *)(node)) +#define RNODE_LVAR(node) ((rb_node_lvar_t *)(node)) +#define RNODE_DVAR(node) ((rb_node_dvar_t *)(node)) +#define RNODE_GVAR(node) ((rb_node_gvar_t *)(node)) +#define RNODE_IVAR(node) ((rb_node_ivar_t *)(node)) +#define RNODE_CONST(node) ((rb_node_const_t *)(node)) +#define RNODE_CVAR(node) ((rb_node_cvar_t *)(node)) +#define RNODE_NTH_REF(node) ((rb_node_nth_ref_t *)(node)) +#define RNODE_BACK_REF(node) ((rb_node_back_ref_t *)(node)) +#define RNODE_MATCH(node) ((rb_node_match_t *)(node)) +#define RNODE_MATCH2(node) ((rb_node_match2_t *)(node)) +#define RNODE_MATCH3(node) ((rb_node_match3_t *)(node)) +#define RNODE_INTEGER(node) ((rb_node_integer_t *)(node)) +#define RNODE_FLOAT(node) ((rb_node_float_t *)(node)) +#define RNODE_RATIONAL(node) ((rb_node_rational_t *)(node)) +#define RNODE_IMAGINARY(node) ((rb_node_imaginary_t *)(node)) +#define RNODE_STR(node) ((rb_node_str_t *)(node)) +#define RNODE_DSTR(node) ((rb_node_dstr_t *)(node)) +#define RNODE_XSTR(node) ((rb_node_xstr_t *)(node)) +#define RNODE_DXSTR(node) ((rb_node_dxstr_t *)(node)) +#define RNODE_EVSTR(node) ((rb_node_evstr_t *)(node)) +#define RNODE_REGX(node) ((rb_node_regx_t *)(node)) +#define RNODE_DREGX(node) ((rb_node_dregx_t *)(node)) +#define RNODE_ONCE(node) ((rb_node_once_t *)(node)) +#define RNODE_ARGS(node) ((rb_node_args_t *)(node)) +#define RNODE_ARGS_AUX(node) ((rb_node_args_aux_t *)(node)) +#define RNODE_OPT_ARG(node) ((rb_node_opt_arg_t *)(node)) +#define RNODE_KW_ARG(node) ((rb_node_kw_arg_t *)(node)) +#define RNODE_POSTARG(node) ((rb_node_postarg_t *)(node)) +#define RNODE_ARGSCAT(node) ((rb_node_argscat_t *)(node)) +#define RNODE_ARGSPUSH(node) ((rb_node_argspush_t *)(node)) +#define RNODE_SPLAT(node) ((rb_node_splat_t *)(node)) +#define RNODE_BLOCK_PASS(node) ((rb_node_block_pass_t *)(node)) +#define RNODE_DEFN(node) ((rb_node_defn_t *)(node)) +#define RNODE_DEFS(node) ((rb_node_defs_t *)(node)) +#define RNODE_ALIAS(node) ((rb_node_alias_t *)(node)) +#define RNODE_VALIAS(node) ((rb_node_valias_t *)(node)) +#define RNODE_UNDEF(node) ((rb_node_undef_t *)(node)) +#define RNODE_CLASS(node) ((rb_node_class_t *)(node)) +#define RNODE_MODULE(node) ((rb_node_module_t *)(node)) +#define RNODE_SCLASS(node) ((rb_node_sclass_t *)(node)) +#define RNODE_COLON2(node) ((rb_node_colon2_t *)(node)) +#define RNODE_COLON3(node) ((rb_node_colon3_t *)(node)) +#define RNODE_DOT2(node) ((rb_node_dot2_t *)(node)) +#define RNODE_DOT3(node) ((rb_node_dot3_t *)(node)) +#define RNODE_FLIP2(node) ((rb_node_flip2_t *)(node)) +#define RNODE_FLIP3(node) ((rb_node_flip3_t *)(node)) +#define RNODE_SELF(node) ((rb_node_self_t *)(node)) +#define RNODE_NIL(node) ((rb_node_nil_t *)(node)) +#define RNODE_TRUE(node) ((rb_node_true_t *)(node)) +#define RNODE_FALSE(node) ((rb_node_false_t *)(node)) +#define RNODE_ERRINFO(node) ((rb_node_errinfo_t *)(node)) +#define RNODE_DEFINED(node) ((rb_node_defined_t *)(node)) +#define RNODE_POSTEXE(node) ((rb_node_postexe_t *)(node)) +#define RNODE_SYM(node) ((rb_node_sym_t *)(node)) +#define RNODE_DSYM(node) ((rb_node_dsym_t *)(node)) +#define RNODE_ATTRASGN(node) ((rb_node_attrasgn_t *)(node)) +#define RNODE_LAMBDA(node) ((rb_node_lambda_t *)(node)) +#define RNODE_ARYPTN(node) ((rb_node_aryptn_t *)(node)) +#define RNODE_HSHPTN(node) ((rb_node_hshptn_t *)(node)) +#define RNODE_FNDPTN(node) ((rb_node_fndptn_t *)(node)) +#define RNODE_LINE(node) ((rb_node_line_t *)(node)) +#define RNODE_FILE(node) ((rb_node_file_t *)(node)) +#define RNODE_ENCODING(node) ((rb_node_encoding_t *)(node)) /* FL : 0..4: T_TYPES, 5: KEEP_WB, 6: PROMOTED, 7: FINALIZE, 8: UNUSED, 9: UNUSED, 10: EXIVAR, 11: FREEZE */ /* NODE_FL: 0..4: UNUSED, 5: UNUSED, 6: UNUSED, 7: NODE_FL_NEWLINE, @@ -1301,7 +1269,6 @@ typedef struct rb_parser_config_struct { VALUE (*enc_str_new)(const char *ptr, long len, rb_encoding *enc); RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 2, 0) VALUE (*str_vcatf)(VALUE str, const char *fmt, va_list ap); - char *(*string_value_cstr)(volatile VALUE *ptr); RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2) VALUE (*rb_sprintf)(const char *format, ...); char *(*rstring_ptr)(VALUE str); @@ -1338,7 +1305,6 @@ typedef struct rb_parser_config_struct { rb_encoding *(*ascii8bit_encoding)(void); int (*enc_codelen)(int c, rb_encoding *enc); int (*enc_mbcput)(unsigned int c, void *buf, rb_encoding *enc); - int (*enc_mbclen)(const char *p, const char *e, rb_encoding *enc); int (*enc_find_index)(const char *name); rb_encoding *(*enc_from_index)(int idx); int (*enc_isspace)(OnigCodePoint c, rb_encoding *enc); diff --git a/sample/find_calls.rb b/sample/find_calls.rb new file mode 100644 index 0000000000..30af56c719 --- /dev/null +++ b/sample/find_calls.rb @@ -0,0 +1,105 @@ +# This script finds calls to a specific method with a certain keyword parameter +# within a given source file. + +require "prism" +require "pp" + +# For deprecation or refactoring purposes, it's often useful to find all of the +# places that call a specific method with a specific k eyword parameter. This is +# easily accomplished with a visitor such as this one. +class QuxParameterVisitor < Prism::Visitor + def initialize(calls) + @calls = calls + end + + def visit_call_node(node) + @calls << node if qux?(node) + super + end + + private + + def qux?(node) + # All nodes implement pattern matching, so you can use the `in` operator to + # pull out all of their individual fields. As you can see by this extensive + # pattern match, this is quite a powerful feature. + node in { + # This checks that the receiver is the constant Qux or the constant path + # ::Qux. We are assuming relative constants are fine in this case. + receiver: ( + Prism::ConstantReadNode[name: :Qux] | + Prism::ConstantPathNode[parent: nil, name: :Qux] + ), + # This checks that the name of the method is qux. We purposefully are not + # checking the call operator (., ::, or &.) because we want all of them. + # In other ASTs, this would be multiple node types, but prism combines + # them all into one for convenience. + name: :qux, + arguments: Prism::ArgumentsNode[ + # Here we're going to use the "find" pattern to find the keyword hash + # node that has the correct key. + arguments: [ + *, + Prism::KeywordHashNode[ + # Here we'll use another "find" pattern to find the key that we are + # specifically looking for. + elements: [ + *, + # Finally, we can assert against the key itself. Note that we are + # not looking at the value of hash pair, because we are only + # specifically looking for a key. + Prism::AssocNode[key: Prism::SymbolNode[unescaped: "qux"]], + * + ] + ], + * + ] + ] + } + end +end + +calls = [] +Prism.parse_stream(DATA).value.accept(QuxParameterVisitor.new(calls)) + +calls.each do |call| + print "CallNode " + puts PP.pp(call.location, +"") + print " " + puts call.slice +end + +# => +# CallNode (5,6)-(5,29) +# Qux.qux(222, qux: true) +# CallNode (9,6)-(9,30) +# Qux&.qux(333, qux: true) +# CallNode (20,6)-(20,51) +# Qux::qux(888, qux: ::Qux.qux(999, qux: true)) +# CallNode (20,25)-(20,50) +# ::Qux.qux(999, qux: true) + +__END__ +module Foo + class Bar + def baz1 + Qux.qux(111) + Qux.qux(222, qux: true) + end + + def baz2 + Qux&.qux(333, qux: true) + Qux&.qux(444) + end + + def baz3 + qux(555, qux: false) + 666.qux(666) + end + + def baz4 + Qux::qux(777) + Qux::qux(888, qux: ::Qux.qux(999, qux: true)) + end + end +end diff --git a/sample/find_comments.rb b/sample/find_comments.rb new file mode 100644 index 0000000000..6a26cd32b7 --- /dev/null +++ b/sample/find_comments.rb @@ -0,0 +1,100 @@ +# This script finds all of the comments within a given source file for a method. + +require "prism" + +class FindMethodComments < Prism::Visitor + def initialize(target, comments, nesting = []) + @target = target + @comments = comments + @nesting = nesting + end + + # These visit methods are specific to each class. Defining a visitor allows + # you to group functionality that applies to all node types into a single + # class. You can find which method corresponds to which node type by looking + # at the class name, calling #type on the node, or by looking at the #accept + # method definition on the node. + def visit_module_node(node) + visitor = FindMethodComments.new(@target, @comments, [*@nesting, node.name]) + node.compact_child_nodes.each { |child| child.accept(visitor) } + end + + def visit_class_node(node) + # We could keep track of an internal state where we push the class name here + # and then pop it after the visit is complete. However, it is often simpler + # and cleaner to generate a new visitor instance when the state changes, + # because then the state is immutable and it's easier to reason about. This + # also provides for more debugging opportunity in the initializer. + visitor = FindMethodComments.new(@target, @comments, [*@nesting, node.name]) + node.compact_child_nodes.each { |child| child.accept(visitor) } + end + + def visit_def_node(node) + if [*@nesting, node.name] == @target + # Comments are always attached to locations (either inner locations on a + # node like the location of a keyword or the location on the node itself). + # Nodes are considered either "leading" or "trailing", which means that + # they occur before or after the location, respectively. In this case of + # documentation, we only want to consider leading comments. You can also + # fetch all of the comments on a location with #comments. + @comments.concat(node.location.leading_comments) + else + super + end + end +end + +# Most of the time, the concept of "finding" something in the AST can be +# accomplished either with a queue or with a visitor. In this case we will use a +# visitor, but a queue would work just as well. +def find_comments(result, path) + target = path.split(/::|#/).map(&:to_sym) + comments = [] + + result.value.accept(FindMethodComments.new(target, comments)) + comments +end + +result = Prism.parse_stream(DATA) +result.attach_comments! + +find_comments(result, "Foo#foo").each do |comment| + puts comment.inspect + puts comment.slice +end + +# => +# #<Prism::InlineComment @location=#<Prism::Location @start_offset=205 @length=27 start_line=13>> +# # This is the documentation +# #<Prism::InlineComment @location=#<Prism::Location @start_offset=235 @length=21 start_line=14>> +# # for the foo method. + +find_comments(result, "Foo::Bar#bar").each do |comment| + puts comment.inspect + puts comment.slice +end + +# => +# #<Prism::InlineComment @location=#<Prism::Location @start_offset=126 @length=23 start_line=7>> +# # This is documentation +# #<Prism::InlineComment @location=#<Prism::Location @start_offset=154 @length=21 start_line=8>> +# # for the bar method. + +__END__ +# This is the documentation +# for the Foo module. +module Foo + # This is documentation + # for the Bar class. + class Bar + # This is documentation + # for the bar method. + def bar + end + end + + # This is the documentation + # for the foo method. + def foo + end +end diff --git a/sample/locate_nodes.rb b/sample/locate_nodes.rb new file mode 100644 index 0000000000..7a51db4367 --- /dev/null +++ b/sample/locate_nodes.rb @@ -0,0 +1,84 @@ +# This script locates a set of nodes determined by a line and column (in bytes). + +require "prism" +require "pp" + +# This method determines if the given location covers the given line and column. +# It's important to note that columns (and offsets) in prism are always in +# bytes. This is because prism supports all 90 source encodings that Ruby +# supports. You can always retrieve the column (or offset) of a location in +# other units with other provided APIs, like #start_character_column or +# #start_code_units_column. +def covers?(location, line:, column:) + start_line = location.start_line + end_line = location.end_line + + if start_line == end_line + # If the location only spans one line, then we only check if the line + # matches and that the column is covered by the column range. + line == start_line && (location.start_column...location.end_column).cover?(column) + else + # Otherwise, we check that it is on the start line and the column is greater + # than or equal to the start column, or that it is on the end line and the + # column is less than the end column, or that it is between the start and + # end lines. + (line == start_line && column >= location.start_column) || + (line == end_line && column < location.end_column) || + (line > start_line && line < end_line) + end +end + +# This method descends down into the AST whose root is `node` and returns the +# array of all of the nodes that cover the given line and column. +def locate(node, line:, column:) + queue = [node] + result = [] + + # We could use a recursive method here instead if we wanted, but it's + # important to note that that will not work for ASTs that are nested deeply + # enough to cause a stack overflow. + while (node = queue.shift) + result << node + + # Nodes have `child_nodes` and `compact_child_nodes`. `child_nodes` have + # consistent indices but include `nil` for optional fields that are not + # present, whereas `compact_child_nodes` has inconsistent indices but does + # not include `nil` for optional fields that are not present. + node.compact_child_nodes.find do |child| + queue << child if covers?(child.location, line: line, column: column) + end + end + + result +end + +result = Prism.parse_stream(DATA) +locate(result.value, line: 4, column: 14).each_with_index do |node, index| + print " " * index + print node.class.name.split("::", 2).last + print " " + puts PP.pp(node.location, +"") +end + +# => +# ProgramNode (1,0)-(7,3) +# StatementsNode (1,0)-(7,3) +# ModuleNode (1,0)-(7,3) +# StatementsNode (2,2)-(6,5) +# ClassNode (2,2)-(6,5) +# StatementsNode (3,4)-(5,7) +# DefNode (3,4)-(5,7) +# StatementsNode (4,6)-(4,21) +# CallNode (4,6)-(4,21) +# CallNode (4,6)-(4,15) +# ArgumentsNode (4,12)-(4,15) +# IntegerNode (4,12)-(4,15) + +__END__ +module Foo + class Bar + def baz + 111 + 222 + 333 + end + end +end diff --git a/sample/visit_nodes.rb b/sample/visit_nodes.rb new file mode 100644 index 0000000000..5ba703b0a3 --- /dev/null +++ b/sample/visit_nodes.rb @@ -0,0 +1,63 @@ +# This script visits all of the nodes of a specific type within a given source +# file. It uses the visitor class to traverse the AST. + +require "prism" +require "pp" + +class CaseInsensitiveRegularExpressionVisitor < Prism::Visitor + def initialize(regexps) + @regexps = regexps + end + + # As the visitor is walking the tree, this method will only be called when it + # encounters a regular expression node. We can then call any regular + # expression -specific APIs. In this case, we are only interested in the + # regular expressions that are case-insensitive, which we can retrieve with + # the #ignore_case? method. + def visit_regular_expression_node(node) + @regexps << node if node.ignore_case? + super + end + + def visit_interpolated_regular_expression_node(node) + @regexps << node if node.ignore_case? + + # The default behavior of the visitor is to continue visiting the children + # of the node. Because Ruby is so dynamic, it's actually possible for + # another regular expression to be interpolated in statements contained + # within the #{} contained in this interpolated regular expression node. By + # calling `super`, we ensure the visitor will continue. Failing to call + # `super` will cause the visitor to stop the traversal of the tree, which + # can also be useful in some cases. + super + end +end + +result = Prism.parse_stream(DATA) +regexps = [] + +result.value.accept(CaseInsensitiveRegularExpressionVisitor.new(regexps)) +regexps.each do |node| + print node.class.name.split("::", 2).last + print " " + puts PP.pp(node.location, +"") + + if node.is_a?(Prism::RegularExpressionNode) + print " " + p node.unescaped + end +end + +# => +# InterpolatedRegularExpressionNode (3,9)-(3,47) +# RegularExpressionNode (3,16)-(3,22) +# "bar" +# RegularExpressionNode (4,9)-(4,15) +# "bar" + +__END__ +class Foo + REG1 = /foo/ + REG2 = /foo #{/bar/i =~ "" ? "bar" : "baz"}/i + REG3 = /bar/i +end @@ -118,6 +118,7 @@ redblack_value(redblack_node_t * node) return (rb_shape_t *)((uintptr_t)node->value & (((uintptr_t)-1) - 1)); } +#ifdef HAVE_MMAP static redblack_id_t redblack_id_for(redblack_node_t * node) { @@ -292,6 +293,7 @@ redblack_insert(redblack_node_t * tree, ID key, rb_shape_t * value) return root; } } +#endif rb_shape_tree_t *rb_shape_tree_ptr = NULL; @@ -694,8 +696,8 @@ rb_shape_get_next_iv_shape(rb_shape_t* shape, ID id) return get_next_shape_internal(shape, id, SHAPE_IVAR, &dont_care, true); } -rb_shape_t * -rb_shape_get_next(rb_shape_t *shape, VALUE obj, ID id) +static inline rb_shape_t * +shape_get_next(rb_shape_t *shape, VALUE obj, ID id, bool emit_warnings) { RUBY_ASSERT(!is_instance_id(id) || RTEST(rb_sym2str(ID2SYM(id)))); if (UNLIKELY(shape->type == SHAPE_OBJ_TOO_COMPLEX)) { @@ -728,7 +730,7 @@ rb_shape_get_next(rb_shape_t *shape, VALUE obj, ID id) if (variation_created) { RCLASS_EXT(klass)->variation_count++; - if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_PERFORMANCE)) { + if (emit_warnings && rb_warning_category_enabled_p(RB_WARN_CATEGORY_PERFORMANCE)) { if (RCLASS_EXT(klass)->variation_count >= SHAPE_MAX_VARIATIONS) { rb_category_warn( RB_WARN_CATEGORY_PERFORMANCE, @@ -745,6 +747,18 @@ rb_shape_get_next(rb_shape_t *shape, VALUE obj, ID id) return new_shape; } +rb_shape_t * +rb_shape_get_next(rb_shape_t *shape, VALUE obj, ID id) +{ + return shape_get_next(shape, obj, id, true); +} + +rb_shape_t * +rb_shape_get_next_no_warnings(rb_shape_t *shape, VALUE obj, ID id) +{ + return shape_get_next(shape, obj, id, false); +} + // Same as rb_shape_get_iv_index, but uses a provided valid shape id and index // to return a result faster if branches of the shape tree are closely related. bool @@ -164,6 +164,7 @@ int rb_shape_frozen_shape_p(rb_shape_t* shape); rb_shape_t* rb_shape_transition_shape_frozen(VALUE obj); bool rb_shape_transition_shape_remove_ivar(VALUE obj, ID id, rb_shape_t *shape, VALUE * removed); rb_shape_t* rb_shape_get_next(rb_shape_t* shape, VALUE obj, ID id); +rb_shape_t* rb_shape_get_next_no_warnings(rb_shape_t* shape, VALUE obj, ID id); rb_shape_t * rb_shape_rebuild_shape(rb_shape_t * initial_shape, rb_shape_t * dest_shape); diff --git a/spec/bundler/bundler/bundler_spec.rb b/spec/bundler/bundler/bundler_spec.rb index 6a2e435e54..4cd21375b5 100644 --- a/spec/bundler/bundler/bundler_spec.rb +++ b/spec/bundler/bundler/bundler_spec.rb @@ -233,8 +233,8 @@ RSpec.describe Bundler do allow(Bundler).to receive(:root).and_return(bundled_app) - Bundler.mkdir_p(bundled_app.join("foo", "bar")) - expect(bundled_app.join("foo", "bar")).to exist + Bundler.mkdir_p(bundled_app("foo", "bar")) + expect(bundled_app("foo", "bar")).to exist end end diff --git a/spec/bundler/bundler/compact_index_client/parser_spec.rb b/spec/bundler/bundler/compact_index_client/parser_spec.rb new file mode 100644 index 0000000000..45a08fd9ff --- /dev/null +++ b/spec/bundler/bundler/compact_index_client/parser_spec.rb @@ -0,0 +1,259 @@ +# frozen_string_literal: true + +require "bundler/compact_index_client" +require "bundler/compact_index_client/parser" + +TestCompactIndexClient = Struct.new(:names, :versions, :info_data) do + # Requiring the checksum to match the input data helps ensure + # that we are parsing the correct checksum from the versions file + def info(name, checksum) + info_data.dig(name, checksum) + end + + def set_info_data(name, value) + info_data[name] = value + end +end + +RSpec.describe Bundler::CompactIndexClient::Parser do + subject(:parser) { described_class.new(compact_index) } + + let(:compact_index) { TestCompactIndexClient.new(names, versions, info_data) } + let(:names) { "a\nb\nc\n" } + let(:versions) { <<~VERSIONS.dup } + created_at: 2024-05-01T00:00:04Z + --- + a 1.0.0,1.0.1,1.1.0 aaa111 + b 2.0.0,2.0.0-java bbb222 + c 3.0.0,3.0.3,3.3.3 ccc333 + c -3.0.3 ccc333yanked + VERSIONS + let(:info_data) do + { + "a" => { "aaa111" => a_info }, + "b" => { "bbb222" => b_info }, + "c" => { "ccc333yanked" => c_info }, + } + end + let(:a_info) { <<~INFO.dup } + --- + 1.0.0 |checksum:aaa1,ruby:>= 3.0.0,rubygems:>= 3.2.3 + 1.0.1 |checksum:aaa2,ruby:>= 3.0.0,rubygems:>= 3.2.3 + 1.1.0 |checksum:aaa3,ruby:>= 3.0.0,rubygems:>= 3.2.3 + INFO + let(:b_info) { <<~INFO } + 2.0.0 a:~> 1.0&<= 3.0|checksum:bbb1 + 2.0.0-java a:~> 1.0&<= 3.0|checksum:bbb2 + INFO + let(:c_info) { <<~INFO } + 3.0.0 a:= 1.0.0,b:~> 2.0|checksum:ccc1,ruby:>= 2.7.0,rubygems:>= 3.0.0 + 3.3.3 a:>= 1.1.0,b:~> 2.0|checksum:ccc3,ruby:>= 3.0.0,rubygems:>= 3.2.3 + INFO + + describe "#available?" do + it "returns true versions are available" do + expect(parser).to be_available + end + + it "returns true when versions has only one gem" do + compact_index.versions = +"a 1.0.0 aaa1\n" + expect(parser).to be_available + end + + it "returns true when versions has a gem and a header" do + compact_index.versions = +"---\na 1.0.0 aaa1\n" + expect(parser).to be_available + end + + it "returns true when versions has a gem and a header with header data" do + compact_index.versions = +"created_at: 2024-05-01T00:00:04Z\n---\na 1.0.0 aaa1\n" + expect(parser).to be_available + end + + it "returns false when versions has only the header" do + compact_index.versions = +"---\n" + expect(parser).not_to be_available + end + + it "returns false when versions has only the header with header data" do + compact_index.versions = +"created_at: 2024-05-01T00:00:04Z\n---\n" + expect(parser).not_to be_available + end + + it "returns false when versions index is not available" do + compact_index.versions = nil + expect(parser).not_to be_available + end + + it "returns false when versions is empty" do + compact_index.versions = +"" + expect(parser).not_to be_available + end + + it "returns false when versions ends improperly without a newline" do + compact_index.versions = "a 1.0.0 aaa1" + expect(parser).not_to be_available + end + end + + describe "#names" do + it "returns the names" do + expect(parser.names).to eq(%w[a b c]) + end + + it "returns an empty array when names is empty" do + compact_index.names = "" + expect(parser.names).to eq([]) + end + + it "returns an empty array when names is not readable" do + compact_index.names = nil + expect(parser.names).to eq([]) + end + end + + describe "#versions" do + it "returns the versions" do + expect(parser.versions).to eq( + "a" => [ + ["a", "1.0.0"], + ["a", "1.0.1"], + ["a", "1.1.0"], + ], + "b" => [ + ["b", "2.0.0"], + ["b", "2.0.0", "java"], + ], + "c" => [ + ["c", "3.0.0"], + ["c", "3.3.3"], + ], + ) + end + + it "returns an empty hash when versions is empty" do + compact_index.versions = "" + expect(parser.versions).to eq({}) + end + + it "returns an empty hash when versions is not readable" do + compact_index.versions = nil + expect(parser.versions).to eq({}) + end + end + + describe "#info" do + it "returns the info for example gem 'a' which has no deps" do + expect(parser.info("a")).to eq( + [ + [ + "a", + "1.0.0", + nil, + [], + [ + ["checksum", ["aaa1"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + [ + "a", + "1.0.1", + nil, + [], + [ + ["checksum", ["aaa2"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + [ + "a", + "1.1.0", + nil, + [], + [ + ["checksum", ["aaa3"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + ] + ) + end + + it "returns the info for example gem 'b' which has platform and compound deps" do + expect(parser.info("b")).to eq( + [ + [ + "b", + "2.0.0", + nil, + [ + ["a", ["~> 1.0", "<= 3.0"]], + ], + [ + ["checksum", ["bbb1"]], + ], + ], + [ + "b", + "2.0.0", + "java", + [ + ["a", ["~> 1.0", "<= 3.0"]], + ], + [ + ["checksum", ["bbb2"]], + ], + ], + ] + ) + end + + it "returns the info for example gem 'c' which has deps and yanked version (requires use of correct info checksum)" do + expect(parser.info("c")).to eq( + [ + [ + "c", + "3.0.0", + nil, + [ + ["a", ["= 1.0.0"]], + ["b", ["~> 2.0"]], + ], + [ + ["checksum", ["ccc1"]], + ["ruby", [">= 2.7.0"]], + ["rubygems", [">= 3.0.0"]], + ], + ], + [ + "c", + "3.3.3", + nil, + [ + ["a", [">= 1.1.0"]], + ["b", ["~> 2.0"]], + ], + [ + ["checksum", ["ccc3"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + ] + ) + end + + it "returns an empty array when the info is empty" do + compact_index.set_info_data("a", {}) + expect(parser.info("a")).to eq([]) + end + + it "returns an empty array when the info is not readable" do + expect(parser.info("d")).to eq([]) + end + end +end diff --git a/spec/bundler/bundler/compact_index_client/updater_spec.rb b/spec/bundler/bundler/compact_index_client/updater_spec.rb index 6eed88ca9e..87a73d993f 100644 --- a/spec/bundler/bundler/compact_index_client/updater_spec.rb +++ b/spec/bundler/bundler/compact_index_client/updater_spec.rb @@ -119,23 +119,7 @@ RSpec.describe Bundler::CompactIndexClient::Updater do context "without an etag file" do let(:headers) do - { - "Range" => "bytes=2-", - # This MD5 feature should be deleted after sufficient time has passed since release. - # From then on, requests that still don't have a saved etag will be made without this header. - "If-None-Match" => %("#{Digest::MD5.hexdigest(local_body)}"), - } - end - - it "saves only the etag_path if generated etag matches" do - expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) - allow(response).to receive(:is_a?).with(Gem::Net::HTTPPartialContent) { false } - allow(response).to receive(:is_a?).with(Gem::Net::HTTPNotModified) { true } - - updater.update(remote_path, local_path, etag_path) - - expect(local_path.read).to eq("abc") - expect(%("#{etag_path.read}")).to eq(headers["If-None-Match"]) + { "Range" => "bytes=2-" } end it "appends the file" do diff --git a/spec/bundler/bundler/env_spec.rb b/spec/bundler/bundler/env_spec.rb index 7997cb0c40..c15aa9af67 100644 --- a/spec/bundler/bundler/env_spec.rb +++ b/spec/bundler/bundler/env_spec.rb @@ -150,7 +150,7 @@ RSpec.describe Bundler::Env do before do gemfile("source \"#{file_uri_for(gem_repo1)}\"; gemspec") - File.open(bundled_app.join("foo.gemspec"), "wb") do |f| + File.open(bundled_app("foo.gemspec"), "wb") do |f| f.write(gemspec) end diff --git a/spec/bundler/bundler/fetcher/compact_index_spec.rb b/spec/bundler/bundler/fetcher/compact_index_spec.rb index a988171f34..aa536673d9 100644 --- a/spec/bundler/bundler/fetcher/compact_index_spec.rb +++ b/spec/bundler/bundler/fetcher/compact_index_spec.rb @@ -4,14 +4,18 @@ require "bundler/compact_index_client" RSpec.describe Bundler::Fetcher::CompactIndex do - let(:downloader) { double(:downloader) } + let(:response) { double(:response) } + let(:downloader) { double(:downloader, fetch: response) } let(:display_uri) { Gem::URI("http://sampleuri.com") } let(:remote) { double(:remote, cache_slug: "lsjdf", uri: display_uri) } let(:gem_remote_fetcher) { nil } let(:compact_index) { described_class.new(downloader, remote, display_uri, gem_remote_fetcher) } + let(:compact_index_client) { double(:compact_index_client, available?: true, info: [["lskdjf", "1", nil, [], []]]) } before do + allow(response).to receive(:is_a?).with(Gem::Net::HTTPNotModified).and_return(true) allow(compact_index).to receive(:log_specs) {} + allow(compact_index).to receive(:compact_index_client).and_return(compact_index_client) end describe "#specs_for_names" do @@ -32,11 +36,6 @@ RSpec.describe Bundler::Fetcher::CompactIndex do end describe "#available?" do - before do - allow(compact_index).to receive(:compact_index_client). - and_return(double(:compact_index_client, update_and_parse_checksums!: true)) - end - it "returns true" do expect(compact_index).to be_available end diff --git a/spec/bundler/bundler/gem_helper_spec.rb b/spec/bundler/bundler/gem_helper_spec.rb index 940e5df9de..187dab0e7c 100644 --- a/spec/bundler/bundler/gem_helper_spec.rb +++ b/spec/bundler/bundler/gem_helper_spec.rb @@ -11,7 +11,7 @@ RSpec.describe Bundler::GemHelper do before(:each) do global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__LINTER" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__CHANGELOG" => "false" - sys_exec("git config --global init.defaultBranch main") + git("config --global init.defaultBranch main") bundle "gem #{app_name}" prepare_gemspec(app_gemspec_path) end @@ -253,11 +253,11 @@ RSpec.describe Bundler::GemHelper do end before do - sys_exec("git init", dir: app_path) - sys_exec("git config user.email \"you@example.com\"", dir: app_path) - sys_exec("git config user.name \"name\"", dir: app_path) - sys_exec("git config commit.gpgsign false", dir: app_path) - sys_exec("git config push.default simple", dir: app_path) + git("init", app_path) + git("config user.email \"you@example.com\"", app_path) + git("config user.name \"name\"", app_path) + git("config commit.gpgsign false", app_path) + git("config push.default simple", app_path) # silence messages allow(Bundler.ui).to receive(:confirm) @@ -271,13 +271,13 @@ RSpec.describe Bundler::GemHelper do end it "when there are uncommitted files" do - sys_exec("git add .", dir: app_path) + git("add .", app_path) expect { Rake.application["release"].invoke }. to raise_error("There are files that need to be committed first.") end it "when there is no git remote" do - sys_exec("git commit -a -m \"initial commit\"", dir: app_path) + git("commit -a -m \"initial commit\"", app_path) expect { Rake.application["release"].invoke }.to raise_error(RuntimeError) end end @@ -286,8 +286,8 @@ RSpec.describe Bundler::GemHelper do let(:repo) { build_git("foo", bare: true) } before do - sys_exec("git remote add origin #{file_uri_for(repo.path)}", dir: app_path) - sys_exec('git commit -a -m "initial commit"', dir: app_path) + git("remote add origin #{file_uri_for(repo.path)}", app_path) + git('commit -a -m "initial commit"', app_path) end context "on releasing" do @@ -296,7 +296,7 @@ RSpec.describe Bundler::GemHelper do mock_confirm_message "Tagged v#{app_version}." mock_confirm_message "Pushed git commits and release tag." - sys_exec("git push -u origin main", dir: app_path) + git("push -u origin main", app_path) end it "calls rubygem_push with proper arguments" do @@ -314,8 +314,8 @@ RSpec.describe Bundler::GemHelper do it "also works when releasing from an ambiguous reference" do # Create a branch with the same name as the tag - sys_exec("git checkout -b v#{app_version}", dir: app_path) - sys_exec("git push -u origin v#{app_version}", dir: app_path) + git("checkout -b v#{app_version}", app_path) + git("push -u origin v#{app_version}", app_path) expect(subject).to receive(:rubygem_push).with(app_gem_path.to_s) @@ -323,7 +323,7 @@ RSpec.describe Bundler::GemHelper do end it "also works with releasing from a branch not yet pushed" do - sys_exec("git checkout -b module_function", dir: app_path) + git("checkout -b module_function", app_path) expect(subject).to receive(:rubygem_push).with(app_gem_path.to_s) @@ -337,7 +337,7 @@ RSpec.describe Bundler::GemHelper do mock_build_message app_name, app_version mock_confirm_message "Pushed git commits and release tag." - sys_exec("git push -u origin main", dir: app_path) + git("push -u origin main", app_path) expect(subject).to receive(:rubygem_push).with(app_gem_path.to_s) end @@ -353,7 +353,7 @@ RSpec.describe Bundler::GemHelper do mock_confirm_message "Tag v#{app_version} has already been created." expect(subject).to receive(:rubygem_push).with(app_gem_path.to_s) - sys_exec("git tag -a -m \"Version #{app_version}\" v#{app_version}", dir: app_path) + git("tag -a -m \"Version #{app_version}\" v#{app_version}", app_path) Rake.application["release"].invoke end @@ -374,10 +374,10 @@ RSpec.describe Bundler::GemHelper do end before do - sys_exec("git init", dir: app_path) - sys_exec("git config user.email \"you@example.com\"", dir: app_path) - sys_exec("git config user.name \"name\"", dir: app_path) - sys_exec("git config push.gpgsign simple", dir: app_path) + git("init", app_path) + git("config user.email \"you@example.com\"", app_path) + git("config user.name \"name\"", app_path) + git("config push.gpgsign simple", app_path) # silence messages allow(Bundler.ui).to receive(:confirm) diff --git a/spec/bundler/bundler/installer/gem_installer_spec.rb b/spec/bundler/bundler/installer/gem_installer_spec.rb index 4b6a07f344..ea506c36c8 100644 --- a/spec/bundler/bundler/installer/gem_installer_spec.rb +++ b/spec/bundler/bundler/installer/gem_installer_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Bundler::GemInstaller do it "invokes install method with empty build_args" do allow(spec_source).to receive(:install).with( spec, - { force: false, ensure_builtin_gems_cached: false, build_args: [], previous_spec: nil } + { force: false, build_args: [], previous_spec: nil } ) subject.install_from_spec end @@ -28,7 +28,7 @@ RSpec.describe Bundler::GemInstaller do allow(Bundler.settings).to receive(:[]).with("build.dummy").and_return("--with-dummy-config=dummy") expect(spec_source).to receive(:install).with( spec, - { force: false, ensure_builtin_gems_cached: false, build_args: ["--with-dummy-config=dummy"], previous_spec: nil } + { force: false, build_args: ["--with-dummy-config=dummy"], previous_spec: nil } ) subject.install_from_spec end @@ -42,7 +42,7 @@ RSpec.describe Bundler::GemInstaller do allow(Bundler.settings).to receive(:[]).with("build.dummy").and_return("--with-dummy-config=dummy --with-another-dummy-config") expect(spec_source).to receive(:install).with( spec, - { force: false, ensure_builtin_gems_cached: false, build_args: ["--with-dummy-config=dummy", "--with-another-dummy-config"], previous_spec: nil } + { force: false, build_args: ["--with-dummy-config=dummy", "--with-another-dummy-config"], previous_spec: nil } ) subject.install_from_spec end diff --git a/spec/bundler/bundler/plugin_spec.rb b/spec/bundler/bundler/plugin_spec.rb index f41b4eff3a..3ac7b251a3 100644 --- a/spec/bundler/bundler/plugin_spec.rb +++ b/spec/bundler/bundler/plugin_spec.rb @@ -247,7 +247,7 @@ RSpec.describe Bundler::Plugin do end it "returns plugin dir in app .bundle path" do - expect(subject.root).to eq(bundled_app.join(".bundle/plugin")) + expect(subject.root).to eq(bundled_app(".bundle/plugin")) end end @@ -257,7 +257,7 @@ RSpec.describe Bundler::Plugin do end it "returns plugin dir in global bundle path" do - expect(subject.root).to eq(home.join(".bundle/plugin")) + expect(subject.root).to eq(home(".bundle/plugin")) end end end diff --git a/spec/bundler/bundler/settings_spec.rb b/spec/bundler/bundler/settings_spec.rb index 634e0faf91..768372c608 100644 --- a/spec/bundler/bundler/settings_spec.rb +++ b/spec/bundler/bundler/settings_spec.rb @@ -6,12 +6,18 @@ RSpec.describe Bundler::Settings do subject(:settings) { described_class.new(bundled_app) } describe "#set_local" do - context "when the local config file is not found" do + context "root is nil" do subject(:settings) { described_class.new(nil) } - it "raises a GemfileNotFound error with explanation" do - expect { subject.set_local("foo", "bar") }. - to raise_error(Bundler::GemfileNotFound, "Could not locate Gemfile") + before do + allow(Pathname).to receive(:new).and_call_original + allow(Pathname).to receive(:new).with(".bundle").and_return home(".bundle") + end + + it "works" do + subject.set_local("foo", "bar") + + expect(subject["foo"]).to eq("bar") end end end diff --git a/spec/bundler/bundler/source/git/git_proxy_spec.rb b/spec/bundler/bundler/source/git/git_proxy_spec.rb index 1450316d59..f7c883eed4 100644 --- a/spec/bundler/bundler/source/git/git_proxy_spec.rb +++ b/spec/bundler/bundler/source/git/git_proxy_spec.rb @@ -197,4 +197,18 @@ RSpec.describe Bundler::Source::Git::GitProxy do expect(Pathname.new(bundled_app("canary"))).not_to exist end + + context "URI is HTTP" do + let(:uri) { "http://github.com/rubygems/rubygems.git" } + let(:without_depth_arguments) { ["clone", "--bare", "--no-hardlinks", "--quiet", "--no-tags", "--single-branch"] } + let(:fail_clone_result) { double(Process::Status, success?: false) } + + it "retries without --depth when git url is http and fails" do + allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.14.0") + allow(git_proxy).to receive(:capture).with([*base_clone_args, "--", uri, path.to_s], nil).and_return(["", "dumb http transport does not support shallow capabilities", fail_clone_result]) + expect(git_proxy).to receive(:capture).with([*without_depth_arguments, "--", uri, path.to_s], nil).and_return(["", "", clone_result]) + + subject.checkout + end + end end diff --git a/spec/bundler/cache/gems_spec.rb b/spec/bundler/cache/gems_spec.rb index abbc2c3cf2..3c734b79c3 100644 --- a/spec/bundler/cache/gems_spec.rb +++ b/spec/bundler/cache/gems_spec.rb @@ -93,68 +93,80 @@ RSpec.describe "bundle cache" do let(:default_json_version) { ruby "gem 'json'; require 'json'; puts JSON::VERSION" } before :each do - build_repo2 do - build_gem "json", default_json_version - end - build_gem "json", default_json_version, to_system: true, default: true end - it "uses builtin gems when installing to system gems" do - bundle "config set path.system true" - install_gemfile %(source "#{file_uri_for(gem_repo1)}"; gem 'json', '#{default_json_version}'), verbose: true - expect(out).to include("Using json #{default_json_version}") - end + context "when a remote gem is available for caching" do + before do + build_repo2 do + build_gem "json", default_json_version + end + end - it "caches remote and builtin gems" do - install_gemfile <<-G - source "#{file_uri_for(gem_repo2)}" - gem 'json', '#{default_json_version}' - gem 'rack', '1.0.0' - G + it "uses remote gems when installing to system gems" do + bundle "config set path.system true" + install_gemfile %(source "#{file_uri_for(gem_repo2)}"; gem 'json', '#{default_json_version}'), verbose: true + expect(out).to include("Installing json #{default_json_version}") + end - bundle :cache - expect(bundled_app("vendor/cache/rack-1.0.0.gem")).to exist - expect(bundled_app("vendor/cache/json-#{default_json_version}.gem")).to exist - end + it "caches remote and builtin gems" do + install_gemfile <<-G + source "#{file_uri_for(gem_repo2)}" + gem 'json', '#{default_json_version}' + gem 'rack', '1.0.0' + G - it "caches builtin gems when cache_all_platforms is set" do - gemfile <<-G - source "#{file_uri_for(gem_repo2)}" - gem "json" - G + bundle :cache + expect(bundled_app("vendor/cache/rack-1.0.0.gem")).to exist + expect(bundled_app("vendor/cache/json-#{default_json_version}.gem")).to exist + end - bundle "config set cache_all_platforms true" + it "caches builtin gems when cache_all_platforms is set" do + gemfile <<-G + source "#{file_uri_for(gem_repo2)}" + gem "json" + G - bundle :cache - expect(bundled_app("vendor/cache/json-#{default_json_version}.gem")).to exist - end + bundle "config set cache_all_platforms true" - it "doesn't make remote request after caching the gem" do - build_gem "builtin_gem_2", "1.0.2", path: bundled_app("vendor/cache") do |s| - s.summary = "This builtin_gem is bundled with Ruby" + bundle :cache + expect(bundled_app("vendor/cache/json-#{default_json_version}.gem")).to exist end - install_gemfile <<-G - source "#{file_uri_for(gem_repo2)}" - gem 'builtin_gem_2', '1.0.2' - G + it "doesn't make remote request after caching the gem" do + build_gem "builtin_gem_2", "1.0.2", path: bundled_app("vendor/cache") do |s| + s.summary = "This builtin_gem is bundled with Ruby" + end - bundle "install --local" - expect(the_bundle).to include_gems("builtin_gem_2 1.0.2") + install_gemfile <<-G + source "#{file_uri_for(gem_repo2)}" + gem 'builtin_gem_2', '1.0.2' + G + + bundle "install --local" + expect(the_bundle).to include_gems("builtin_gem_2 1.0.2") + end end - it "errors if the builtin gem isn't available to cache" do - bundle "config set path.system true" + context "when a remote gem is not available for caching" do + it "uses builtin gems when installing to system gems" do + bundle "config set path.system true" + install_gemfile %(source "#{file_uri_for(gem_repo1)}"; gem 'json', '#{default_json_version}'), verbose: true + expect(out).to include("Using json #{default_json_version}") + end - install_gemfile <<-G - source "#{file_uri_for(gem_repo1)}" - gem 'json', '#{default_json_version}' - G + it "errors when explicitly caching" do + bundle "config set path.system true" - bundle :cache, raise_on_error: false - expect(exitstatus).to_not eq(0) - expect(err).to include("json-#{default_json_version} is built in to Ruby, and can't be cached") + install_gemfile <<-G + source "#{file_uri_for(gem_repo1)}" + gem 'json', '#{default_json_version}' + G + + bundle :cache, raise_on_error: false + expect(exitstatus).to_not eq(0) + expect(err).to include("json-#{default_json_version} is built in to Ruby, and can't be cached") + end end end diff --git a/spec/bundler/cache/git_spec.rb b/spec/bundler/cache/git_spec.rb index 4b3cd4d2eb..51897ebe20 100644 --- a/spec/bundler/cache/git_spec.rb +++ b/spec/bundler/cache/git_spec.rb @@ -164,8 +164,8 @@ RSpec.describe "bundle cache with git" do s.add_dependency "submodule" end - sys_exec "git submodule add #{lib_path("submodule-1.0")} submodule-1.0", dir: lib_path("has_submodule-1.0") - sys_exec "git commit -m \"submodulator\"", dir: lib_path("has_submodule-1.0") + git "submodule add #{lib_path("submodule-1.0")} submodule-1.0", lib_path("has_submodule-1.0") + git "commit -m \"submodulator\"", lib_path("has_submodule-1.0") install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" diff --git a/spec/bundler/commands/cache_spec.rb b/spec/bundler/commands/cache_spec.rb index 70e2c84961..37d8b3ac1a 100644 --- a/spec/bundler/commands/cache_spec.rb +++ b/spec/bundler/commands/cache_spec.rb @@ -386,6 +386,66 @@ RSpec.describe "bundle install with gem sources" do expect(the_bundle).to include_gems "rack 1.0.0" end + it "uses cached gems for secondary sources when cache_all_platforms configured" do + build_repo4 do + build_gem "foo", "1.0.0" do |s| + s.platform = "x86_64-linux" + end + + build_gem "foo", "1.0.0" do |s| + s.platform = "arm64-darwin" + end + end + + gemfile <<~G + source "https://gems.repo2" + + source "https://gems.repo4" do + gem "foo" + end + G + + lockfile <<~L + GEM + remote: https://gems.repo2/ + specs: + + GEM + remote: https://gems.repo4/ + specs: + foo (1.0.0-x86_64-linux) + foo (1.0.0-arm64-darwin) + + PLATFORMS + arm64-darwin + ruby + x86_64-linux + + DEPENDENCIES + foo + + BUNDLED WITH + #{Bundler::VERSION} + L + + simulate_platform "x86_64-linux" do + bundle "config set cache_all_platforms true" + bundle "config set path vendor/bundle" + bundle :cache, artifice: "compact_index", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + + build_repo4 do + # simulate removal of all remote gems + end + + # delete compact index cache + FileUtils.rm_rf home(".bundle/cache/compact_index") + + bundle "install", artifice: "compact_index", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + + expect(the_bundle).to include_gems "foo 1.0.0 x86_64-linux" + end + end + it "does not reinstall already-installed gems" do install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" diff --git a/spec/bundler/commands/check_spec.rb b/spec/bundler/commands/check_spec.rb index 02f9bb5b7a..4bad16a55e 100644 --- a/spec/bundler/commands/check_spec.rb +++ b/spec/bundler/commands/check_spec.rb @@ -447,7 +447,7 @@ RSpec.describe "bundle check" do build_gem "dex-dispatch-engine" end - build_lib("bundle-check-issue", path: tmp.join("bundle-check-issue")) do |s| + build_lib("bundle-check-issue", path: tmp("bundle-check-issue")) do |s| s.write "Gemfile", <<-G source "https://localgemserver.test" @@ -461,14 +461,14 @@ RSpec.describe "bundle check" do s.add_dependency "awesome_print" end - bundle "install", artifice: "compact_index_extra", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }, dir: tmp.join("bundle-check-issue") + bundle "install", artifice: "compact_index_extra", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }, dir: tmp("bundle-check-issue") end it "does not corrupt lockfile when changing version" do - version_file = tmp.join("bundle-check-issue/bundle-check-issue.gemspec") + version_file = tmp("bundle-check-issue/bundle-check-issue.gemspec") File.write(version_file, File.read(version_file).gsub(/s\.version = .+/, "s.version = '9999'")) - bundle "check --verbose", dir: tmp.join("bundle-check-issue") + bundle "check --verbose", dir: tmp("bundle-check-issue") checksums = checksums_section_when_existing do |c| c.checksum gem_repo4, "awesome_print", "1.0" @@ -476,7 +476,7 @@ RSpec.describe "bundle check" do c.checksum gem_repo2, "dex-dispatch-engine", "1.0" end - expect(File.read(tmp.join("bundle-check-issue/Gemfile.lock"))).to eq <<~L + expect(File.read(tmp("bundle-check-issue/Gemfile.lock"))).to eq <<~L PATH remote: . specs: diff --git a/spec/bundler/commands/config_spec.rb b/spec/bundler/commands/config_spec.rb index 547fd2d869..d6a30eae5b 100644 --- a/spec/bundler/commands/config_spec.rb +++ b/spec/bundler/commands/config_spec.rb @@ -79,6 +79,14 @@ RSpec.describe ".bundle/config" do expect(home(".bundle/config")).to exist end + it "does not list global settings as local" do + bundle "config set --global foo bar" + bundle "config list", dir: home + + expect(out).to include("for the current user") + expect(out).not_to include("for your local app") + end + it "works with an absolute path" do ENV["BUNDLE_APP_CONFIG"] = tmp("foo/bar").to_s bundle "config set --local path vendor/bundle" @@ -350,6 +358,12 @@ end E expect(out).to eq("http://gems.example.org/ => http://gem-mirror.example.org/") end + + it "allows configuring fallback timeout for each mirror, and does not duplicate configs", rubygems: ">= 3.5.12" do + bundle "config set --global mirror.https://rubygems.org.fallback_timeout 1" + bundle "config set --global mirror.https://rubygems.org.fallback_timeout 2" + expect(File.read(home(".bundle/config"))).to include("BUNDLE_MIRROR").once + end end describe "quoting" do @@ -439,7 +453,7 @@ E it "does not make bundler crash and ignores the configuration" do bundle "config list --parseable" - expect(out).to eq("#mirror.https://rails-assets.org/=http://localhost:9292") + expect(out).to be_empty expect(err).to be_empty ruby(<<~RUBY) diff --git a/spec/bundler/commands/exec_spec.rb b/spec/bundler/commands/exec_spec.rb index d59b690d2f..9f5f12739a 100644 --- a/spec/bundler/commands/exec_spec.rb +++ b/spec/bundler/commands/exec_spec.rb @@ -885,7 +885,7 @@ RSpec.describe "bundle exec" do let(:exit_code) { Bundler::GemNotFound.new.status_code } let(:expected) { "" } let(:expected_err) { <<-EOS.strip } -Could not find gem 'rack (= 2)' in cached gems or installed locally. +Could not find gem 'rack (= 2)' in locally installed gems. The source contains the following gems matching 'rack': * rack-0.9.1 @@ -915,7 +915,7 @@ Run `bundle install` to install missing gems. let(:exit_code) { Bundler::GemNotFound.new.status_code } let(:expected) { "" } let(:expected_err) { <<-EOS.strip } -Could not find gem 'rack (= 2)' in cached gems or installed locally. +Could not find gem 'rack (= 2)' in locally installed gems. The source contains the following gems matching 'rack': * rack-1.0.0 diff --git a/spec/bundler/commands/info_spec.rb b/spec/bundler/commands/info_spec.rb index a5a09bc147..7ddc5c2363 100644 --- a/spec/bundler/commands/info_spec.rb +++ b/spec/bundler/commands/info_spec.rb @@ -215,7 +215,7 @@ RSpec.describe "bundle info" do G bundle "info rac" - expect(out).to match(/\A1 : rack\n2 : rack-obama\n0 : - exit -(\n>.*)?\z/) + expect(out).to match(/\A1 : rack\n2 : rack-obama\n0 : - exit -(\n>|\z)/) end end diff --git a/spec/bundler/commands/init_spec.rb b/spec/bundler/commands/init_spec.rb index 0a1336572a..564a4bdc2d 100644 --- a/spec/bundler/commands/init_spec.rb +++ b/spec/bundler/commands/init_spec.rb @@ -79,7 +79,7 @@ RSpec.describe "bundle init" do end context "given --gemspec option" do - let(:spec_file) { tmp.join("test.gemspec") } + let(:spec_file) { tmp("test.gemspec") } it "should generate from an existing gemspec" do File.open(spec_file, "w") do |file| @@ -160,7 +160,7 @@ RSpec.describe "bundle init" do end context "given --gemspec option" do - let(:spec_file) { tmp.join("test.gemspec") } + let(:spec_file) { tmp("test.gemspec") } before do File.open(spec_file, "w") do |file| diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index 1e57414377..fed78fc4e0 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -461,8 +461,8 @@ RSpec.describe "bundle install with gem sources" do end it "includes the gem without warning if two gemspecs add it with the same requirement" do - gem1 = tmp.join("my-gem-1") - gem2 = tmp.join("my-gem-2") + gem1 = tmp("my-gem-1") + gem2 = tmp("my-gem-2") build_lib "my-gem", path: gem1 do |s| s.add_development_dependency "rubocop", "~> 1.36.0" @@ -1407,4 +1407,33 @@ RSpec.describe "bundle install with gem sources" do expect(bundled_app(".bundle/config")).not_to exist end end + + context "when bundler installation is corrupt" do + before do + system_gems "bundler-9.99.8" + + replace_version_file("9.99.9", dir: system_gem_path("gems/bundler-9.99.8")) + end + + it "shows a proper error" do + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo1)}/ + specs: + + PLATFORMS + #{lockfile_platforms} + + DEPENDENCIES + + BUNDLED WITH + 9.99.8 + L + + install_gemfile "source \"#{file_uri_for(gem_repo1)}\"", env: { "BUNDLER_VERSION" => "9.99.8" }, raise_on_error: false + + expect(err).not_to include("ERROR REPORT TEMPLATE") + expect(err).to include("The running version of Bundler (9.99.9) does not match the version of the specification installed for it (9.99.8)") + end + end end diff --git a/spec/bundler/commands/list_spec.rb b/spec/bundler/commands/list_spec.rb index 5ac2077d81..660935f06f 100644 --- a/spec/bundler/commands/list_spec.rb +++ b/spec/bundler/commands/list_spec.rb @@ -126,7 +126,7 @@ RSpec.describe "bundle list" do build_git "git_test", "1.0.0", path: lib_path("git_test") - build_lib("gemspec_test", path: tmp.join("gemspec_test")) do |s| + build_lib("gemspec_test", path: tmp("gemspec_test")) do |s| s.add_dependency "bar", "=1.0.0" end @@ -135,7 +135,7 @@ RSpec.describe "bundle list" do gem "rack" gem "rails" gem "git_test", :git => "#{lib_path("git_test")}" - gemspec :path => "#{tmp.join("gemspec_test")}" + gemspec :path => "#{tmp("gemspec_test")}" G end diff --git a/spec/bundler/commands/lock_spec.rb b/spec/bundler/commands/lock_spec.rb index c6bb0f58af..b0d6fa9134 100644 --- a/spec/bundler/commands/lock_spec.rb +++ b/spec/bundler/commands/lock_spec.rb @@ -138,7 +138,7 @@ RSpec.describe "bundle lock" do it "does not fetch remote specs when using the --local option" do bundle "lock --update --local", raise_on_error: false - expect(err).to match(/cached gems or installed locally/) + expect(err).to match(/locally installed gems/) end it "does not fetch remote checksums with --local" do @@ -1349,7 +1349,7 @@ RSpec.describe "bundle lock" do Because rails >= 7.0.4 depends on railties = 7.0.4 and rails < 7.0.4 depends on railties = 7.0.3.1, railties = 7.0.3.1 OR = 7.0.4 is required. - So, because railties = 7.0.3.1 OR = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally, + So, because railties = 7.0.3.1 OR = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally, version solving has failed. ERR end @@ -1460,7 +1460,7 @@ RSpec.describe "bundle lock" do Thus, rails >= 7.0.2.3, < 7.0.4 cannot be used. And because rails >= 7.0.4 depends on activemodel = 7.0.4, rails >= 7.0.2.3 requires activemodel = 7.0.4. - So, because activemodel = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally + So, because activemodel = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally and Gemfile depends on rails >= 7.0.2.3, version solving has failed. ERR diff --git a/spec/bundler/commands/newgem_spec.rb b/spec/bundler/commands/newgem_spec.rb index 199340b131..9ff10158a4 100644 --- a/spec/bundler/commands/newgem_spec.rb +++ b/spec/bundler/commands/newgem_spec.rb @@ -33,25 +33,28 @@ RSpec.describe "bundle gem" do let(:minitest_test_class_name) { "class TestMygem < Minitest::Test" } before do - sys_exec("git config --global user.name 'Bundler User'") - sys_exec("git config --global user.email user@example.com") - sys_exec("git config --global github.user bundleuser") + git("config --global user.name 'Bundler User'") + git("config --global user.email user@example.com") + git("config --global github.user bundleuser") + + global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__LINTER" => "false", + "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__CHANGELOG" => "false" end describe "git repo initialization" do - it "generates a gem skeleton with a .git folder", :readline do + it "generates a gem skeleton with a .git folder" do bundle "gem #{gem_name}" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).to exist end - it "generates a gem skeleton with a .git folder when passing --git", :readline do + it "generates a gem skeleton with a .git folder when passing --git" do bundle "gem #{gem_name} --git" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).to exist end - it "generates a gem skeleton without a .git folder when passing --no-git", :readline do + it "generates a gem skeleton without a .git folder when passing --no-git" do bundle "gem #{gem_name} --no-git" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).not_to exist @@ -62,7 +65,9 @@ RSpec.describe "bundle gem" do Dir.mkdir(bundled_app("path with spaces")) end - it "properly initializes git repo", :readline do + it "properly initializes git repo" do + skip "path with spaces needs special handling on Windows" if Gem.win_platform? + bundle "gem #{gem_name}", dir: bundled_app("path with spaces") expect(bundled_app("path with spaces/#{gem_name}/.git")).to exist end @@ -104,7 +109,7 @@ RSpec.describe "bundle gem" do end it "generates the README with a section for the Code of Conduct, respecting the configured git default branch", git: ">= 2.28.0" do - sys_exec("git config --global init.defaultBranch main") + git("config --global init.defaultBranch main") bundle "gem #{gem_name} --coc" expect(bundled_app("#{gem_name}/README.md").read).to include("## Code of Conduct") @@ -131,7 +136,7 @@ RSpec.describe "bundle gem" do before do bundle "gem #{gem_name} --changelog" end - it "generates a gem skeleton with a CHANGELOG", :readline do + it "generates a gem skeleton with a CHANGELOG" do gem_skeleton_assertions expect(bundled_app("#{gem_name}/CHANGELOG.md")).to exist end @@ -141,7 +146,7 @@ RSpec.describe "bundle gem" do before do bundle "gem #{gem_name} --no-changelog" end - it "generates a gem skeleton without a CHANGELOG", :readline do + it "generates a gem skeleton without a CHANGELOG" do gem_skeleton_assertions expect(bundled_app("#{gem_name}/CHANGELOG.md")).to_not exist end @@ -150,6 +155,7 @@ RSpec.describe "bundle gem" do shared_examples_for "--rubocop flag" do context "is deprecated", bundler: "< 3" do before do + global_config "BUNDLE_GEM__LINTER" => nil bundle "gem #{gem_name} --rubocop" end @@ -303,49 +309,49 @@ RSpec.describe "bundle gem" do end end - it "has no rubocop offenses when using --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=minitest, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=minitest, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=minitest --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=rspec, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=rspec, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=rspec --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=test-unit, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=test-unit, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=test-unit --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no standard offenses when using --linter=standard flag", :readline do + it "has no standard offenses when using --linter=standard flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --linter=standard" bundle_exec_standardrb expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -354,7 +360,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=minitest, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=minitest, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -363,7 +369,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=rspec, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=rspec, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -372,7 +378,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=test-unit, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=test-unit, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -399,7 +405,7 @@ RSpec.describe "bundle gem" do end end - context "README.md", :readline do + context "README.md" do context "git config github.user present" do before do bundle "gem #{gem_name}" @@ -413,7 +419,7 @@ RSpec.describe "bundle gem" do context "git config github.user is absent" do before do - sys_exec("git config --global --unset github.user") + git("config --global --unset github.user") bundle "gem #{gem_name}" end @@ -424,12 +430,12 @@ RSpec.describe "bundle gem" do end end - it "creates a new git repository", :readline do + it "creates a new git repository" do bundle "gem #{gem_name}" expect(bundled_app("#{gem_name}/.git")).to exist end - context "when git is not available", :readline do + context "when git is not available" do # This spec cannot have `git` available in the test env before do load_paths = [lib_dir, spec_dir] @@ -451,7 +457,7 @@ RSpec.describe "bundle gem" do end end - it "generates a valid gemspec", :readline, :ruby_repo do + it "generates a valid gemspec", :ruby_repo do bundle "gem newgem --bin" prepare_gemspec(bundled_app("newgem", "newgem.gemspec")) @@ -464,7 +470,7 @@ RSpec.describe "bundle gem" do expect(last_command.stdboth).not_to include("ERROR") end - context "gem naming with relative paths", :readline do + context "gem naming with relative paths" do it "resolves ." do create_temporary_dir("tmp") @@ -557,8 +563,12 @@ RSpec.describe "bundle gem" do expect(bundled_app("#{gem_name}/bin/setup")).to exist expect(bundled_app("#{gem_name}/bin/console")).to exist - expect(bundled_app("#{gem_name}/bin/setup")).to be_executable - expect(bundled_app("#{gem_name}/bin/console")).to be_executable + + unless Gem.win_platform? + expect(bundled_app("#{gem_name}/bin/setup")).to be_executable + expect(bundled_app("#{gem_name}/bin/console")).to be_executable + end + expect(bundled_app("#{gem_name}/bin/setup").read).to start_with("#!") expect(bundled_app("#{gem_name}/bin/console").read).to start_with("#!") end @@ -591,8 +601,8 @@ RSpec.describe "bundle gem" do context "git config user.{name,email} is not set" do before do - sys_exec("git config --global --unset user.name") - sys_exec("git config --global --unset user.email") + git("config --global --unset user.name") + git("config --global --unset user.email") bundle "gem #{gem_name}" end @@ -870,7 +880,7 @@ RSpec.describe "bundle gem" do end end - context "gem.test set to rspec and --test with no arguments", :hint_text do + context "gem.test set to rspec and --test with no arguments" do before do bundle "config set gem.test rspec" bundle "gem #{gem_name} --test" @@ -887,10 +897,12 @@ RSpec.describe "bundle gem" do end end - context "gem.test setting set to false and --test with no arguments", :hint_text do + context "gem.test setting set to false and --test with no arguments", :readline do before do bundle "config set gem.test false" - bundle "gem #{gem_name} --test" + bundle "gem #{gem_name} --test" do |input, _, _| + input.puts + end end it "asks to generate test files" do @@ -904,9 +916,12 @@ RSpec.describe "bundle gem" do it_behaves_like "test framework is absent" end - context "gem.test setting not set and --test with no arguments", :hint_text do + context "gem.test setting not set and --test with no arguments", :readline do before do - bundle "gem #{gem_name} --test" + global_config "BUNDLE_GEM__TEST" => nil + bundle "gem #{gem_name} --test" do |input, _, _| + input.puts + end end it "asks to generate test files" do @@ -1009,7 +1024,7 @@ RSpec.describe "bundle gem" do end end - context "gem.ci set to github and --ci with no arguments", :hint_text do + context "gem.ci set to github and --ci with no arguments" do before do bundle "config set gem.ci github" bundle "gem #{gem_name} --ci" @@ -1024,10 +1039,12 @@ RSpec.describe "bundle gem" do end end - context "gem.ci setting set to false and --ci with no arguments", :hint_text do + context "gem.ci setting set to false and --ci with no arguments", :readline do before do bundle "config set gem.ci false" - bundle "gem #{gem_name} --ci" + bundle "gem #{gem_name} --ci" do |input, _, _| + input.puts "github" + end end it "asks to setup CI" do @@ -1039,9 +1056,12 @@ RSpec.describe "bundle gem" do end end - context "gem.ci setting not set and --ci with no arguments", :hint_text do + context "gem.ci setting not set and --ci with no arguments", :readline do before do - bundle "gem #{gem_name} --ci" + global_config "BUNDLE_GEM__CI" => nil + bundle "gem #{gem_name} --ci" do |input, _, _| + input.puts "github" + end end it "asks to setup CI" do @@ -1111,6 +1131,7 @@ RSpec.describe "bundle gem" do context "gem.rubocop setting set to true", bundler: "< 3" do before do + global_config "BUNDLE_GEM__LINTER" => nil bundle "config set gem.rubocop true" bundle "gem #{gem_name}" end @@ -1130,7 +1151,7 @@ RSpec.describe "bundle gem" do end end - context "gem.linter set to rubocop and --linter with no arguments", :hint_text do + context "gem.linter set to rubocop and --linter with no arguments" do before do bundle "config set gem.linter rubocop" bundle "gem #{gem_name} --linter" @@ -1145,10 +1166,12 @@ RSpec.describe "bundle gem" do end end - context "gem.linter setting set to false and --linter with no arguments", :hint_text do + context "gem.linter setting set to false and --linter with no arguments", :readline do before do bundle "config set gem.linter false" - bundle "gem #{gem_name} --linter" + bundle "gem #{gem_name} --linter" do |input, _, _| + input.puts "rubocop" + end end it "asks to setup a linter" do @@ -1160,9 +1183,12 @@ RSpec.describe "bundle gem" do end end - context "gem.linter setting not set and --linter with no arguments", :hint_text do + context "gem.linter setting not set and --linter with no arguments", :readline do before do - bundle "gem #{gem_name} --linter" + global_config "BUNDLE_GEM__LINTER" => nil + bundle "gem #{gem_name} --linter" do |input, _, _| + input.puts "rubocop" + end end it "asks to setup a linter" do @@ -1185,7 +1211,7 @@ RSpec.describe "bundle gem" do end end - context "testing --mit and --coc options against bundle config settings", :readline do + context "testing --mit and --coc options against bundle config settings" do let(:gem_name) { "test-gem" } let(:require_path) { "test/gem" } @@ -1288,10 +1314,10 @@ RSpec.describe "bundle gem" do end end - context "testing --github-username option against git and bundle config settings", :readline do + context "testing --github-username option against git and bundle config settings" do context "without git config set" do before do - sys_exec("git config --global --unset github.user") + git("config --global --unset github.user") end context "with github-username option in bundle config settings set to some value" do before do @@ -1325,10 +1351,10 @@ RSpec.describe "bundle gem" do end end - context "testing github_username bundle config against git config settings", :readline do + context "testing github_username bundle config against git config settings" do context "without git config set" do before do - sys_exec("git config --global --unset github.user") + git("config --global --unset github.user") end it_behaves_like "github_username configuration" @@ -1339,7 +1365,7 @@ RSpec.describe "bundle gem" do end end - context "gem naming with underscore", :readline do + context "gem naming with underscore" do let(:gem_name) { "test_gem" } let(:require_path) { "test_gem" } @@ -1485,7 +1511,7 @@ RSpec.describe "bundle gem" do end end - context "gem naming with dashed", :readline do + context "gem naming with dashed" do let(:gem_name) { "test-gem" } let(:require_path) { "test/gem" } @@ -1506,7 +1532,7 @@ RSpec.describe "bundle gem" do end describe "uncommon gem names" do - it "can deal with two dashes", :readline do + it "can deal with two dashes" do bundle "gem a--a" expect(bundled_app("a--a/a--a.gemspec")).to exist @@ -1536,7 +1562,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - describe "#ensure_safe_gem_name", :readline do + describe "#ensure_safe_gem_name" do before do bundle "gem #{subject}", raise_on_error: false end @@ -1564,7 +1590,7 @@ Usage: "bundle gem NAME [OPTIONS]" context "on first run", :readline do it "asks about test framework" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__COC" => "false" + global_config "BUNDLE_GEM__TEST" => nil bundle "gem foobar" do |input, _, _| input.puts "rspec" @@ -1587,7 +1613,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CI service" do - global_config "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__CI" => nil bundle "gem foobar" do |input, _, _| input.puts "github" @@ -1597,7 +1623,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about MIT license" do - global_config "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__MIT" => nil bundle "config list" @@ -1609,7 +1635,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CoC" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__COC" => nil bundle "gem foobar" do |input, _, _| input.puts "yes" @@ -1619,8 +1645,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CHANGELOG" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false", - "BUNDLE_GEM__COC" => "false" + global_config "BUNDLE_GEM__CHANGELOG" => nil bundle "gem foobar" do |input, _, _| input.puts "yes" @@ -1630,7 +1655,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - context "on conflicts with a previously created file", :readline do + context "on conflicts with a previously created file" do it "should fail gracefully" do FileUtils.touch(bundled_app("conflict-foobar")) bundle "gem conflict-foobar", raise_on_error: false @@ -1639,7 +1664,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - context "on conflicts with a previously created directory", :readline do + context "on conflicts with a previously created directory" do it "should succeed" do FileUtils.mkdir_p(bundled_app("conflict-foobar/Gemfile")) bundle "gem conflict-foobar" diff --git a/spec/bundler/commands/open_spec.rb b/spec/bundler/commands/open_spec.rb index 97374f30c3..18f4db38c9 100644 --- a/spec/bundler/commands/open_spec.rb +++ b/spec/bundler/commands/open_spec.rb @@ -44,7 +44,7 @@ RSpec.describe "bundle open" do G bundle "open foo", env: { "EDITOR" => "echo editor", "VISUAL" => "", "BUNDLER_EDITOR" => "" } - expect(out).to include("editor #{default_bundle_path.join("bundler", "gems", "foo-1.0-#{ref}")}") + expect(out).to include("editor #{default_bundle_path("bundler", "gems", "foo-1.0-#{ref}")}") end it "suggests alternatives for similar-sounding gems" do @@ -164,7 +164,6 @@ RSpec.describe "bundle open" do install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" - gem "json" G end diff --git a/spec/bundler/commands/post_bundle_message_spec.rb b/spec/bundler/commands/post_bundle_message_spec.rb index 07fd5a79e9..3c7fd3486d 100644 --- a/spec/bundler/commands/post_bundle_message_spec.rb +++ b/spec/bundler/commands/post_bundle_message_spec.rb @@ -120,7 +120,7 @@ RSpec.describe "post bundle message" do gem "not-a-gem", :group => :development G expect(err).to include <<-EOS.strip -Could not find gem 'not-a-gem' in rubygems repository #{file_uri_for(gem_repo1)}/, cached gems or installed locally. +Could not find gem 'not-a-gem' in rubygems repository #{file_uri_for(gem_repo1)}/ or installed locally. EOS end diff --git a/spec/bundler/commands/remove_spec.rb b/spec/bundler/commands/remove_spec.rb index 197fcde091..23ce7dde47 100644 --- a/spec/bundler/commands/remove_spec.rb +++ b/spec/bundler/commands/remove_spec.rb @@ -634,14 +634,14 @@ RSpec.describe "bundle remove" do context "with gemspec" do it "should not remove the gem" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("foo.gemspec", "") s.add_dependency "rack" end install_gemfile(<<-G) source "#{file_uri_for(gem_repo1)}" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G bundle "remove foo" diff --git a/spec/bundler/commands/show_spec.rb b/spec/bundler/commands/show_spec.rb index 2b6d4d2d00..4f02818cef 100644 --- a/spec/bundler/commands/show_spec.rb +++ b/spec/bundler/commands/show_spec.rb @@ -173,7 +173,7 @@ RSpec.describe "bundle show", bundler: "< 3" do G bundle "show rac" - expect(out).to match(/\A1 : rack\n2 : rack-obama\n0 : - exit -(\n>.*)?\z/) + expect(out).to match(/\A1 : rack\n2 : rack-obama\n0 : - exit -(\n>|\z)/) end end diff --git a/spec/bundler/commands/update_spec.rb b/spec/bundler/commands/update_spec.rb index 8565e27ebf..d5d38b81f1 100644 --- a/spec/bundler/commands/update_spec.rb +++ b/spec/bundler/commands/update_spec.rb @@ -849,8 +849,7 @@ RSpec.describe "bundle update" do end bundle "update", all: true - out.sub!("Removing foo (1.0)\n", "") - expect(out).to match(/Resolving dependencies\.\.\.\.*\nFetching foo 2\.0 \(was 1\.0\)\nInstalling foo 2\.0 \(was 1\.0\)\nBundle updated/) + expect(out.sub("Removing foo (1.0)\n", "")).to match(/Resolving dependencies\.\.\.\.*\nFetching foo 2\.0 \(was 1\.0\)\nInstalling foo 2\.0 \(was 1\.0\)\nBundle updated/) end it "shows error message when Gemfile.lock is not preset and gem is specified" do diff --git a/spec/bundler/install/gemfile/eval_gemfile_spec.rb b/spec/bundler/install/gemfile/eval_gemfile_spec.rb index cfa66e5986..7d2f9d46f9 100644 --- a/spec/bundler/install/gemfile/eval_gemfile_spec.rb +++ b/spec/bundler/install/gemfile/eval_gemfile_spec.rb @@ -2,7 +2,7 @@ RSpec.describe "bundle install with gemfile that uses eval_gemfile" do before do - build_lib("gunks", path: bundled_app.join("gems/gunks")) do |s| + build_lib("gunks", path: bundled_app("gems/gunks")) do |s| s.name = "gunks" s.version = "0.0.1" end diff --git a/spec/bundler/install/gemfile/gemspec_spec.rb b/spec/bundler/install/gemfile/gemspec_spec.rb index 63778567cf..2932228a54 100644 --- a/spec/bundler/install/gemfile/gemspec_spec.rb +++ b/spec/bundler/install/gemfile/gemspec_spec.rb @@ -39,14 +39,14 @@ RSpec.describe "bundle install from an existing gemspec" do end it "should install runtime and development dependencies" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("Gemfile", "source :rubygems\ngemspec") s.add_dependency "bar", "=1.0.0" s.add_development_dependency "bar-dev", "=1.0.0" end install_gemfile <<-G source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G expect(the_bundle).to include_gems "bar 1.0.0" @@ -54,16 +54,16 @@ RSpec.describe "bundle install from an existing gemspec" do end it "that is hidden should install runtime and development dependencies" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("Gemfile", "source :rubygems\ngemspec") s.add_dependency "bar", "=1.0.0" s.add_development_dependency "bar-dev", "=1.0.0" end - FileUtils.mv tmp.join("foo", "foo.gemspec"), tmp.join("foo", ".gemspec") + FileUtils.mv tmp("foo", "foo.gemspec"), tmp("foo", ".gemspec") install_gemfile <<-G source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G expect(the_bundle).to include_gems "bar 1.0.0" @@ -76,42 +76,42 @@ RSpec.describe "bundle install from an existing gemspec" do build_gem "baz", "1.1" end - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("Gemfile", "source :rubygems\ngemspec") s.add_dependency "baz", ">= 1.0", "< 1.1" end install_gemfile <<-G source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G expect(the_bundle).to include_gems "baz 1.0" end it "should raise if there are no gemspecs available" do - build_lib("foo", path: tmp.join("foo"), gemspec: false) + build_lib("foo", path: tmp("foo"), gemspec: false) install_gemfile <<-G, raise_on_error: false source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G - expect(err).to match(/There are no gemspecs at #{tmp.join("foo")}/) + expect(err).to match(/There are no gemspecs at #{tmp("foo")}/) end it "should raise if there are too many gemspecs available" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("foo2.gemspec", build_spec("foo", "4.0").first.to_ruby) end install_gemfile <<-G, raise_on_error: false source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G - expect(err).to match(/There are multiple gemspecs at #{tmp.join("foo")}/) + expect(err).to match(/There are multiple gemspecs at #{tmp("foo")}/) end it "should pick a specific gemspec" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("foo2.gemspec", "") s.add_dependency "bar", "=1.0.0" s.add_development_dependency "bar-dev", "=1.0.0" @@ -119,7 +119,7 @@ RSpec.describe "bundle install from an existing gemspec" do install_gemfile(<<-G) source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G expect(the_bundle).to include_gems "bar 1.0.0" @@ -127,7 +127,7 @@ RSpec.describe "bundle install from an existing gemspec" do end it "should use a specific group for development dependencies" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("foo2.gemspec", "") s.add_dependency "bar", "=1.0.0" s.add_development_dependency "bar-dev", "=1.0.0" @@ -135,7 +135,7 @@ RSpec.describe "bundle install from an existing gemspec" do install_gemfile(<<-G) source "#{file_uri_for(gem_repo2)}" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo', :development_group => :dev + gemspec :path => '#{tmp("foo")}', :name => 'foo', :development_group => :dev G expect(the_bundle).to include_gems "bar 1.0.0" @@ -144,33 +144,33 @@ RSpec.describe "bundle install from an existing gemspec" do end it "should match a lockfile even if the gemspec defines development dependencies" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.write("Gemfile", "source '#{file_uri_for(gem_repo1)}'\ngemspec") s.add_dependency "actionpack", "=2.3.2" s.add_development_dependency "rake", rake_version end - bundle "install", dir: tmp.join("foo") + bundle "install", dir: tmp("foo") # This should really be able to rely on $stderr, but, it's not written # right, so we can't. In fact, this is a bug negation test, and so it'll # ghost pass in future, and will only catch a regression if the message # doesn't change. Exit codes should be used correctly (they can be more # than just 0 and 1). bundle "config set --local deployment true" - output = bundle("install", dir: tmp.join("foo")) + output = bundle("install", dir: tmp("foo")) expect(output).not_to match(/You have added to the Gemfile/) expect(output).not_to match(/You have deleted from the Gemfile/) expect(output).not_to match(/the lockfile can't be updated because frozen mode is set/) end it "should match a lockfile without needing to re-resolve" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.add_dependency "rack" end install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G bundle "install", verbose: true @@ -182,14 +182,14 @@ RSpec.describe "bundle install from an existing gemspec" do it "should match a lockfile without needing to re-resolve with development dependencies" do simulate_platform java - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.add_dependency "rack" s.add_development_dependency "thin" end install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G bundle "install", verbose: true @@ -199,14 +199,14 @@ RSpec.describe "bundle install from an existing gemspec" do end it "should match a lockfile on non-ruby platforms with a transitive platform dependency", :jruby_only do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.add_dependency "platform_specific" end system_gems "platform_specific-1.0-java", path: default_bundle_path install_gemfile <<-G - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G bundle "update --bundler", artifice: "compact_index", verbose: true @@ -214,13 +214,13 @@ RSpec.describe "bundle install from an existing gemspec" do end it "should evaluate the gemspec in its directory" do - build_lib("foo", path: tmp.join("foo")) - File.open(tmp.join("foo/foo.gemspec"), "w") do |s| - s.write "raise 'ahh' unless Dir.pwd == '#{tmp.join("foo")}'" + build_lib("foo", path: tmp("foo")) + File.open(tmp("foo/foo.gemspec"), "w") do |s| + s.write "raise 'ahh' unless Dir.pwd == '#{tmp("foo")}'" end install_gemfile <<-G, raise_on_error: false - gemspec :path => '#{tmp.join("foo")}' + gemspec :path => '#{tmp("foo")}' G expect(last_command.stdboth).not_to include("ahh") end @@ -248,7 +248,7 @@ RSpec.describe "bundle install from an existing gemspec" do end it "allows conflicts" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.version = "1.0.0" s.add_dependency "bar", "= 1.0.0" end @@ -260,14 +260,14 @@ RSpec.describe "bundle install from an existing gemspec" do install_gemfile <<-G source "#{file_uri_for(gem_repo2)}" gem "deps" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G expect(the_bundle).to include_gems "foo 1.0.0" end it "does not break Gem.finish_resolve with conflicts" do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.version = "1.0.0" s.add_dependency "bar", "= 1.0.0" end @@ -281,7 +281,7 @@ RSpec.describe "bundle install from an existing gemspec" do install_gemfile <<-G source "#{file_uri_for(gem_repo2)}" gem "deps" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G expect(the_bundle).to include_gems "foo 1.0.0" @@ -359,7 +359,7 @@ RSpec.describe "bundle install from an existing gemspec" do let(:source_uri) { "http://localgemserver.test" } before do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.add_dependency "rack", "=1.0.0" end @@ -398,7 +398,7 @@ RSpec.describe "bundle install from an existing gemspec" do context "using JRuby with explicit platform", :jruby_only do before do create_file( - tmp.join("foo", "foo-java.gemspec"), + tmp("foo", "foo-java.gemspec"), build_spec("foo", "1.0", "java") do dep "rack", "=1.0.0" @spec.authors = "authors" @@ -589,7 +589,7 @@ RSpec.describe "bundle install from an existing gemspec" do context "with multiple platforms" do before do - build_lib("foo", path: tmp.join("foo")) do |s| + build_lib("foo", path: tmp("foo")) do |s| s.version = "1.0.0" s.add_development_dependency "rack" s.write "foo-universal-java.gemspec", build_spec("foo", "1.0.0", "universal-java") {|sj| sj.runtime "rack", "1.0.0" }.first.to_ruby @@ -601,7 +601,7 @@ RSpec.describe "bundle install from an existing gemspec" do install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G expect(the_bundle).to include_gems "foo 1.0.0", "rack 1.0.0" @@ -613,7 +613,7 @@ RSpec.describe "bundle install from an existing gemspec" do bundle "config set --local without development" install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" - gemspec :path => '#{tmp.join("foo")}', :name => 'foo' + gemspec :path => '#{tmp("foo")}', :name => 'foo' G expect(the_bundle).to include_gem "foo 1.0.0" @@ -623,7 +623,7 @@ RSpec.describe "bundle install from an existing gemspec" do context "with multiple platforms and resolving for more specific platforms" do before do - build_lib("chef", path: tmp.join("chef")) do |s| + build_lib("chef", path: tmp("chef")) do |s| s.version = "17.1.17" s.write "chef-universal-mingw32.gemspec", build_spec("chef", "17.1.17", "universal-mingw32") {|sw| sw.runtime "win32-api", "~> 1.5.3" }.first.to_ruby end @@ -682,7 +682,7 @@ RSpec.describe "bundle install from an existing gemspec" do context "with multiple locked platforms" do before do - build_lib("activeadmin", path: tmp.join("activeadmin")) do |s| + build_lib("activeadmin", path: tmp("activeadmin")) do |s| s.version = "2.9.0" s.add_dependency "railties", ">= 5.2", "< 6.2" end @@ -735,7 +735,7 @@ RSpec.describe "bundle install from an existing gemspec" do #{Bundler::VERSION} L - gemspec = tmp.join("activeadmin/activeadmin.gemspec") + gemspec = tmp("activeadmin/activeadmin.gemspec") File.write(gemspec, File.read(gemspec).sub(">= 5.2", ">= 6.0")) previous_lockfile = lockfile diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index 45ee7b44d1..21216594b1 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -31,7 +31,7 @@ RSpec.describe "bundle install with git sources" do end it "does not write to cache on bundler/setup" do - cache_path = default_bundle_path.join("cache") + cache_path = default_bundle_path("cache") FileUtils.rm_rf(cache_path) ruby "require 'bundler/setup'" expect(cache_path).not_to exist @@ -59,7 +59,7 @@ RSpec.describe "bundle install with git sources" do bundle "update foo" sha = git.ref_for("main", 11) - spec_file = default_bundle_path.join("bundler/gems/foo-1.0-#{sha}/foo.gemspec") + spec_file = default_bundle_path("bundler/gems/foo-1.0-#{sha}/foo.gemspec") expect(spec_file).to exist ruby_code = Gem::Specification.load(spec_file.to_s).to_ruby file_code = File.read(spec_file) @@ -272,7 +272,7 @@ RSpec.describe "bundle install with git sources" do s.write("lib/foo.rb", "raise 'FAIL'") end - sys_exec("git update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", dir: lib_path("foo-1.0")) + git("update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", lib_path("foo-1.0")) # want to ensure we don't fallback to HEAD update_git "foo", path: lib_path("foo-1.0"), branch: "rando" do |s| @@ -308,7 +308,7 @@ RSpec.describe "bundle install with git sources" do s.write("lib/foo.rb", "raise 'FAIL'") end - sys_exec("git update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", dir: lib_path("foo-1.0")) + git("update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", lib_path("foo-1.0")) # want to ensure we don't fallback to HEAD update_git "foo", path: lib_path("foo-1.0"), branch: "rando" do |s| @@ -332,7 +332,7 @@ RSpec.describe "bundle install with git sources" do end it "does not download random non-head refs" do - sys_exec("git update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", dir: lib_path("foo-1.0")) + git("update-ref -m \"Bundler Spec!\" refs/bundler/1 main~1", lib_path("foo-1.0")) bundle "config set global_gem_cache true" @@ -346,7 +346,7 @@ RSpec.describe "bundle install with git sources" do # ensure we also git fetch after cloning bundle :update, all: true - sys_exec("git ls-remote .", dir: Dir[home(".bundle/cache/git/foo-*")].first) + git("ls-remote .", Dir[home(".bundle/cache/git/foo-*")].first) expect(out).not_to include("refs/bundler/1") end @@ -906,7 +906,7 @@ RSpec.describe "bundle install with git sources" do bundle "update", all: true expect(the_bundle).to include_gems "forced 1.1" - sys_exec("git reset --hard HEAD^", dir: lib_path("forced-1.0")) + git("reset --hard HEAD^", lib_path("forced-1.0")) bundle "update", all: true expect(the_bundle).to include_gems "forced 1.0" @@ -920,8 +920,8 @@ RSpec.describe "bundle install with git sources" do build_git "has_submodule", "1.0" do |s| s.add_dependency "submodule" end - sys_exec "git submodule add #{lib_path("submodule-1.0")} submodule-1.0", dir: lib_path("has_submodule-1.0") - sys_exec "git commit -m \"submodulator\"", dir: lib_path("has_submodule-1.0") + git "submodule add #{lib_path("submodule-1.0")} submodule-1.0", lib_path("has_submodule-1.0") + git "commit -m \"submodulator\"", lib_path("has_submodule-1.0") install_gemfile <<-G, raise_on_error: false source "#{file_uri_for(gem_repo1)}" @@ -929,7 +929,7 @@ RSpec.describe "bundle install with git sources" do gem "has_submodule" end G - expect(err).to match(%r{submodule >= 0 could not be found in rubygems repository #{file_uri_for(gem_repo1)}/, cached gems or installed locally}) + expect(err).to match(%r{submodule >= 0 could not be found in rubygems repository #{file_uri_for(gem_repo1)}/ or installed locally}) expect(the_bundle).not_to include_gems "has_submodule 1.0" end @@ -942,8 +942,8 @@ RSpec.describe "bundle install with git sources" do build_git "has_submodule", "1.0" do |s| s.add_dependency "submodule" end - sys_exec "git submodule add #{lib_path("submodule-1.0")} submodule-1.0", dir: lib_path("has_submodule-1.0") - sys_exec "git commit -m \"submodulator\"", dir: lib_path("has_submodule-1.0") + git "submodule add #{lib_path("submodule-1.0")} submodule-1.0", lib_path("has_submodule-1.0") + git "commit -m \"submodulator\"", lib_path("has_submodule-1.0") install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" @@ -962,8 +962,8 @@ RSpec.describe "bundle install with git sources" do build_git "submodule", "1.0" build_git "has_submodule", "1.0" - sys_exec "git submodule add #{lib_path("submodule-1.0")} submodule-1.0", dir: lib_path("has_submodule-1.0") - sys_exec "git commit -m \"submodulator\"", dir: lib_path("has_submodule-1.0") + git "submodule add #{lib_path("submodule-1.0")} submodule-1.0", lib_path("has_submodule-1.0") + git "commit -m \"submodulator\"", lib_path("has_submodule-1.0") install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" @@ -1292,7 +1292,7 @@ RSpec.describe "bundle install with git sources" do void Init_foo() { rb_define_global_function("foo", &foo, 0); } C end - sys_exec("git commit -m \"commit for iteration #{i}\" ext/foo.c", dir: git_reader.path) + git("commit -m \"commit for iteration #{i}\" ext/foo.c", git_reader.path) git_commit_sha = git_reader.ref_for("HEAD") diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb index a5ba76f4d9..3de77ff7e3 100644 --- a/spec/bundler/install/gemfile/sources_spec.rb +++ b/spec/bundler/install/gemfile/sources_spec.rb @@ -520,7 +520,7 @@ RSpec.describe "bundle install with gems on multiple sources" do it "fails" do bundle :install, artifice: "compact_index", raise_on_error: false - expect(err).to include("Could not find gem 'private_gem_1' in rubygems repository https://gem.repo2/, cached gems or installed locally.") + expect(err).to include("Could not find gem 'private_gem_1' in rubygems repository https://gem.repo2/ or installed locally.") end end @@ -611,7 +611,7 @@ RSpec.describe "bundle install with gems on multiple sources" do Could not find compatible versions Because every version of depends_on_rack depends on rack >= 0 - and rack >= 0 could not be found in rubygems repository https://gem.repo2/, cached gems or installed locally, + and rack >= 0 could not be found in rubygems repository https://gem.repo2/ or installed locally, depends_on_rack cannot be used. So, because Gemfile depends on depends_on_rack >= 0, version solving has failed. @@ -1484,14 +1484,14 @@ RSpec.describe "bundle install with gems on multiple sources" do build_gem "bar" end - build_lib("gemspec_test", path: tmp.join("gemspec_test")) do |s| + build_lib("gemspec_test", path: tmp("gemspec_test")) do |s| s.add_dependency "bar", "=1.0.0" end install_gemfile <<-G, artifice: "compact_index" source "https://gem.repo2" gem "rack" - gemspec :path => "#{tmp.join("gemspec_test")}" + gemspec :path => "#{tmp("gemspec_test")}" G end @@ -1506,7 +1506,7 @@ RSpec.describe "bundle install with gems on multiple sources" do build_gem "bar" end - build_lib("gemspec_test", path: tmp.join("gemspec_test")) do |s| + build_lib("gemspec_test", path: tmp("gemspec_test")) do |s| s.add_development_dependency "bar" end @@ -1517,7 +1517,7 @@ RSpec.describe "bundle install with gems on multiple sources" do gem "bar" end - gemspec :path => "#{tmp.join("gemspec_test")}" + gemspec :path => "#{tmp("gemspec_test")}" G end diff --git a/spec/bundler/install/gemfile/specific_platform_spec.rb b/spec/bundler/install/gemfile/specific_platform_spec.rb index 5f1b034bfc..1a33053dc6 100644 --- a/spec/bundler/install/gemfile/specific_platform_spec.rb +++ b/spec/bundler/install/gemfile/specific_platform_spec.rb @@ -395,7 +395,7 @@ RSpec.describe "bundle install with specific platforms" do G error_message = <<~ERROR.strip - Could not find gem 'sorbet-static (= 0.5.6433)' with platform 'arm64-darwin-21' in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gem 'sorbet-static (= 0.5.6433)' with platform 'arm64-darwin-21' in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.6433)': * sorbet-static-0.5.6433-universal-darwin-20 @@ -434,7 +434,7 @@ RSpec.describe "bundle install with specific platforms" do Could not find compatible versions Because every version of sorbet depends on sorbet-static = 0.5.6433 - and sorbet-static = 0.5.6433 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally for any resolution platforms (arm64-darwin-21), + and sorbet-static = 0.5.6433 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally for any resolution platforms (arm64-darwin-21), sorbet cannot be used. So, because Gemfile depends on sorbet = 0.5.6433, version solving has failed. @@ -473,7 +473,7 @@ RSpec.describe "bundle install with specific platforms" do bundle "lock", raise_on_error: false, env: { "BUNDLE_FORCE_RUBY_PLATFORM" => "true" } expect(err).to include <<~ERROR.rstrip - Could not find gem 'sorbet-static (= 0.5.9889)' with platform 'ruby' in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gem 'sorbet-static (= 0.5.9889)' with platform 'ruby' in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.9889)': * sorbet-static-0.5.9889-#{Gem::Platform.local} diff --git a/spec/bundler/install/gems/flex_spec.rb b/spec/bundler/install/gems/flex_spec.rb index 8ef3984975..5e0c88fc95 100644 --- a/spec/bundler/install/gems/flex_spec.rb +++ b/spec/bundler/install/gems/flex_spec.rb @@ -197,7 +197,7 @@ RSpec.describe "bundle flex_install" do Could not find compatible versions Because rack-obama >= 2.0 depends on rack = 1.2 - and rack = 1.2 could not be found in rubygems repository #{file_uri_for(gem_repo2)}/, cached gems or installed locally, + and rack = 1.2 could not be found in rubygems repository #{file_uri_for(gem_repo2)}/ or installed locally, rack-obama >= 2.0 cannot be used. So, because Gemfile depends on rack-obama = 2.0, version solving has failed. diff --git a/spec/bundler/install/gems/resolving_spec.rb b/spec/bundler/install/gems/resolving_spec.rb index c5f9c4a3d3..b54674898d 100644 --- a/spec/bundler/install/gems/resolving_spec.rb +++ b/spec/bundler/install/gems/resolving_spec.rb @@ -434,7 +434,7 @@ RSpec.describe "bundle install with install-time dependencies" do end nice_error = <<~E.strip - Could not find gems matching 'sorbet-static (= 0.5.10554)' valid for all resolution platforms (arm64-darwin-21, aarch64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gems matching 'sorbet-static (= 0.5.10554)' valid for all resolution platforms (arm64-darwin-21, aarch64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.10554)': * sorbet-static-0.5.10554-universal-darwin-21 @@ -490,7 +490,7 @@ RSpec.describe "bundle install with install-time dependencies" do it "raises a proper error" do nice_error = <<~E.strip - Could not find gems matching 'sorbet-static' valid for all resolution platforms (arm-linux, x86_64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gems matching 'sorbet-static' valid for all resolution platforms (arm-linux, x86_64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static': * sorbet-static-0.5.10696-x86_64-linux diff --git a/spec/bundler/install/yanked_spec.rb b/spec/bundler/install/yanked_spec.rb index 7408c24327..5aeabd2f23 100644 --- a/spec/bundler/install/yanked_spec.rb +++ b/spec/bundler/install/yanked_spec.rb @@ -188,7 +188,7 @@ RSpec.context "when using gem before installing" do bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1 in locally installed gems") expect(err).to_not include("Your bundle is locked to rack (0.9.1) from") expect(err).to_not include("If you haven't changed sources, that means the author of rack (0.9.1) has removed it.") expect(err).to_not include("You'll need to update your bundle to a different version of rack (0.9.1) that hasn't been removed in order to install.") @@ -197,7 +197,7 @@ RSpec.context "when using gem before installing" do lockfile lockfile.gsub(/PLATFORMS\n #{lockfile_platforms}/m, "PLATFORMS\n #{lockfile_platforms("ruby")}") bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1 in locally installed gems") end it "does not suggest the author has yanked the gem when using more than one gem, but shows all gems that couldn't be found in the source" do @@ -224,7 +224,7 @@ RSpec.context "when using gem before installing" do bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1, rack_middleware-1.0 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1, rack_middleware-1.0 in locally installed gems") expect(err).to include("Install missing gems with `bundle install`.") expect(err).to_not include("Your bundle is locked to rack (0.9.1) from") expect(err).to_not include("If you haven't changed sources, that means the author of rack (0.9.1) has removed it.") diff --git a/spec/bundler/lock/lockfile_spec.rb b/spec/bundler/lock/lockfile_spec.rb index 4fd081e7d0..a1bbd050e5 100644 --- a/spec/bundler/lock/lockfile_spec.rb +++ b/spec/bundler/lock/lockfile_spec.rb @@ -1117,7 +1117,7 @@ RSpec.describe "the lockfile format" do end it "stores relative paths when the path is provided for gemspec" do - build_lib("foo", path: tmp.join("foo")) + build_lib("foo", path: tmp("foo")) checksums = checksums_section_when_existing do |c| c.no_checksum "foo", "1.0" diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index 939b68a0bb..e7577d38b4 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -618,7 +618,12 @@ RSpec.describe "major deprecations" do pending "fails with a helpful message", bundler: "3" end - describe "deprecating rubocop", :readline do + describe "deprecating rubocop" do + before do + global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", + "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__CHANGELOG" => "false" + end + context "bundle gem --rubocop" do before do bundle "gem my_new_gem --rubocop", raise_on_error: false diff --git a/spec/bundler/runtime/gem_tasks_spec.rb b/spec/bundler/runtime/gem_tasks_spec.rb index f7afc0eb92..d086a72c9c 100644 --- a/spec/bundler/runtime/gem_tasks_spec.rb +++ b/spec/bundler/runtime/gem_tasks_spec.rb @@ -57,7 +57,7 @@ RSpec.describe "require 'bundler/gem_tasks'" do context "rake build when path has spaces", :ruby_repo do before do - spaced_bundled_app = tmp.join("bundled app") + spaced_bundled_app = tmp("bundled app") FileUtils.cp_r bundled_app, spaced_bundled_app bundle "exec rake build", dir: spaced_bundled_app end @@ -69,7 +69,7 @@ RSpec.describe "require 'bundler/gem_tasks'" do context "rake build when path has brackets", :ruby_repo do before do - bracketed_bundled_app = tmp.join("bundled[app") + bracketed_bundled_app = tmp("bundled[app") FileUtils.cp_r bundled_app, bracketed_bundled_app bundle "exec rake build", dir: bracketed_bundled_app end diff --git a/spec/bundler/runtime/require_spec.rb b/spec/bundler/runtime/require_spec.rb index 76271a5593..e630e902c9 100644 --- a/spec/bundler/runtime/require_spec.rb +++ b/spec/bundler/runtime/require_spec.rb @@ -430,6 +430,30 @@ RSpec.describe "Bundler.require" do expect(out).to eq("WIN") end + + it "does not extract gemspecs from application cache packages" do + gemfile <<-G + source "#{file_uri_for(gem_repo1)}" + gem "rack" + G + + bundle :cache + + path = cached_gem("rack-1.0.0") + + run <<-R + File.open("#{path}", "w") do |f| + f.write "broken package" + end + R + + run <<-R + Bundler.require + puts "WIN" + R + + expect(out).to eq("WIN") + end end RSpec.describe "Bundler.require with platform specific dependencies" do diff --git a/spec/bundler/runtime/setup_spec.rb b/spec/bundler/runtime/setup_spec.rb index 2d78825de4..7f4c0759d1 100644 --- a/spec/bundler/runtime/setup_spec.rb +++ b/spec/bundler/runtime/setup_spec.rb @@ -767,7 +767,7 @@ end expect(err).to be_empty end - it "can require rubygems without warnings, when using a local cache", rubygems: ">= 3.5.10" do + it "can require rubygems without warnings, when using a local cache", :truffleruby do install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" gem "rack" @@ -942,9 +942,9 @@ end G run <<-R - puts Bundler.rubygems.all_specs.map(&:name) + puts Bundler.rubygems.installed_specs.map(&:name) Gem.refresh - puts Bundler.rubygems.all_specs.map(&:name) + puts Bundler.rubygems.installed_specs.map(&:name) R expect(out).to eq("activesupport\nbundler\nactivesupport\nbundler") @@ -1376,6 +1376,24 @@ end expect(out).to eq("undefined\nconstant") end + it "activates default gems when they are part of the bundle, but not installed explicitly", :ruby_repo do + default_json_version = ruby "gem 'json'; require 'json'; puts JSON::VERSION" + + build_repo2 do + build_gem "json", default_json_version + end + + gemfile "source \"#{file_uri_for(gem_repo2)}\"; gem 'json'" + + ruby <<-RUBY + require "bundler/setup" + require "json" + puts defined?(::JSON) ? "JSON defined" : "JSON undefined" + RUBY + + expect(err).to be_empty + end + describe "default gem activation" do let(:exemptions) do exempts = %w[did_you_mean bundler uri pathname] diff --git a/spec/bundler/support/build_metadata.rb b/spec/bundler/support/build_metadata.rb index 5898e7f3bd..189100edb7 100644 --- a/spec/bundler/support/build_metadata.rb +++ b/spec/bundler/support/build_metadata.rb @@ -41,7 +41,7 @@ module Spec end def git_commit_sha - ruby_core_tarball? ? "unknown" : sys_exec("git rev-parse --short HEAD", dir: source_root).strip + ruby_core_tarball? ? "unknown" : git("rev-parse --short HEAD", source_root).strip end extend self diff --git a/spec/bundler/support/command_execution.rb b/spec/bundler/support/command_execution.rb index 5639fda3b6..02726744d3 100644 --- a/spec/bundler/support/command_execution.rb +++ b/spec/bundler/support/command_execution.rb @@ -1,7 +1,37 @@ # frozen_string_literal: true module Spec - CommandExecution = Struct.new(:command, :working_directory, :exitstatus, :original_stdout, :original_stderr) do + class CommandExecution + def initialize(command, working_directory:, timeout:) + @command = command + @working_directory = working_directory + @timeout = timeout + @original_stdout = String.new + @original_stderr = String.new + end + + attr_accessor :exitstatus, :command, :original_stdout, :original_stderr + attr_reader :timeout + attr_writer :failure_reason + + def raise_error! + return unless failure? + + error_header = if failure_reason == :timeout + "Invoking `#{command}` was aborted after #{timeout} seconds with output:" + else + "Invoking `#{command}` failed with output:" + end + + raise <<~ERROR + #{error_header} + + ---------------------------------------------------------------------- + #{stdboth} + ---------------------------------------------------------------------- + ERROR + end + def to_s "$ #{command}" end @@ -12,16 +42,11 @@ module Spec end def stdout - original_stdout + normalize(original_stdout) end - # Can be removed once/if https://github.com/oneclick/rubyinstaller2/pull/369 is resolved def stderr - return original_stderr unless Gem.win_platform? - - original_stderr.split("\n").reject do |l| - l.include?("operating_system_defaults") - end.join("\n") + normalize(original_stderr) end def to_s_verbose @@ -42,5 +67,13 @@ module Spec return true unless exitstatus exitstatus > 0 end + + private + + attr_reader :failure_reason + + def normalize(string) + string.force_encoding(Encoding::UTF_8).strip.gsub("\r\n", "\n") + end end end diff --git a/spec/bundler/support/env.rb b/spec/bundler/support/env.rb new file mode 100644 index 0000000000..4d99c892cd --- /dev/null +++ b/spec/bundler/support/env.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Spec + module Env + def ruby_core? + !ENV["GEM_COMMAND"].nil? + end + end +end diff --git a/spec/bundler/support/filters.rb b/spec/bundler/support/filters.rb index 8e164af756..e1683ae75b 100644 --- a/spec/bundler/support/filters.rb +++ b/spec/bundler/support/filters.rb @@ -14,7 +14,7 @@ class RequirementChecker < Proc attr_accessor :provided def inspect - "\"!= #{provided}\"" + "\"#{provided}\"" end end diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb index 1ad9cc78ca..392ed200dd 100644 --- a/spec/bundler/support/helpers.rb +++ b/spec/bundler/support/helpers.rb @@ -1,12 +1,17 @@ # frozen_string_literal: true -require_relative "command_execution" require_relative "the_bundle" require_relative "path" +require_relative "options" +require_relative "subprocess" module Spec module Helpers include Spec::Path + include Spec::Options + include Spec::Subprocess + + class TimeoutExceeded < StandardError; end def reset! Dir.glob("#{tmp}/{gems/*,*}", File::FNM_DOTMATCH).each do |dir| @@ -27,22 +32,6 @@ module Spec TheBundle.new(*args) end - def command_executions - @command_executions ||= [] - end - - def last_command - command_executions.last || raise("There is no last command") - end - - def out - last_command.stdout - end - - def err - last_command.stderr - end - MAJOR_DEPRECATION = /^\[DEPRECATED\]\s*/ def err_without_deprecations @@ -53,10 +42,6 @@ module Spec err.split("\n").select {|l| l =~ MAJOR_DEPRECATION }.join("\n").split(MAJOR_DEPRECATION) end - def exitstatus - last_command.exitstatus - end - def run(cmd, *args) opts = args.last.is_a?(Hash) ? args.pop : {} groups = args.map(&:inspect).join(", ") @@ -122,7 +107,7 @@ module Spec end def bundler(cmd, options = {}) - options[:bundle_bin] = system_gem_path.join("bin/bundler") + options[:bundle_bin] = system_gem_path("bin/bundler") bundle(cmd, options) end @@ -175,63 +160,30 @@ module Spec "#{Gem.ruby} -S #{ENV["GEM_PATH"]}/bin/rake" end - def git(cmd, path, options = {}) - sys_exec("git #{cmd}", options.merge(dir: path)) - end - - def sys_exec(cmd, options = {}) + def sys_exec(cmd, options = {}, &block) env = options[:env] || {} env["RUBYOPT"] = opt_add(opt_add("-r#{spec_dir}/support/switch_rubygems.rb", env["RUBYOPT"]), ENV["RUBYOPT"]) - dir = options[:dir] || bundled_app - command_execution = CommandExecution.new(cmd.to_s, dir) - - require "open3" - require "shellwords" - Open3.popen3(env, *cmd.shellsplit, chdir: dir) do |stdin, stdout, stderr, wait_thr| - yield stdin, stdout, wait_thr if block_given? - stdin.close - - stdout_read_thread = Thread.new { stdout.read } - stderr_read_thread = Thread.new { stderr.read } - command_execution.original_stdout = stdout_read_thread.value.strip - command_execution.original_stderr = stderr_read_thread.value.strip - - status = wait_thr.value - command_execution.exitstatus = if status.exited? - status.exitstatus - elsif status.signaled? - exit_status_for_signal(status.termsig) - end - end - - unless options[:raise_on_error] == false || command_execution.success? - raise <<~ERROR - - Invoking `#{cmd}` failed with output: - ---------------------------------------------------------------------- - #{command_execution.stdboth} - ---------------------------------------------------------------------- - ERROR - end - - command_executions << command_execution + options[:env] = env + options[:dir] ||= bundled_app - command_execution.stdout + sh(cmd, options, &block) end - def all_commands_output - return "" if command_executions.empty? + def config(config = nil, path = bundled_app(".bundle/config")) + current = File.exist?(path) ? Psych.load_file(path) : {} + return current unless config - "\n\nCommands:\n#{command_executions.map(&:to_s_verbose).join("\n\n")}" - end + current = {} if current.empty? - def config(config = nil, path = bundled_app(".bundle/config")) - return Psych.load_file(path) unless config FileUtils.mkdir_p(File.dirname(path)) - File.open(path, "w") do |f| - f.puts config.to_yaml + + new_config = current.merge(config).compact + + File.open(path, "w+") do |f| + f.puts new_config.to_yaml end - config + + new_config end def global_config(config = nil) @@ -361,16 +313,6 @@ module Spec end end - def opt_add(option, options) - [option.strip, options].compact.reject(&:empty?).join(" ") - end - - def opt_remove(option, options) - return unless options - - options.split(" ").reject {|opt| opt.strip == option.strip }.join(" ") - end - def break_git! FileUtils.mkdir_p(tmp("broken_path")) File.open(tmp("broken_path/git"), "w", 0o755) do |f| @@ -471,7 +413,7 @@ module Spec end def revision_for(path) - sys_exec("git rev-parse HEAD", dir: path).strip + git("rev-parse HEAD", path).strip end def with_read_only(pattern) diff --git a/spec/bundler/support/options.rb b/spec/bundler/support/options.rb new file mode 100644 index 0000000000..551fa1acd8 --- /dev/null +++ b/spec/bundler/support/options.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Spec + module Options + def opt_add(option, options) + [option.strip, options].compact.reject(&:empty?).join(" ") + end + + def opt_remove(option, options) + return unless options + + options.split(" ").reject {|opt| opt.strip == option.strip }.join(" ") + end + end +end diff --git a/spec/bundler/support/path.rb b/spec/bundler/support/path.rb index 7352d5a353..26be5488c3 100644 --- a/spec/bundler/support/path.rb +++ b/spec/bundler/support/path.rb @@ -3,8 +3,12 @@ require "pathname" require "rbconfig" +require_relative "env" + module Spec module Path + include Spec::Env + def source_root @source_root ||= Pathname.new(ruby_core? ? "../../.." : "../..").expand_path(__dir__) end @@ -58,7 +62,7 @@ module Spec end def gem_bin - @gem_bin ||= ruby_core? ? ENV["GEM_COMMAND"] : "gem" + @gem_bin ||= ENV["GEM_COMMAND"] || "gem" end def path @@ -109,7 +113,7 @@ module Spec end def home(*path) - tmp.join("home", *path) + tmp("home", *path) end def default_bundle_path(*path) @@ -129,13 +133,13 @@ module Spec end def bundled_app(*path) - root = tmp.join("bundled_app") + root = tmp("bundled_app") FileUtils.mkdir_p(root) root.join(*path) end def bundled_app2(*path) - root = tmp.join("bundled_app2") + root = tmp("bundled_app2") FileUtils.mkdir_p(root) root.join(*path) end @@ -161,15 +165,15 @@ module Spec end def base_system_gems - tmp.join("gems/base") + tmp("gems/base") end def rubocop_gems - tmp.join("gems/rubocop") + tmp("gems/rubocop") end def standard_gems - tmp.join("gems/standard") + tmp("gems/standard") end def file_uri_for(path) @@ -257,17 +261,6 @@ module Spec File.open(gemspec_file, "w") {|f| f << contents } end - def ruby_core? - # avoid to warnings - @ruby_core ||= nil - - if @ruby_core.nil? - @ruby_core = true & ENV["GEM_COMMAND"] - else - @ruby_core - end - end - def git_root ruby_core? ? source_root : source_root.parent end @@ -277,7 +270,7 @@ module Spec def git_ls_files(glob) skip "Not running on a git context, since running tests from a tarball" if ruby_core_tarball? - sys_exec("git ls-files -z -- #{glob}", dir: source_root).split("\x0") + git("ls-files -z -- #{glob}", source_root).split("\x0") end def tracked_files_glob diff --git a/spec/bundler/support/rubygems_ext.rb b/spec/bundler/support/rubygems_ext.rb index 889ebc90c3..7748234abc 100644 --- a/spec/bundler/support/rubygems_ext.rb +++ b/spec/bundler/support/rubygems_ext.rb @@ -70,7 +70,7 @@ module Spec ENV["BUNDLE_PATH"] = nil ENV["GEM_HOME"] = ENV["GEM_PATH"] = Path.base_system_gem_path.to_s - ENV["PATH"] = [Path.system_gem_path.join("bin"), ENV["PATH"]].join(File::PATH_SEPARATOR) + ENV["PATH"] = [Path.system_gem_path("bin"), ENV["PATH"]].join(File::PATH_SEPARATOR) ENV["PATH"] = [Path.bindir, ENV["PATH"]].join(File::PATH_SEPARATOR) if Path.ruby_core? end @@ -117,7 +117,14 @@ module Spec def gem_activate_and_possibly_install(gem_name) gem_activate(gem_name) rescue Gem::LoadError => e - Gem.install(gem_name, e.requirement) + # Windows 3.0 puts a Windows stub script as `rake` while it should be + # named `rake.bat`. RubyGems does not like that and avoids overwriting it + # unless explicitly instructed to do so with `force`. + if RUBY_VERSION.start_with?("3.0") && Gem.win_platform? + Gem.install(gem_name, e.requirement, force: true) + else + Gem.install(gem_name, e.requirement) + end retry end diff --git a/spec/bundler/support/rubygems_version_manager.rb b/spec/bundler/support/rubygems_version_manager.rb index 88da14b67e..c174c461f0 100644 --- a/spec/bundler/support/rubygems_version_manager.rb +++ b/spec/bundler/support/rubygems_version_manager.rb @@ -1,12 +1,13 @@ # frozen_string_literal: true -require "pathname" -require_relative "helpers" -require_relative "path" +require_relative "options" +require_relative "env" +require_relative "subprocess" class RubygemsVersionManager - include Spec::Helpers - include Spec::Path + include Spec::Options + include Spec::Env + include Spec::Subprocess def initialize(source) @source = source @@ -57,7 +58,7 @@ class RubygemsVersionManager cmd = [RbConfig.ruby, $0, *ARGV].compact - ENV["RUBYOPT"] = opt_add("-I#{local_copy_path.join("lib")}", opt_remove("--disable-gems", ENV["RUBYOPT"])) + ENV["RUBYOPT"] = opt_add("-I#{File.join(local_copy_path, "lib")}", opt_remove("--disable-gems", ENV["RUBYOPT"])) exec(ENV, *cmd) end @@ -65,14 +66,14 @@ class RubygemsVersionManager def switch_local_copy_if_needed return unless local_copy_switch_needed? - sys_exec("git checkout #{target_tag}", dir: local_copy_path) + git("checkout #{target_tag}", local_copy_path) - ENV["RGV"] = local_copy_path.to_s + ENV["RGV"] = local_copy_path end def rubygems_unrequire_needed? require "rubygems" - !$LOADED_FEATURES.include?(local_copy_path.join("lib/rubygems.rb").to_s) + !$LOADED_FEATURES.include?(File.join(local_copy_path, "lib/rubygems.rb")) end def local_copy_switch_needed? @@ -84,7 +85,7 @@ class RubygemsVersionManager end def local_copy_tag - sys_exec("git rev-parse --abbrev-ref HEAD", dir: local_copy_path) + git("rev-parse --abbrev-ref HEAD", local_copy_path) end def local_copy_path @@ -94,21 +95,25 @@ class RubygemsVersionManager def resolve_local_copy_path return expanded_source if source_is_path? - rubygems_path = source_root.join("tmp/rubygems") + rubygems_path = File.join(source_root, "tmp/rubygems") - unless rubygems_path.directory? - sys_exec("git clone .. #{rubygems_path}", dir: source_root) + unless File.directory?(rubygems_path) + git("clone .. #{rubygems_path}", source_root) end rubygems_path end def source_is_path? - expanded_source.directory? + File.directory?(expanded_source) end def expanded_source - @expanded_source ||= Pathname.new(@source).expand_path(source_root) + @expanded_source ||= File.expand_path(@source, source_root) + end + + def source_root + @source_root ||= File.expand_path(ruby_core? ? "../../.." : "../..", __dir__) end def resolve_target_tag diff --git a/spec/bundler/support/subprocess.rb b/spec/bundler/support/subprocess.rb new file mode 100644 index 0000000000..711bfbbeed --- /dev/null +++ b/spec/bundler/support/subprocess.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require_relative "command_execution" + +module Spec + module Subprocess + def command_executions + @command_executions ||= [] + end + + def last_command + command_executions.last || raise("There is no last command") + end + + def out + last_command.stdout + end + + def err + last_command.stderr + end + + def exitstatus + last_command.exitstatus + end + + def git(cmd, path = Dir.pwd, options = {}) + sh("git #{cmd}", options.merge(dir: path)) + end + + def sh(cmd, options = {}) + dir = options[:dir] + env = options[:env] || {} + + command_execution = CommandExecution.new(cmd.to_s, working_directory: dir, timeout: 60) + + require "open3" + require "shellwords" + Open3.popen3(env, *cmd.shellsplit, chdir: dir) do |stdin, stdout, stderr, wait_thr| + yield stdin, stdout, wait_thr if block_given? + stdin.close + + stdout_handler = ->(data) { command_execution.original_stdout << data } + stderr_handler = ->(data) { command_execution.original_stderr << data } + + stdout_thread = read_stream(stdout, stdout_handler, timeout: command_execution.timeout) + stderr_thread = read_stream(stderr, stderr_handler, timeout: command_execution.timeout) + + stdout_thread.join + stderr_thread.join + + status = wait_thr.value + command_execution.exitstatus = if status.exited? + status.exitstatus + elsif status.signaled? + exit_status_for_signal(status.termsig) + end + rescue TimeoutExceeded + command_execution.failure_reason = :timeout + command_execution.exitstatus = exit_status_for_signal(Signal.list["INT"]) + end + + unless options[:raise_on_error] == false || command_execution.success? + command_execution.raise_error! + end + + command_executions << command_execution + + command_execution.stdout + end + + # Mostly copied from https://github.com/piotrmurach/tty-command/blob/49c37a895ccea107e8b78d20e4cb29de6a1a53c8/lib/tty/command/process_runner.rb#L165-L193 + def read_stream(stream, handler, timeout:) + Thread.new do + Thread.current.report_on_exception = false + cmd_start = Time.now + readers = [stream] + + while readers.any? + ready = IO.select(readers, nil, readers, timeout) + raise TimeoutExceeded if ready.nil? + + ready[0].each do |reader| + chunk = reader.readpartial(16 * 1024) + handler.call(chunk) + + # control total time spent reading + runtime = Time.now - cmd_start + time_left = timeout - runtime + raise TimeoutExceeded if time_left < 0.0 + rescue Errno::EAGAIN, Errno::EINTR + rescue EOFError, Errno::EPIPE, Errno::EIO + readers.delete(reader) + reader.close + end + end + end + end + + def all_commands_output + return "" if command_executions.empty? + + "\n\nCommands:\n#{command_executions.map(&:to_s_verbose).join("\n\n")}" + end + end +end diff --git a/spec/bundler/update/git_spec.rb b/spec/bundler/update/git_spec.rb index 3b7bbfd979..f80281e8ce 100644 --- a/spec/bundler/update/git_spec.rb +++ b/spec/bundler/update/git_spec.rb @@ -142,8 +142,8 @@ RSpec.describe "bundle update" do s.add_dependency "submodule" end - sys_exec "git submodule add #{lib_path("submodule-1.0")} submodule-1.0", dir: lib_path("has_submodule-1.0") - sys_exec "git commit -m \"submodulator\"", dir: lib_path("has_submodule-1.0") + git "submodule add #{lib_path("submodule-1.0")} submodule-1.0", lib_path("has_submodule-1.0") + git "commit -m \"submodulator\"", lib_path("has_submodule-1.0") end it "it unlocks the source when submodules are added to a git source" do diff --git a/spec/default.mspec b/spec/default.mspec index cae5fa374f..57142d6dec 100644 --- a/spec/default.mspec +++ b/spec/default.mspec @@ -90,6 +90,7 @@ require 'mspec/runner/formatters/dotted' class DottedFormatter prepend Module.new { BASE = __dir__ + "/ruby/" unless defined?(BASE) + COUNT_WIDTH = 6 def initialize(out = nil) super @@ -97,7 +98,10 @@ class DottedFormatter @columns = nil else columns = ENV["COLUMNS"]&.to_i - @columns = columns&.nonzero? || 80 + columns = 80 unless columns&.nonzero? + w = COUNT_WIDTH + 1 + round = 20 + @columns = (columns - w) / round * round + w end @dotted = 0 @loaded = false @@ -113,7 +117,7 @@ class DottedFormatter def after(*) if @columns if @dotted == 0 - s = sprintf("%6d ", @count) + s = sprintf("%*d ", COUNT_WIDTH, @count) print(s) @dotted += s.size end diff --git a/spec/prism.mspec b/spec/prism.mspec deleted file mode 100644 index 3014b9788f..0000000000 --- a/spec/prism.mspec +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -# We are missing emitting some :end event inside eval; we need more -# investigation here. -MSpec.register(:exclude, "TracePoint#path equals \"(eval at __FILE__:__LINE__)\" inside an eval for :end event") - -# We need to respect the eval coverage setting. -MSpec.register(:exclude, "Coverage.result returns the correct results when eval coverage is disabled") diff --git a/spec/ruby/.rubocop.yml b/spec/ruby/.rubocop.yml index be32ce8900..5a902e0f01 100644 --- a/spec/ruby/.rubocop.yml +++ b/spec/ruby/.rubocop.yml @@ -1,7 +1,7 @@ inherit_from: .rubocop_todo.yml AllCops: - TargetRubyVersion: 3.0 + TargetRubyVersion: 3.1 DisplayCopNames: true Exclude: - command_line/fixtures/bad_syntax.rb diff --git a/spec/ruby/command_line/frozen_strings_spec.rb b/spec/ruby/command_line/frozen_strings_spec.rb index 334b98273b..06889755d2 100644 --- a/spec/ruby/command_line/frozen_strings_spec.rb +++ b/spec/ruby/command_line/frozen_strings_spec.rb @@ -42,16 +42,8 @@ describe "With neither --enable-frozen-string-literal nor --disable-frozen-strin ruby_exe(fixture(__FILE__, "freeze_flag_one_literal.rb")).chomp.should == "false" end - ruby_version_is "3.4" do - it "if file has no frozen_string_literal comment produce different frozen strings each time" do - ruby_exe(fixture(__FILE__, "string_literal_raw.rb")).chomp.should == "frozen:true interned:false" - end - end - - ruby_version_is ""..."3.4" do - it "if file has no frozen_string_literal comment produce different mutable strings each time" do - ruby_exe(fixture(__FILE__, "string_literal_raw.rb")).chomp.should == "frozen:false interned:false" - end + it "if file has no frozen_string_literal comment produce different mutable strings each time" do + ruby_exe(fixture(__FILE__, "string_literal_raw.rb")).chomp.should == "frozen:false interned:false" end it "if file has frozen_string_literal:true comment produce same frozen strings each time" do diff --git a/spec/ruby/core/array/fixtures/classes.rb b/spec/ruby/core/array/fixtures/classes.rb index 8596245fb8..05283c0f74 100644 --- a/spec/ruby/core/array/fixtures/classes.rb +++ b/spec/ruby/core/array/fixtures/classes.rb @@ -56,23 +56,20 @@ module ArraySpecs 101.621, 102.816, 104.010, 105.202, 106.393, 107.583, 108.771, 109.958, 111.144, 112.329, 113.512, 114.695, 115.876, 117.057, 118.236, 119.414, 120.591, 121.767, 122.942, 124.116, 125.289, 126.462, 127.633, 128.803, 129.973, 131.141, 132.309, 133.476, 134.642, 135.807, - ] + ] def self.measure_sample_fairness(size, samples, iters) ary = Array.new(size) { |x| x } + expected = iters.fdiv size (samples).times do |i| chi_results = [] 3.times do - counts = Array.new(size) { 0 } - expected = iters / size + counts = Array.new(size, 0) iters.times do x = ary.sample(samples)[i] counts[x] += 1 end - chi_squared = 0.0 - counts.each do |count| - chi_squared += (((count - expected) ** 2) * 1.0 / expected) - end + chi_squared = counts.sum {|count| (count - expected) ** 2} / expected chi_results << chi_squared break if chi_squared <= CHI_SQUARED_CRITICAL_VALUES[size] end @@ -83,17 +80,14 @@ module ArraySpecs def self.measure_sample_fairness_large_sample_size(size, samples, iters) ary = Array.new(size) { |x| x } - counts = Array.new(size) { 0 } - expected = iters * samples / size + counts = Array.new(size, 0) + expected = (iters * samples).fdiv size iters.times do ary.sample(samples).each do |sample| counts[sample] += 1 end end - chi_squared = 0.0 - counts.each do |count| - chi_squared += (((count - expected) ** 2) * 1.0 / expected) - end + chi_squared = counts.sum {|count| (count - expected) ** 2} / expected # Chi squared critical values for tests with 4 degrees of freedom # Values obtained from NIST Engineering Statistic Handbook at @@ -223,366 +217,370 @@ module ArraySpecs obj end - LargeArray = ["test_create_table_with_force_true_does_not_drop_nonexisting_table", - "test_add_table", - "assert_difference", - "assert_operator", - "instance_variables", - "class", - "instance_variable_get", - "__class__", - "expects", - "assert_no_difference", - "name", - "assert_blank", - "assert_not_same", - "is_a?", - "test_add_table_with_decimals", - "test_create_table_with_timestamps_should_create_datetime_columns", - "assert_present", - "assert_no_match", - "__instance_of__", - "assert_deprecated", - "assert", - "assert_throws", - "kind_of?", - "try", - "__instance_variable_get__", - "object_id", - "timeout", - "instance_variable_set", - "assert_nothing_thrown", - "__instance_variable_set__", - "copy_object", - "test_create_table_with_timestamps_should_create_datetime_columns_with_options", - "assert_not_deprecated", - "assert_in_delta", - "id", - "copy_metaclass", - "test_create_table_without_a_block", - "dup", - "assert_not_nil", - "send", - "__instance_variables__", - "to_sql", - "mock", - "assert_send", - "instance_variable_defined?", - "clone", - "require", - "test_migrator", - "__instance_variable_defined_eh__", - "frozen?", - "test_add_column_not_null_with_default", - "freeze", - "test_migrator_one_up", - "test_migrator_one_down", - "singleton_methods", - "method_exists?", - "create_fixtures", - "test_migrator_one_up_one_down", - "test_native_decimal_insert_manual_vs_automatic", - "instance_exec", - "__is_a__", - "test_migrator_double_up", - "stub", - "private_methods", - "stubs", - "test_migrator_double_down", - "fixture_path", - "private_singleton_methods", - "stub_everything", - "test_migrator_one_up_with_exception_and_rollback", - "sequence", - "protected_methods", - "enum_for", - "test_finds_migrations", - "run_before_mocha", - "states", - "protected_singleton_methods", - "to_json", - "instance_values", - "==", - "mocha_setup", - "public_methods", - "test_finds_pending_migrations", - "mocha_verify", - "assert_kind_of", - "===", - "=~", - "test_relative_migrations", - "mocha_teardown", - "gem", - "mocha", - "test_only_loads_pending_migrations", - "test_add_column_with_precision_and_scale", - "require_or_load", - "eql?", - "require_dependency", - "test_native_types", - "test_target_version_zero_should_run_only_once", - "extend", - "to_matcher", - "unloadable", - "require_association", - "hash", - "__id__", - "load_dependency", - "equals", - "test_migrator_db_has_no_schema_migrations_table", - "test_migrator_verbosity", - "kind_of", - "to_yaml", - "to_bool", - "test_migrator_verbosity_off", - "taint", - "test_migrator_going_down_due_to_version_target", - "tainted?", - "mocha_inspect", - "test_migrator_rollback", - "vim", - "untaint", - "taguri=", - "test_migrator_forward", - "test_schema_migrations_table_name", - "test_proper_table_name", - "all_of", - "test_add_drop_table_with_prefix_and_suffix", - "_setup_callbacks", - "setup", - "Not", - "test_create_table_with_binary_column", - "assert_not_equal", - "enable_warnings", - "acts_like?", - "Rational", - "_removed_setup_callbacks", - "Table", - "bind", - "any_of", - "__method__", - "test_migrator_with_duplicates", - "_teardown_callbacks", - "method", - "test_migrator_with_duplicate_names", - "_removed_teardown_callbacks", - "any_parameters", - "test_migrator_with_missing_version_numbers", - "test_add_remove_single_field_using_string_arguments", - "test_create_table_with_custom_sequence_name", - "test_add_remove_single_field_using_symbol_arguments", - "_one_time_conditions_valid_14?", - "_one_time_conditions_valid_16?", - "run_callbacks", - "anything", - "silence_warnings", - "instance_variable_names", - "_fixture_path", - "copy_instance_variables_from", - "fixture_path?", - "has_entry", - "__marshal__", - "_fixture_table_names", - "__kind_of__", - "fixture_table_names?", - "test_add_rename", - "assert_equal", - "_fixture_class_names", - "fixture_class_names?", - "has_entries", - "_use_transactional_fixtures", - "people", - "test_rename_column_using_symbol_arguments", - "use_transactional_fixtures?", - "instance_eval", - "blank?", - "with_warnings", - "__nil__", - "load", - "metaclass", - "_use_instantiated_fixtures", - "has_key", - "class_eval", - "present?", - "test_rename_column", - "teardown", - "use_instantiated_fixtures?", - "method_name", - "silence_stderr", - "presence", - "test_rename_column_preserves_default_value_not_null", - "silence_stream", - "_pre_loaded_fixtures", - "__metaclass__", - "__fixnum__", - "pre_loaded_fixtures?", - "has_value", - "suppress", - "to_yaml_properties", - "test_rename_nonexistent_column", - "test_add_index", - "includes", - "find_correlate_in", - "equality_predicate_sql", - "assert_nothing_raised", - "let", - "not_predicate_sql", - "test_rename_column_with_sql_reserved_word", - "singleton_class", - "test_rename_column_with_an_index", - "display", - "taguri", - "to_yaml_style", - "test_remove_column_with_index", - "size", - "current_adapter?", - "test_remove_column_with_multi_column_index", - "respond_to?", - "test_change_type_of_not_null_column", - "is_a", - "to_a", - "test_rename_table_for_sqlite_should_work_with_reserved_words", - "require_library_or_gem", - "setup_fixtures", - "equal?", - "teardown_fixtures", - "nil?", - "fixture_table_names", - "fixture_class_names", - "test_create_table_without_id", - "use_transactional_fixtures", - "test_add_column_with_primary_key_attribute", - "repair_validations", - "use_instantiated_fixtures", - "instance_of?", - "test_create_table_adds_id", - "test_rename_table", - "pre_loaded_fixtures", - "to_enum", - "test_create_table_with_not_null_column", - "instance_of", - "test_change_column_nullability", - "optionally", - "test_rename_table_with_an_index", - "run", - "test_change_column", - "default_test", - "assert_raise", - "test_create_table_with_defaults", - "assert_nil", - "flunk", - "regexp_matches", - "duplicable?", - "reset_mocha", - "stubba_method", - "filter_backtrace", - "test_create_table_with_limits", - "responds_with", - "stubba_object", - "test_change_column_with_nil_default", - "assert_block", - "__show__", - "assert_date_from_db", - "__respond_to_eh__", - "run_in_transaction?", - "inspect", - "assert_sql", - "test_change_column_with_new_default", - "yaml_equivalent", - "build_message", - "to_s", - "test_change_column_default", - "assert_queries", - "pending", - "as_json", - "assert_no_queries", - "test_change_column_quotes_column_names", - "assert_match", - "test_keeping_default_and_notnull_constraint_on_change", - "methods", - "connection_allow_concurrency_setup", - "connection_allow_concurrency_teardown", - "test_create_table_with_primary_key_prefix_as_table_name_with_underscore", - "__send__", - "make_connection", - "assert_raises", - "tap", - "with_kcode", - "assert_instance_of", - "test_create_table_with_primary_key_prefix_as_table_name", - "assert_respond_to", - "test_change_column_default_to_null", - "assert_same", - "__extend__"] - - LargeTestArraySorted = ["test_add_column_not_null_with_default", - "test_add_column_with_precision_and_scale", - "test_add_column_with_primary_key_attribute", - "test_add_drop_table_with_prefix_and_suffix", - "test_add_index", - "test_add_remove_single_field_using_string_arguments", - "test_add_remove_single_field_using_symbol_arguments", - "test_add_rename", - "test_add_table", - "test_add_table_with_decimals", - "test_change_column", - "test_change_column_default", - "test_change_column_default_to_null", - "test_change_column_nullability", - "test_change_column_quotes_column_names", - "test_change_column_with_new_default", - "test_change_column_with_nil_default", - "test_change_type_of_not_null_column", - "test_create_table_adds_id", - "test_create_table_with_binary_column", - "test_create_table_with_custom_sequence_name", - "test_create_table_with_defaults", - "test_create_table_with_force_true_does_not_drop_nonexisting_table", - "test_create_table_with_limits", - "test_create_table_with_not_null_column", - "test_create_table_with_primary_key_prefix_as_table_name", - "test_create_table_with_primary_key_prefix_as_table_name_with_underscore", - "test_create_table_with_timestamps_should_create_datetime_columns", - "test_create_table_with_timestamps_should_create_datetime_columns_with_options", - "test_create_table_without_a_block", - "test_create_table_without_id", - "test_finds_migrations", - "test_finds_pending_migrations", - "test_keeping_default_and_notnull_constraint_on_change", - "test_migrator", - "test_migrator_db_has_no_schema_migrations_table", - "test_migrator_double_down", - "test_migrator_double_up", - "test_migrator_forward", - "test_migrator_going_down_due_to_version_target", - "test_migrator_one_down", - "test_migrator_one_up", - "test_migrator_one_up_one_down", - "test_migrator_one_up_with_exception_and_rollback", - "test_migrator_rollback", - "test_migrator_verbosity", - "test_migrator_verbosity_off", - "test_migrator_with_duplicate_names", - "test_migrator_with_duplicates", - "test_migrator_with_missing_version_numbers", - "test_native_decimal_insert_manual_vs_automatic", - "test_native_types", - "test_only_loads_pending_migrations", - "test_proper_table_name", - "test_relative_migrations", - "test_remove_column_with_index", - "test_remove_column_with_multi_column_index", - "test_rename_column", - "test_rename_column_preserves_default_value_not_null", - "test_rename_column_using_symbol_arguments", - "test_rename_column_with_an_index", - "test_rename_column_with_sql_reserved_word", - "test_rename_nonexistent_column", - "test_rename_table", - "test_rename_table_for_sqlite_should_work_with_reserved_words", - "test_rename_table_with_an_index", - "test_schema_migrations_table_name", - "test_target_version_zero_should_run_only_once"] + LargeArray = [ + "test_create_table_with_force_true_does_not_drop_nonexisting_table", + "test_add_table", + "assert_difference", + "assert_operator", + "instance_variables", + "class", + "instance_variable_get", + "__class__", + "expects", + "assert_no_difference", + "name", + "assert_blank", + "assert_not_same", + "is_a?", + "test_add_table_with_decimals", + "test_create_table_with_timestamps_should_create_datetime_columns", + "assert_present", + "assert_no_match", + "__instance_of__", + "assert_deprecated", + "assert", + "assert_throws", + "kind_of?", + "try", + "__instance_variable_get__", + "object_id", + "timeout", + "instance_variable_set", + "assert_nothing_thrown", + "__instance_variable_set__", + "copy_object", + "test_create_table_with_timestamps_should_create_datetime_columns_with_options", + "assert_not_deprecated", + "assert_in_delta", + "id", + "copy_metaclass", + "test_create_table_without_a_block", + "dup", + "assert_not_nil", + "send", + "__instance_variables__", + "to_sql", + "mock", + "assert_send", + "instance_variable_defined?", + "clone", + "require", + "test_migrator", + "__instance_variable_defined_eh__", + "frozen?", + "test_add_column_not_null_with_default", + "freeze", + "test_migrator_one_up", + "test_migrator_one_down", + "singleton_methods", + "method_exists?", + "create_fixtures", + "test_migrator_one_up_one_down", + "test_native_decimal_insert_manual_vs_automatic", + "instance_exec", + "__is_a__", + "test_migrator_double_up", + "stub", + "private_methods", + "stubs", + "test_migrator_double_down", + "fixture_path", + "private_singleton_methods", + "stub_everything", + "test_migrator_one_up_with_exception_and_rollback", + "sequence", + "protected_methods", + "enum_for", + "test_finds_migrations", + "run_before_mocha", + "states", + "protected_singleton_methods", + "to_json", + "instance_values", + "==", + "mocha_setup", + "public_methods", + "test_finds_pending_migrations", + "mocha_verify", + "assert_kind_of", + "===", + "=~", + "test_relative_migrations", + "mocha_teardown", + "gem", + "mocha", + "test_only_loads_pending_migrations", + "test_add_column_with_precision_and_scale", + "require_or_load", + "eql?", + "require_dependency", + "test_native_types", + "test_target_version_zero_should_run_only_once", + "extend", + "to_matcher", + "unloadable", + "require_association", + "hash", + "__id__", + "load_dependency", + "equals", + "test_migrator_db_has_no_schema_migrations_table", + "test_migrator_verbosity", + "kind_of", + "to_yaml", + "to_bool", + "test_migrator_verbosity_off", + "taint", + "test_migrator_going_down_due_to_version_target", + "tainted?", + "mocha_inspect", + "test_migrator_rollback", + "vim", + "untaint", + "taguri=", + "test_migrator_forward", + "test_schema_migrations_table_name", + "test_proper_table_name", + "all_of", + "test_add_drop_table_with_prefix_and_suffix", + "_setup_callbacks", + "setup", + "Not", + "test_create_table_with_binary_column", + "assert_not_equal", + "enable_warnings", + "acts_like?", + "Rational", + "_removed_setup_callbacks", + "Table", + "bind", + "any_of", + "__method__", + "test_migrator_with_duplicates", + "_teardown_callbacks", + "method", + "test_migrator_with_duplicate_names", + "_removed_teardown_callbacks", + "any_parameters", + "test_migrator_with_missing_version_numbers", + "test_add_remove_single_field_using_string_arguments", + "test_create_table_with_custom_sequence_name", + "test_add_remove_single_field_using_symbol_arguments", + "_one_time_conditions_valid_14?", + "_one_time_conditions_valid_16?", + "run_callbacks", + "anything", + "silence_warnings", + "instance_variable_names", + "_fixture_path", + "copy_instance_variables_from", + "fixture_path?", + "has_entry", + "__marshal__", + "_fixture_table_names", + "__kind_of__", + "fixture_table_names?", + "test_add_rename", + "assert_equal", + "_fixture_class_names", + "fixture_class_names?", + "has_entries", + "_use_transactional_fixtures", + "people", + "test_rename_column_using_symbol_arguments", + "use_transactional_fixtures?", + "instance_eval", + "blank?", + "with_warnings", + "__nil__", + "load", + "metaclass", + "_use_instantiated_fixtures", + "has_key", + "class_eval", + "present?", + "test_rename_column", + "teardown", + "use_instantiated_fixtures?", + "method_name", + "silence_stderr", + "presence", + "test_rename_column_preserves_default_value_not_null", + "silence_stream", + "_pre_loaded_fixtures", + "__metaclass__", + "__fixnum__", + "pre_loaded_fixtures?", + "has_value", + "suppress", + "to_yaml_properties", + "test_rename_nonexistent_column", + "test_add_index", + "includes", + "find_correlate_in", + "equality_predicate_sql", + "assert_nothing_raised", + "let", + "not_predicate_sql", + "test_rename_column_with_sql_reserved_word", + "singleton_class", + "test_rename_column_with_an_index", + "display", + "taguri", + "to_yaml_style", + "test_remove_column_with_index", + "size", + "current_adapter?", + "test_remove_column_with_multi_column_index", + "respond_to?", + "test_change_type_of_not_null_column", + "is_a", + "to_a", + "test_rename_table_for_sqlite_should_work_with_reserved_words", + "require_library_or_gem", + "setup_fixtures", + "equal?", + "teardown_fixtures", + "nil?", + "fixture_table_names", + "fixture_class_names", + "test_create_table_without_id", + "use_transactional_fixtures", + "test_add_column_with_primary_key_attribute", + "repair_validations", + "use_instantiated_fixtures", + "instance_of?", + "test_create_table_adds_id", + "test_rename_table", + "pre_loaded_fixtures", + "to_enum", + "test_create_table_with_not_null_column", + "instance_of", + "test_change_column_nullability", + "optionally", + "test_rename_table_with_an_index", + "run", + "test_change_column", + "default_test", + "assert_raise", + "test_create_table_with_defaults", + "assert_nil", + "flunk", + "regexp_matches", + "duplicable?", + "reset_mocha", + "stubba_method", + "filter_backtrace", + "test_create_table_with_limits", + "responds_with", + "stubba_object", + "test_change_column_with_nil_default", + "assert_block", + "__show__", + "assert_date_from_db", + "__respond_to_eh__", + "run_in_transaction?", + "inspect", + "assert_sql", + "test_change_column_with_new_default", + "yaml_equivalent", + "build_message", + "to_s", + "test_change_column_default", + "assert_queries", + "pending", + "as_json", + "assert_no_queries", + "test_change_column_quotes_column_names", + "assert_match", + "test_keeping_default_and_notnull_constraint_on_change", + "methods", + "connection_allow_concurrency_setup", + "connection_allow_concurrency_teardown", + "test_create_table_with_primary_key_prefix_as_table_name_with_underscore", + "__send__", + "make_connection", + "assert_raises", + "tap", + "with_kcode", + "assert_instance_of", + "test_create_table_with_primary_key_prefix_as_table_name", + "assert_respond_to", + "test_change_column_default_to_null", + "assert_same", + "__extend__", + ] + + LargeTestArraySorted = [ + "test_add_column_not_null_with_default", + "test_add_column_with_precision_and_scale", + "test_add_column_with_primary_key_attribute", + "test_add_drop_table_with_prefix_and_suffix", + "test_add_index", + "test_add_remove_single_field_using_string_arguments", + "test_add_remove_single_field_using_symbol_arguments", + "test_add_rename", + "test_add_table", + "test_add_table_with_decimals", + "test_change_column", + "test_change_column_default", + "test_change_column_default_to_null", + "test_change_column_nullability", + "test_change_column_quotes_column_names", + "test_change_column_with_new_default", + "test_change_column_with_nil_default", + "test_change_type_of_not_null_column", + "test_create_table_adds_id", + "test_create_table_with_binary_column", + "test_create_table_with_custom_sequence_name", + "test_create_table_with_defaults", + "test_create_table_with_force_true_does_not_drop_nonexisting_table", + "test_create_table_with_limits", + "test_create_table_with_not_null_column", + "test_create_table_with_primary_key_prefix_as_table_name", + "test_create_table_with_primary_key_prefix_as_table_name_with_underscore", + "test_create_table_with_timestamps_should_create_datetime_columns", + "test_create_table_with_timestamps_should_create_datetime_columns_with_options", + "test_create_table_without_a_block", + "test_create_table_without_id", + "test_finds_migrations", + "test_finds_pending_migrations", + "test_keeping_default_and_notnull_constraint_on_change", + "test_migrator", + "test_migrator_db_has_no_schema_migrations_table", + "test_migrator_double_down", + "test_migrator_double_up", + "test_migrator_forward", + "test_migrator_going_down_due_to_version_target", + "test_migrator_one_down", + "test_migrator_one_up", + "test_migrator_one_up_one_down", + "test_migrator_one_up_with_exception_and_rollback", + "test_migrator_rollback", + "test_migrator_verbosity", + "test_migrator_verbosity_off", + "test_migrator_with_duplicate_names", + "test_migrator_with_duplicates", + "test_migrator_with_missing_version_numbers", + "test_native_decimal_insert_manual_vs_automatic", + "test_native_types", + "test_only_loads_pending_migrations", + "test_proper_table_name", + "test_relative_migrations", + "test_remove_column_with_index", + "test_remove_column_with_multi_column_index", + "test_rename_column", + "test_rename_column_preserves_default_value_not_null", + "test_rename_column_using_symbol_arguments", + "test_rename_column_with_an_index", + "test_rename_column_with_sql_reserved_word", + "test_rename_nonexistent_column", + "test_rename_table", + "test_rename_table_for_sqlite_should_work_with_reserved_words", + "test_rename_table_with_an_index", + "test_schema_migrations_table_name", + "test_target_version_zero_should_run_only_once", + ] class PrivateToAry private diff --git a/spec/ruby/core/array/pack/buffer_spec.rb b/spec/ruby/core/array/pack/buffer_spec.rb index f1206efb3e..b77b2d1efa 100644 --- a/spec/ruby/core/array/pack/buffer_spec.rb +++ b/spec/ruby/core/array/pack/buffer_spec.rb @@ -28,6 +28,16 @@ describe "Array#pack with :buffer option" do TypeError, "buffer must be String, not Array") end + it "raise FrozenError if buffer is frozen" do + -> { [65].pack("c", buffer: "frozen-string".freeze) }.should raise_error(FrozenError) + end + + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + [65, 66, 67].pack("ccc", buffer: buffer) + buffer.encoding.should == Encoding::ISO_8859_1 + end + context "offset (@) is specified" do it 'keeps buffer content if it is longer than offset' do n = [ 65, 66, 67 ] diff --git a/spec/ruby/core/binding/dup_spec.rb b/spec/ruby/core/binding/dup_spec.rb index 55fac6e333..4eff66bd9a 100644 --- a/spec/ruby/core/binding/dup_spec.rb +++ b/spec/ruby/core/binding/dup_spec.rb @@ -10,4 +10,21 @@ describe "Binding#dup" do bind.frozen?.should == true bind.dup.frozen?.should == false end + + it "retains original binding variables but the list is distinct" do + bind1 = binding + eval "a = 1", bind1 + + bind2 = bind1.dup + eval("a = 2", bind2) + eval("a", bind1).should == 2 + eval("a", bind2).should == 2 + + eval("b = 2", bind2) + -> { eval("b", bind1) }.should raise_error(NameError) + eval("b", bind2).should == 2 + + bind1.local_variables.sort.should == [:a, :bind1, :bind2] + bind2.local_variables.sort.should == [:a, :b, :bind1, :bind2] + end end diff --git a/spec/ruby/core/enumerator/next_values_spec.rb b/spec/ruby/core/enumerator/next_values_spec.rb index 201b5d323f..2202700c58 100644 --- a/spec/ruby/core/enumerator/next_values_spec.rb +++ b/spec/ruby/core/enumerator/next_values_spec.rb @@ -11,6 +11,7 @@ describe "Enumerator#next_values" do yield :e1, :e2, :e3 yield nil yield + yield [:f1, :f2] end @e = o.to_enum @@ -48,8 +49,13 @@ describe "Enumerator#next_values" do @e.next_values.should == [] end - it "raises StopIteration if called on a finished enumerator" do + it "returns an array of array if yield is called with an array" do 7.times { @e.next } + @e.next_values.should == [[:f1, :f2]] + end + + it "raises StopIteration if called on a finished enumerator" do + 8.times { @e.next } -> { @e.next_values }.should raise_error(StopIteration) end end diff --git a/spec/ruby/core/enumerator/peek_values_spec.rb b/spec/ruby/core/enumerator/peek_values_spec.rb index 7865546515..8b84fc8afc 100644 --- a/spec/ruby/core/enumerator/peek_values_spec.rb +++ b/spec/ruby/core/enumerator/peek_values_spec.rb @@ -11,6 +11,7 @@ describe "Enumerator#peek_values" do yield :e1, :e2, :e3 yield nil yield + yield [:f1, :f2] end @e = o.to_enum @@ -50,8 +51,13 @@ describe "Enumerator#peek_values" do @e.peek_values.should == [] end - it "raises StopIteration if called on a finished enumerator" do + it "returns an array of array if yield is called with an array" do 7.times { @e.next } + @e.peek_values.should == [[:f1, :f2]] + end + + it "raises StopIteration if called on a finished enumerator" do + 8.times { @e.next } -> { @e.peek_values }.should raise_error(StopIteration) end end diff --git a/spec/ruby/core/io/pread_spec.rb b/spec/ruby/core/io/pread_spec.rb index 28afc80e5c..6d93b432c2 100644 --- a/spec/ruby/core/io/pread_spec.rb +++ b/spec/ruby/core/io/pread_spec.rb @@ -22,7 +22,7 @@ guard -> { platform_is_not :windows or ruby_version_is "3.3" } do it "accepts a length, an offset, and an output buffer" do buffer = +"foo" - @file.pread(3, 4, buffer) + @file.pread(3, 4, buffer).should.equal?(buffer) buffer.should == "567" end @@ -38,6 +38,13 @@ guard -> { platform_is_not :windows or ruby_version_is "3.3" } do buffer.should == "12345" end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @file.pread(10, 0, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + it "does not advance the file pointer" do @file.pread(4, 0).should == "1234" @file.read.should == "1234567890" diff --git a/spec/ruby/core/io/read_spec.rb b/spec/ruby/core/io/read_spec.rb index eb3652e692..8741d9f017 100644 --- a/spec/ruby/core/io/read_spec.rb +++ b/spec/ruby/core/io/read_spec.rb @@ -376,6 +376,21 @@ describe "IO#read" do buf.should == @contents[0..4] end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.read(10, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + + # https://bugs.ruby-lang.org/issues/20416 + it "does not preserve the encoding of the given buffer when max length is not provided" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.read(nil, buffer) + + buffer.encoding.should_not == Encoding::ISO_8859_1 + end + it "returns the given buffer" do buf = +"" diff --git a/spec/ruby/core/io/readpartial_spec.rb b/spec/ruby/core/io/readpartial_spec.rb index 0060beb545..0852f20b2d 100644 --- a/spec/ruby/core/io/readpartial_spec.rb +++ b/spec/ruby/core/io/readpartial_spec.rb @@ -62,7 +62,7 @@ describe "IO#readpartial" do buffer = +"existing content" @wr.write("hello world") @wr.close - @rd.readpartial(11, buffer) + @rd.readpartial(11, buffer).should.equal?(buffer) buffer.should == "hello world" end @@ -106,6 +106,7 @@ describe "IO#readpartial" do @wr.write("abc") @wr.close @rd.readpartial(10, buffer) + buffer.encoding.should == Encoding::ISO_8859_1 end end diff --git a/spec/ruby/core/io/sysread_spec.rb b/spec/ruby/core/io/sysread_spec.rb index 003bb9eb94..8851214283 100644 --- a/spec/ruby/core/io/sysread_spec.rb +++ b/spec/ruby/core/io/sysread_spec.rb @@ -97,7 +97,7 @@ describe "IO#sysread on a file" do it "discards the existing buffer content upon successful read" do buffer = +"existing content" - @file.sysread(11, buffer) + @file.sysread(11, buffer).should.equal?(buffer) buffer.should == "01234567890" end @@ -107,6 +107,13 @@ describe "IO#sysread on a file" do -> { @file.sysread(1, buffer) }.should raise_error(EOFError) buffer.should be_empty end + + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + string = @file.sysread(10, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end end describe "IO#sysread" do diff --git a/spec/ruby/core/module/include_spec.rb b/spec/ruby/core/module/include_spec.rb index c073bc31ca..78f6b41031 100644 --- a/spec/ruby/core/module/include_spec.rb +++ b/spec/ruby/core/module/include_spec.rb @@ -47,6 +47,34 @@ describe "Module#include" do -> { ModuleSpecs::SubclassSpec.include(ModuleSpecs::Subclass.new) }.should_not raise_error(TypeError) end + ruby_version_is ""..."3.2" do + it "raises ArgumentError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.include(refinement) }.should raise_error(ArgumentError, "refinement module is not allowed") + end + end + + ruby_version_is "3.2" do + it "raises a TypeError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.include(refinement) }.should raise_error(TypeError, "Cannot include refinement") + end + end + it "imports constants to modules and classes" do ModuleSpecs::A.constants.should include(:CONSTANT_A) ModuleSpecs::B.constants.should include(:CONSTANT_A, :CONSTANT_B) diff --git a/spec/ruby/core/module/prepend_spec.rb b/spec/ruby/core/module/prepend_spec.rb index c90fa9700e..b40d12f0de 100644 --- a/spec/ruby/core/module/prepend_spec.rb +++ b/spec/ruby/core/module/prepend_spec.rb @@ -75,6 +75,26 @@ describe "Module#prepend" do foo.call.should == 'm' end + it "updates the optimized method when a prepended module is updated" do + out = ruby_exe(<<~RUBY) + module M; end + class Integer + prepend M + end + l = -> { 1 + 2 } + p l.call + M.module_eval do + def +(o) + $called = true + super(o) + end + end + p l.call + p $called + RUBY + out.should == "3\n3\ntrue\n" + end + it "updates the method when there is a base included method and the prepended module overrides it" do base_module = Module.new do def foo @@ -415,6 +435,34 @@ describe "Module#prepend" do -> { ModuleSpecs::SubclassSpec.prepend(ModuleSpecs::Subclass.new) }.should_not raise_error(TypeError) end + ruby_version_is ""..."3.2" do + it "raises ArgumentError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.prepend(refinement) }.should raise_error(ArgumentError, "refinement module is not allowed") + end + end + + ruby_version_is "3.2" do + it "raises a TypeError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.prepend(refinement) }.should raise_error(TypeError, "Cannot prepend refinement") + end + end + it "imports constants" do m1 = Module.new m1::MY_CONSTANT = 1 diff --git a/spec/ruby/core/string/chilled_string_spec.rb b/spec/ruby/core/string/chilled_string_spec.rb index 8de4fc421b..9f81b1af6d 100644 --- a/spec/ruby/core/string/chilled_string_spec.rb +++ b/spec/ruby/core/string/chilled_string_spec.rb @@ -3,8 +3,8 @@ require_relative '../../spec_helper' describe "chilled String" do guard -> { ruby_version_is "3.4" and !"test".equal?("test") } do describe "#frozen?" do - it "returns true" do - "chilled".frozen?.should == true + it "returns false" do + "chilled".frozen?.should == false end end @@ -45,7 +45,7 @@ describe "chilled String" do input.should == "chilled-mutated" end - it "emits a warning on singleton_class creaation" do + it "emits a warning on singleton_class creation" do -> { "chilled".singleton_class }.should complain(/literal string will be frozen in the future/) @@ -57,12 +57,14 @@ describe "chilled String" do }.should complain(/literal string will be frozen in the future/) end - it "raises FrozenError after the string was explictly frozen" do + it "raises FrozenError after the string was explicitly frozen" do input = "chilled" input.freeze -> { + -> { input << "mutated" - }.should raise_error(FrozenError) + }.should raise_error(FrozenError) + }.should_not complain(/literal string will be frozen in the future/) end end end diff --git a/spec/ruby/core/string/index_spec.rb b/spec/ruby/core/string/index_spec.rb index be79708045..835263a2cd 100644 --- a/spec/ruby/core/string/index_spec.rb +++ b/spec/ruby/core/string/index_spec.rb @@ -231,6 +231,17 @@ describe "String#index with Regexp" do $~.should == nil end + ruby_bug "#20421", ""..."3.3" do + it "always clear $~" do + "a".index(/a/) + $~.should_not == nil + + string = "blablabla" + string.index(/bla/, string.length + 1) + $~.should == nil + end + end + it "starts the search at the given offset" do "blablabla".index(/.{0}/, 5).should == 5 "blablabla".index(/.{1}/, 5).should == 5 diff --git a/spec/ruby/core/string/modulo_spec.rb b/spec/ruby/core/string/modulo_spec.rb index bf96a82874..9045afa263 100644 --- a/spec/ruby/core/string/modulo_spec.rb +++ b/spec/ruby/core/string/modulo_spec.rb @@ -55,33 +55,48 @@ describe "String#%" do -> { ("foo%" % [])}.should raise_error(ArgumentError) end - it "formats single % character before a newline as literal %" do - ("%\n" % []).should == "%\n" - ("foo%\n" % []).should == "foo%\n" - ("%\n.3f" % 1.2).should == "%\n.3f" - end + ruby_version_is ""..."3.4" do + it "formats single % character before a newline as literal %" do + ("%\n" % []).should == "%\n" + ("foo%\n" % []).should == "foo%\n" + ("%\n.3f" % 1.2).should == "%\n.3f" + end - it "formats single % character before a NUL as literal %" do - ("%\0" % []).should == "%\0" - ("foo%\0" % []).should == "foo%\0" - ("%\0.3f" % 1.2).should == "%\0.3f" - end + it "formats single % character before a NUL as literal %" do + ("%\0" % []).should == "%\0" + ("foo%\0" % []).should == "foo%\0" + ("%\0.3f" % 1.2).should == "%\0.3f" + end - it "raises an error if single % appears anywhere else" do - -> { (" % " % []) }.should raise_error(ArgumentError) - -> { ("foo%quux" % []) }.should raise_error(ArgumentError) - end + it "raises an error if single % appears anywhere else" do + -> { (" % " % []) }.should raise_error(ArgumentError) + -> { ("foo%quux" % []) }.should raise_error(ArgumentError) + end - it "raises an error if NULL or \\n appear anywhere else in the format string" do - begin - old_debug, $DEBUG = $DEBUG, false + it "raises an error if NULL or \\n appear anywhere else in the format string" do + begin + old_debug, $DEBUG = $DEBUG, false + -> { "%.\n3f" % 1.2 }.should raise_error(ArgumentError) + -> { "%.3\nf" % 1.2 }.should raise_error(ArgumentError) + -> { "%.\03f" % 1.2 }.should raise_error(ArgumentError) + -> { "%.3\0f" % 1.2 }.should raise_error(ArgumentError) + ensure + $DEBUG = old_debug + end + end + end + + ruby_version_is "3.4" do + it "raises an ArgumentError if % is not followed by a conversion specifier" do + -> { "%" % [] }.should raise_error(ArgumentError) + -> { "%\n" % [] }.should raise_error(ArgumentError) + -> { "%\0" % [] }.should raise_error(ArgumentError) + -> { " % " % [] }.should raise_error(ArgumentError) -> { "%.\n3f" % 1.2 }.should raise_error(ArgumentError) -> { "%.3\nf" % 1.2 }.should raise_error(ArgumentError) -> { "%.\03f" % 1.2 }.should raise_error(ArgumentError) -> { "%.3\0f" % 1.2 }.should raise_error(ArgumentError) - ensure - $DEBUG = old_debug end end @@ -125,8 +140,16 @@ describe "String#%" do end end - it "replaces trailing absolute argument specifier without type with percent sign" do - ("hello %1$" % "foo").should == "hello %" + ruby_version_is ""..."3.4" do + it "replaces trailing absolute argument specifier without type with percent sign" do + ("hello %1$" % "foo").should == "hello %" + end + end + + ruby_version_is "3.4" do + it "raises an ArgumentError if absolute argument specifier is followed by a conversion specifier" do + -> { "hello %1$" % "foo" }.should raise_error(ArgumentError) + end end it "raises an ArgumentError when given invalid argument specifiers" do diff --git a/spec/ruby/core/tracepoint/inspect_spec.rb b/spec/ruby/core/tracepoint/inspect_spec.rb index cc6bf0f842..0c94a94d5c 100644 --- a/spec/ruby/core/tracepoint/inspect_spec.rb +++ b/spec/ruby/core/tracepoint/inspect_spec.rb @@ -24,6 +24,8 @@ describe 'TracePoint#inspect' do line = nil TracePoint.new(:line) { |tp| next unless TracePointSpec.target_thread? + next unless tp.path == __FILE__ + inspect ||= tp.inspect }.enable do line = __LINE__ @@ -37,6 +39,8 @@ describe 'TracePoint#inspect' do line = nil TracePoint.new(:call) { |tp| next unless TracePointSpec.target_thread? + next unless tp.path == __FILE__ + inspect ||= tp.inspect }.enable do line = __LINE__ + 1 @@ -52,6 +56,8 @@ describe 'TracePoint#inspect' do line = nil TracePoint.new(:return) { |tp| next unless TracePointSpec.target_thread? + next unless tp.path == __FILE__ + inspect ||= tp.inspect }.enable do line = __LINE__ + 4 @@ -69,6 +75,8 @@ describe 'TracePoint#inspect' do inspect = nil tracepoint = TracePoint.new(:c_call) { |tp| next unless TracePointSpec.target_thread? + next unless tp.path == __FILE__ + inspect ||= tp.inspect } line = __LINE__ + 2 @@ -84,6 +92,8 @@ describe 'TracePoint#inspect' do line = nil TracePoint.new(:class) { |tp| next unless TracePointSpec.target_thread? + next unless tp.path == __FILE__ + inspect ||= tp.inspect }.enable do line = __LINE__ + 1 @@ -100,6 +110,7 @@ describe 'TracePoint#inspect' do thread_inspection = nil TracePoint.new(:thread_begin) { |tp| next unless Thread.current == thread + inspect ||= tp.inspect }.enable(target_thread: nil) do thread = Thread.new {} @@ -116,6 +127,7 @@ describe 'TracePoint#inspect' do thread_inspection = nil TracePoint.new(:thread_end) { |tp| next unless Thread.current == thread + inspect ||= tp.inspect }.enable(target_thread: nil) do thread = Thread.new {} diff --git a/spec/ruby/core/warning/performance_warning_spec.rb b/spec/ruby/core/warning/performance_warning_spec.rb new file mode 100644 index 0000000000..ab0badcd3d --- /dev/null +++ b/spec/ruby/core/warning/performance_warning_spec.rb @@ -0,0 +1,28 @@ +require_relative '../../spec_helper' + + +describe "Performance warnings" do + guard -> { ruby_version_is("3.4") || RUBY_ENGINE == "truffleruby" } do + # Optimising Integer, Float or Symbol methods is kind of implementation detail + # but multiple implementations do so. So it seems reasonable to have a test case + # for at least one such common method. + # See https://bugs.ruby-lang.org/issues/20429 + context "when redefined optimised methods" do + it "emits performance warning for redefining Integer#+" do + code = <<~CODE + Warning[:performance] = true + + class Integer + ORIG_METHOD = instance_method(:+) + + def +(...) + ORIG_METHOD.bind(self).call(...) + end + end + CODE + + ruby_exe(code, args: "2>&1").should.include?("warning: Redefining 'Integer#+' disables interpreter and JIT optimizations") + end + end + end +end diff --git a/spec/ruby/library/io-wait/fixtures/classes.rb b/spec/ruby/fixtures/io.rb index 837c7edd06..87ebbbb2bd 100644 --- a/spec/ruby/library/io-wait/fixtures/classes.rb +++ b/spec/ruby/fixtures/io.rb @@ -1,12 +1,12 @@ -module IOWaitSpec +module IOSpec def self.exhaust_write_buffer(io) written = 0 buf = " " * 4096 - begin + while true written += io.write_nonblock(buf) - rescue Errno::EAGAIN, Errno::EWOULDBLOCK - return written - end while true + end + rescue Errno::EAGAIN, Errno::EWOULDBLOCK + written end end diff --git a/spec/ruby/language/block_spec.rb b/spec/ruby/language/block_spec.rb index 578d9cb3b0..cf0931b688 100644 --- a/spec/ruby/language/block_spec.rb +++ b/spec/ruby/language/block_spec.rb @@ -960,24 +960,27 @@ describe "Post-args" do end describe "with a circular argument reference" do - it "raises a SyntaxError if using an existing local with the same name as the argument" do - a = 1 - -> { - @proc = eval "proc { |a=a| a }" - }.should raise_error(SyntaxError) + ruby_version_is ""..."3.4" do + it "raises a SyntaxError if using the argument in its default value" do + a = 1 + -> { + eval "proc { |a=a| a }" + }.should raise_error(SyntaxError) + end end - it "raises a SyntaxError if there is an existing method with the same name as the argument" do - def a; 1; end - -> { - @proc = eval "proc { |a=a| a }" - }.should raise_error(SyntaxError) + ruby_version_is "3.4" do + it "is nil if using the argument in its default value" do + -> { + eval "proc { |a=a| a }.call" + }.call.should == nil + end end + end - it "calls an existing method with the same name as the argument if explicitly using ()" do - def a; 1; end - proc { |a=a()| a }.call.should == 1 - end + it "calls an existing method with the same name as the argument if explicitly using ()" do + def a; 1; end + proc { |a=a()| a }.call.should == 1 end end diff --git a/spec/ruby/language/break_spec.rb b/spec/ruby/language/break_spec.rb index e725e77e80..7e5b6fb328 100644 --- a/spec/ruby/language/break_spec.rb +++ b/spec/ruby/language/break_spec.rb @@ -252,6 +252,25 @@ describe "Break inside a while loop" do end end +describe "The break statement in a method" do + it "is invalid and raises a SyntaxError" do + -> { + eval("def m; break; end") + }.should raise_error(SyntaxError) + end +end + +describe "The break statement in a module literal" do + it "is invalid and raises a SyntaxError" do + code = <<~RUBY + module BreakSpecs:ModuleWithBreak + break + end + RUBY + + -> { eval(code) }.should raise_error(SyntaxError) + end +end # TODO: Rewrite all the specs from here to the end of the file in the style # above. diff --git a/spec/ruby/language/case_spec.rb b/spec/ruby/language/case_spec.rb index 3262f09dd5..464d06e46a 100644 --- a/spec/ruby/language/case_spec.rb +++ b/spec/ruby/language/case_spec.rb @@ -416,17 +416,34 @@ describe "The 'case'-construct" do self.test(true).should == true end - it "warns if there are identical when clauses" do - -> { - eval <<~RUBY - case 1 - when 2 - :foo - when 2 - :bar - end - RUBY - }.should complain(/warning: duplicated .when' clause with line \d+ is ignored/, verbose: true) + ruby_version_is ""..."3.4" do + it "warns if there are identical when clauses" do + -> { + eval <<~RUBY + case 1 + when 2 + :foo + when 2 + :bar + end + RUBY + }.should complain(/warning: (duplicated .when' clause with line \d+ is ignored|'when' clause on line \d+ duplicates 'when' clause on line \d+ and is ignored)/, verbose: true) + end + end + + ruby_version_is "3.4" do + it "warns if there are identical when clauses" do + -> { + eval <<~RUBY + case 1 + when 2 + :foo + when 2 + :bar + end + RUBY + }.should complain(/warning: 'when' clause on line \d+ duplicates 'when' clause on line \d+ and is ignored/, verbose: true) + end end end diff --git a/spec/ruby/language/def_spec.rb b/spec/ruby/language/def_spec.rb index 42e721c68c..ce8077eb69 100644 --- a/spec/ruby/language/def_spec.rb +++ b/spec/ruby/language/def_spec.rb @@ -197,15 +197,25 @@ describe "An instance method with a default argument" do foo(2,3,3).should == [2,3,[3]] end - it "raises a SyntaxError when there is an existing method with the same name as the local variable" do - def bar - 1 + ruby_version_is ""..."3.4" do + it "raises a SyntaxError if using the argument in its default value" do + -> { + eval "def foo(bar = bar) + bar + end" + }.should raise_error(SyntaxError) + end + end + + ruby_version_is "3.4" do + it "is nil if using the argument in its default value" do + -> { + eval "def foo(bar = bar) + bar + end + foo" + }.call.should == nil end - -> { - eval "def foo(bar = bar) - bar - end" - }.should raise_error(SyntaxError) end it "calls a method with the same name as the local when explicitly using ()" do diff --git a/spec/ruby/language/execution_spec.rb b/spec/ruby/language/execution_spec.rb index ef1de38899..51bcde62e8 100644 --- a/spec/ruby/language/execution_spec.rb +++ b/spec/ruby/language/execution_spec.rb @@ -38,7 +38,7 @@ describe "``" do 2.times do runner.instance_exec do - `test #{:command}` + `test #{:command}` # rubocop:disable Lint/LiteralInInterpolation end end @@ -84,7 +84,7 @@ describe "%x" do 2.times do runner.instance_exec do - %x{test #{:command}} + %x{test #{:command}} # rubocop:disable Lint/LiteralInInterpolation end end diff --git a/spec/ruby/language/hash_spec.rb b/spec/ruby/language/hash_spec.rb index a7631fb0d6..068ac0f39c 100644 --- a/spec/ruby/language/hash_spec.rb +++ b/spec/ruby/language/hash_spec.rb @@ -86,6 +86,30 @@ describe "Hash literal" do -> { eval("{:a ==> 1}") }.should raise_error(SyntaxError) end + it "recognizes '!' at the end of the key" do + eval("{:a! =>1}").should == {:"a!" => 1} + eval("{:a! => 1}").should == {:"a!" => 1} + + eval("{a!:1}").should == {:"a!" => 1} + eval("{a!: 1}").should == {:"a!" => 1} + end + + it "raises a SyntaxError if there is no space between `!` and `=>`" do + -> { eval("{:a!=> 1}") }.should raise_error(SyntaxError) + end + + it "recognizes '?' at the end of the key" do + eval("{:a? =>1}").should == {:"a?" => 1} + eval("{:a? => 1}").should == {:"a?" => 1} + + eval("{a?:1}").should == {:"a?" => 1} + eval("{a?: 1}").should == {:"a?" => 1} + end + + it "raises a SyntaxError if there is no space between `?` and `=>`" do + -> { eval("{:a?=> 1}") }.should raise_error(SyntaxError) + end + it "constructs a new hash with the given elements" do {foo: 123}.should == {foo: 123} h = {rbx: :cool, specs: 'fail_sometimes'} @@ -271,6 +295,14 @@ describe "The ** operator" do a.new.foo(1).should == {bar: "baz", val: 1} end + + it "raises a SyntaxError when the hash key ends with `!`" do + -> { eval("{a!:}") }.should raise_error(SyntaxError, /identifier a! is not valid to get/) + end + + it "raises a SyntaxError when the hash key ends with `?`" do + -> { eval("{a?:}") }.should raise_error(SyntaxError, /identifier a\? is not valid to get/) + end end end end diff --git a/spec/ruby/language/keyword_arguments_spec.rb b/spec/ruby/language/keyword_arguments_spec.rb index ffb5b1fab0..8668799d26 100644 --- a/spec/ruby/language/keyword_arguments_spec.rb +++ b/spec/ruby/language/keyword_arguments_spec.rb @@ -395,4 +395,32 @@ describe "Keyword arguments" do end end end + + context "in define_method(name, &proc)" do + # This tests that a free-standing proc used in define_method and converted to ruby2_keywords adopts that logic. + # See jruby/jruby#8119 for a case where aggressive JIT optimization broke later ruby2_keywords changes. + it "works with ruby2_keywords" do + m = Class.new do + def bar(a, foo: nil) + [a, foo] + end + + # define_method and ruby2_keywords using send to avoid peephole optimizations + def self.setup + pr = make_proc + send :define_method, :foo, &pr + send :ruby2_keywords, :foo + end + + # create proc in isolated method to force jit compilation on some implementations + def self.make_proc + proc { |a, *args| bar(a, *args) } + end + end + + m.setup + + m.new.foo(1, foo:2).should == [1, 2] + end + end end diff --git a/spec/ruby/language/lambda_spec.rb b/spec/ruby/language/lambda_spec.rb index 3ab3569ebe..ed5a1c69e8 100644 --- a/spec/ruby/language/lambda_spec.rb +++ b/spec/ruby/language/lambda_spec.rb @@ -263,18 +263,21 @@ describe "A lambda literal -> () { }" do end describe "with circular optional argument reference" do - it "raises a SyntaxError if using an existing local with the same name as the argument" do - a = 1 - -> { - @proc = eval "-> (a=a) { a }" - }.should raise_error(SyntaxError) + ruby_version_is ""..."3.4" do + it "raises a SyntaxError if using the argument in its default value" do + a = 1 + -> { + eval "-> (a=a) { a }" + }.should raise_error(SyntaxError) + end end - it "raises a SyntaxError if there is an existing method with the same name as the argument" do - def a; 1; end - -> { - @proc = eval "-> (a=a) { a }" - }.should raise_error(SyntaxError) + ruby_version_is "3.4" do + it "is nil if using the argument in its default value" do + -> { + eval "-> (a=a) { a }.call" + }.call.should == nil + end end it "calls an existing method with the same name as the argument if explicitly using ()" do diff --git a/spec/ruby/language/pattern_matching/3.1.rb b/spec/ruby/language/pattern_matching/3.1.rb new file mode 100644 index 0000000000..7a09084e41 --- /dev/null +++ b/spec/ruby/language/pattern_matching/3.1.rb @@ -0,0 +1,75 @@ +describe "Pattern matching" do + before :each do + ScratchPad.record [] + end + + describe "Ruby 3.1 improvements" do + ruby_version_is "3.1" do + it "can omit parentheses in one line pattern matching" do + [1, 2] => a, b + [a, b].should == [1, 2] + + {a: 1} => a: + a.should == 1 + end + + it "supports pinning instance variables" do + @a = /a/ + case 'abc' + in ^@a + true + end.should == true + end + + it "supports pinning class variables" do + result = nil + Module.new do + result = module_eval(<<~RUBY) + @@a = 0..10 + + case 2 + in ^@@a + true + end + RUBY + end + + result.should == true + end + + it "supports pinning global variables" do + $a = /a/ + case 'abc' + in ^$a + true + end.should == true + end + + it "supports pinning expressions" do + case 'abc' + in ^(/a/) + true + end.should == true + + case 0 + in ^(0 + 0) + true + end.should == true + end + + it "supports pinning expressions in array pattern" do + case [3] + in [^(1 + 2)] + true + end.should == true + end + + it "supports pinning expressions in hash pattern" do + case {name: '2.6', released_at: Time.new(2018, 12, 25)} + in {released_at: ^(Time.new(2010)..Time.new(2020))} + true + end.should == true + end + end + end +end diff --git a/spec/ruby/language/pattern_matching_spec.rb b/spec/ruby/language/pattern_matching_spec.rb index a8ec078cd0..8c5df06a17 100644 --- a/spec/ruby/language/pattern_matching_spec.rb +++ b/spec/ruby/language/pattern_matching_spec.rb @@ -1,9 +1,6 @@ require_relative '../spec_helper' describe "Pattern matching" do - # TODO: Remove excessive eval calls when Ruby 3 is the minimum version. - # It is best to keep the eval's longer if other Ruby impls cannot parse pattern matching yet. - before :each do ScratchPad.record [] end @@ -11,145 +8,119 @@ describe "Pattern matching" do describe "can be standalone assoc operator that" do it "deconstructs value" do suppress_warning do - eval(<<-RUBY).should == [0, 1] - [0, 1] => [a, b] - [a, b] - RUBY + [0, 1] => [a, b] + [a, b].should == [0, 1] end end it "deconstructs value and properly scopes variables" do suppress_warning do - eval(<<-RUBY).should == [0, nil] - a = nil - eval(<<-PATTERN) - [0, 1] => [a, b] - PATTERN - [a, defined?(b)] - RUBY + a = nil + 1.times { + [0, 1] => [a, b] + } + [a, defined?(b)].should == [0, nil] end end end describe "find pattern" do it "captures preceding elements to the pattern" do - eval(<<~RUBY).should == [0, 1] - case [0, 1, 2, 3] - in [*pre, 2, 3] - pre - else - false - end - RUBY + case [0, 1, 2, 3] + in [*pre, 2, 3] + pre + else + false + end.should == [0, 1] end it "captures following elements to the pattern" do - eval(<<~RUBY).should == [2, 3] - case [0, 1, 2, 3] - in [0, 1, *post] - post - else - false - end - RUBY + case [0, 1, 2, 3] + in [0, 1, *post] + post + else + false + end.should == [2, 3] end it "captures both preceding and following elements to the pattern" do - eval(<<~RUBY).should == [[0, 1], [3, 4]] - case [0, 1, 2, 3, 4] - in [*pre, 2, *post] - [pre, post] - else - false - end - RUBY + case [0, 1, 2, 3, 4] + in [*pre, 2, *post] + [pre, post] + else + false + end.should == [[0, 1], [3, 4]] end it "can capture the entirety of the pattern" do - eval(<<~RUBY).should == [0, 1, 2, 3, 4] - case [0, 1, 2, 3, 4] - in [*everything] - everything - else - false - end - RUBY + case [0, 1, 2, 3, 4] + in [*everything] + everything + else + false + end.should == [0, 1, 2, 3, 4] end it "will match an empty Array-like structure" do - eval(<<~RUBY).should == [] - case [] - in [*everything] - everything - else - false - end - RUBY + case [] + in [*everything] + everything + else + false + end.should == [] end it "can be nested" do - eval(<<~RUBY).should == [[0, [2, 4, 6]], [[4, 16, 64]], 27] - case [0, [2, 4, 6], [3, 9, 27], [4, 16, 64]] - in [*pre, [*, 9, a], *post] - [pre, post, a] - else - false - end - RUBY + case [0, [2, 4, 6], [3, 9, 27], [4, 16, 64]] + in [*pre, [*, 9, a], *post] + [pre, post, a] + else + false + end.should == [[0, [2, 4, 6]], [[4, 16, 64]], 27] end it "can be nested with an array pattern" do - eval(<<~RUBY).should == [[4, 16, 64]] - case [0, [2, 4, 6], [3, 9, 27], [4, 16, 64]] - in [_, _, [*, 9, *], *post] - post - else - false - end - RUBY + case [0, [2, 4, 6], [3, 9, 27], [4, 16, 64]] + in [_, _, [*, 9, *], *post] + post + else + false + end.should == [[4, 16, 64]] end it "can be nested within a hash pattern" do - eval(<<~RUBY).should == [27] - case {a: [3, 9, 27]} - in {a: [*, 9, *post]} - post - else - false - end - RUBY + case {a: [3, 9, 27]} + in {a: [*, 9, *post]} + post + else + false + end.should == [27] end it "can nest hash and array patterns" do - eval(<<~RUBY).should == [42, 2] - case [0, {a: 42, b: [0, 1]}, {a: 42, b: [1, 2]}] - in [*, {a:, b: [1, c]}, *] - [a, c] - else - false - end - RUBY + case [0, {a: 42, b: [0, 1]}, {a: 42, b: [1, 2]}] + in [*, {a:, b: [1, c]}, *] + [a, c] + else + false + end.should == [42, 2] end end it "extends case expression with case/in construction" do - eval(<<~RUBY).should == :bar - case [0, 1] - in [0] - :foo - in [0, 1] - :bar - end - RUBY + case [0, 1] + in [0] + :foo + in [0, 1] + :bar + end.should == :bar end it "allows using then operator" do - eval(<<~RUBY).should == :bar - case [0, 1] - in [0] then :foo - in [0, 1] then :bar - end - RUBY + case [0, 1] + in [0] then :foo + in [0, 1] then :bar + end.should == :bar end describe "warning" do @@ -191,12 +162,10 @@ describe "Pattern matching" do end it "binds variables" do - eval(<<~RUBY).should == 1 - case [0, 1] - in [0, a] - a - end - RUBY + case [0, 1] + in [0, a] + a + end.should == 1 end it "cannot mix in and when operators" do @@ -220,47 +189,45 @@ describe "Pattern matching" do end it "checks patterns until the first matching" do - eval(<<~RUBY).should == :bar - case [0, 1] - in [0] - :foo - in [0, 1] - :bar - in [0, 1] - :baz - end - RUBY + case [0, 1] + in [0] + :foo + in [0, 1] + :bar + in [0, 1] + :baz + end.should == :bar end it "executes else clause if no pattern matches" do - eval(<<~RUBY).should == false - case [0, 1] - in [0] - true - else - false - end - RUBY + case [0, 1] + in [0] + true + else + false + end.should == false end it "raises NoMatchingPatternError if no pattern matches and no else clause" do -> { - eval <<~RUBY - case [0, 1] - in [0] - end - RUBY + case [0, 1] + in [0] + end }.should raise_error(NoMatchingPatternError, /\[0, 1\]/) + + -> { + case {a: 0, b: 1} + in a: 1, b: 1 + end + }.should raise_error(NoMatchingPatternError, /\{:a=>0, :b=>1\}/) end it "raises NoMatchingPatternError if no pattern matches and evaluates the expression only once" do evals = 0 -> { - eval <<~RUBY - case (evals += 1; [0, 1]) - in [0] - end - RUBY + case (evals += 1; [0, 1]) + in [0] + end }.should raise_error(NoMatchingPatternError, /\[0, 1\]/) evals.should == 1 end @@ -273,230 +240,185 @@ describe "Pattern matching" do true end RUBY - }.should raise_error(SyntaxError, /unexpected|expected a delimiter after the predicates of a `when` clause/) + }.should raise_error(SyntaxError, /unexpected|expected a delimiter after the patterns of an `in` clause/) end it "evaluates the case expression once for multiple patterns, caching the result" do - eval(<<~RUBY).should == true - case (ScratchPad << :foo; 1) - in 0 - false - in 1 - true - end - RUBY + case (ScratchPad << :foo; 1) + in 0 + false + in 1 + true + end.should == true ScratchPad.recorded.should == [:foo] end describe "guards" do it "supports if guard" do - eval(<<~RUBY).should == false - case 0 - in 0 if false - true - else - false - end - RUBY + case 0 + in 0 if false + true + else + false + end.should == false - eval(<<~RUBY).should == true - case 0 - in 0 if true - true - else - false - end - RUBY + case 0 + in 0 if true + true + else + false + end.should == true end it "supports unless guard" do - eval(<<~RUBY).should == false - case 0 - in 0 unless true - true - else - false - end - RUBY + case 0 + in 0 unless true + true + else + false + end.should == false - eval(<<~RUBY).should == true - case 0 - in 0 unless false - true - else - false - end - RUBY + case 0 + in 0 unless false + true + else + false + end.should == true end it "makes bound variables visible in guard" do - eval(<<~RUBY).should == true - case [0, 1] - in [a, 1] if a >= 0 - true - end - RUBY + case [0, 1] + in [a, 1] if a >= 0 + true + end.should == true end it "does not evaluate guard if pattern does not match" do - eval <<~RUBY - case 0 - in 1 if (ScratchPad << :foo) || true - else - end - RUBY + case 0 + in 1 if (ScratchPad << :foo) || true + else + end ScratchPad.recorded.should == [] end it "takes guards into account when there are several matching patterns" do - eval(<<~RUBY).should == :bar - case 0 - in 0 if false - :foo - in 0 if true - :bar - end - RUBY + case 0 + in 0 if false + :foo + in 0 if true + :bar + end.should == :bar end it "executes else clause if no guarded pattern matches" do - eval(<<~RUBY).should == false - case 0 - in 0 if false - true - else - false - end - RUBY + case 0 + in 0 if false + true + else + false + end.should == false end it "raises NoMatchingPatternError if no guarded pattern matches and no else clause" do -> { - eval <<~RUBY - case [0, 1] - in [0, 1] if false - end - RUBY + case [0, 1] + in [0, 1] if false + end }.should raise_error(NoMatchingPatternError, /\[0, 1\]/) end end describe "value pattern" do it "matches an object such that pattern === object" do - eval(<<~RUBY).should == true - case 0 - in 0 - true - end - RUBY + case 0 + in 0 + true + end.should == true - eval(<<~RUBY).should == true - case 0 - in (-1..1) - true - end - RUBY + case 0 + in ( + -1..1) + true + end.should == true - eval(<<~RUBY).should == true - case 0 - in Integer - true - end - RUBY + case 0 + in Integer + true + end.should == true - eval(<<~RUBY).should == true - case "0" - in /0/ - true - end - RUBY + case "0" + in /0/ + true + end.should == true - eval(<<~RUBY).should == true - case "0" - in ->(s) { s == "0" } - true - end - RUBY + case "0" + in -> s { s == "0" } + true + end.should == true end it "allows string literal with interpolation" do x = "x" - eval(<<~RUBY).should == true - case "x" - in "#{x + ""}" - true - end - RUBY + case "x" + in "#{x + ""}" + true + end.should == true end end describe "variable pattern" do it "matches a value and binds variable name to this value" do - eval(<<~RUBY).should == 0 - case 0 - in a - a - end - RUBY + case 0 + in a + a + end.should == 0 end it "makes bounded variable visible outside a case statement scope" do - eval(<<~RUBY).should == 0 - case 0 - in a - end + case 0 + in a + end - a - RUBY + a.should == 0 end it "create local variables even if a pattern doesn't match" do - eval(<<~RUBY).should == [0, nil, nil] - case 0 - in a - in b - in c - end + case 0 + in a + in b + in c + end - [a, b, c] - RUBY + [a, b, c].should == [0, nil, nil] end it "allow using _ name to drop values" do - eval(<<~RUBY).should == 0 - case [0, 1] - in [a, _] - a - end - RUBY + case [0, 1] + in [a, _] + a + end.should == 0 end it "supports using _ in a pattern several times" do - eval(<<~RUBY).should == true - case [0, 1, 2] - in [0, _, _] - true - end - RUBY + case [0, 1, 2] + in [0, _, _] + true + end.should == true end it "supports using any name with _ at the beginning in a pattern several times" do - eval(<<~RUBY).should == true - case [0, 1, 2] - in [0, _x, _x] - true - end - RUBY + case [0, 1, 2] + in [0, _x, _x] + true + end.should == true - eval(<<~RUBY).should == true - case {a: 0, b: 1, c: 2} - in {a: 0, b: _x, c: _x} - true - end - RUBY + case {a: 0, b: 1, c: 2} + in {a: 0, b: _x, c: _x} + true + end.should == true end it "does not support using variable name (except _) several times" do @@ -512,30 +434,24 @@ describe "Pattern matching" do it "supports existing variables in a pattern specified with ^ operator" do a = 0 - eval(<<~RUBY).should == true - case 0 - in ^a - true - end - RUBY + case 0 + in ^a + true + end.should == true end it "allows applying ^ operator to bound variables" do - eval(<<~RUBY).should == 1 - case [1, 1] - in [n, ^n] - n - end - RUBY + case [1, 1] + in [n, ^n] + n + end.should == 1 - eval(<<~RUBY).should == false - case [1, 2] - in [n, ^n] - true - else - false - end - RUBY + case [1, 2] + in [n, ^n] + true + else + false + end.should == false end it "requires bound variable to be specified in a pattern before ^ operator when it relies on a bound variable" do @@ -554,12 +470,10 @@ describe "Pattern matching" do describe "alternative pattern" do it "matches if any of patterns matches" do - eval(<<~RUBY).should == true - case 0 - in 0 | 1 | 2 - true - end - RUBY + case 0 + in 0 | 1 | 2 + true + end.should == true end it "does not support variable binding" do @@ -573,120 +487,99 @@ describe "Pattern matching" do end it "support underscore prefixed variables in alternation" do - eval(<<~RUBY).should == true - case [0, 1] - in [1, _] - false - in [0, 0] | [0, _a] - true - end - RUBY + case [0, 1] + in [1, _] + false + in [0, 0] | [0, _a] + true + end.should == true end it "can be used as a nested pattern" do - eval(<<~RUBY).should == true - case [[1], ["2"]] - in [[0] | nil, _] - false - in [[1], [1]] - false - in [[1], [2 | "2"]] - true - end - RUBY + case [[1], ["2"]] + in [[0] | nil, _] + false + in [[1], [1]] + false + in [[1], [2 | "2"]] + true + end.should == true - eval(<<~RUBY).should == true - case [1, 2] - in [0, _] | {a: 0} - false - in {a: 1, b: 2} | [1, 2] - true - end - RUBY + case [1, 2] + in [0, _] | {a: 0} + false + in {a: 1, b: 2} | [1, 2] + true + end.should == true end end describe "AS pattern" do it "binds a variable to a value if pattern matches" do - eval(<<~RUBY).should == 0 - case 0 - in Integer => n - n - end - RUBY + case 0 + in Integer => n + n + end.should == 0 end it "can be used as a nested pattern" do - eval(<<~RUBY).should == [2, 3] - case [1, [2, 3]] - in [1, Array => ary] - ary - end - RUBY + case [1, [2, 3]] + in [1, Array => ary] + ary + end.should == [2, 3] end end describe "Array pattern" do it "supports form Constant(pat, pat, ...)" do - eval(<<~RUBY).should == true - case [0, 1, 2] - in Array(0, 1, 2) - true - end - RUBY + case [0, 1, 2] + in Array(0, 1, 2) + true + end.should == true end it "supports form Constant[pat, pat, ...]" do - eval(<<~RUBY).should == true - case [0, 1, 2] - in Array[0, 1, 2] - true - end - RUBY + case [0, 1, 2] + in Array[0, 1, 2] + true + end.should == true end it "supports form [pat, pat, ...]" do - eval(<<~RUBY).should == true - case [0, 1, 2] - in [0, 1, 2] - true - end - RUBY + case [0, 1, 2] + in [0, 1, 2] + true + end.should == true end it "supports form pat, pat, ..." do - eval(<<~RUBY).should == true - case [0, 1, 2] - in 0, 1, 2 - true - end - RUBY + case [0, 1, 2] + in 0, 1, 2 + true + end.should == true - eval(<<~RUBY).should == 1 - case [0, 1, 2] - in 0, a, 2 - a - end - RUBY + case [0, 1, 2] + in 0, a, 2 + a + end.should == 1 - eval(<<~RUBY).should == [1, 2] - case [0, 1, 2] - in 0, *rest - rest - end - RUBY + case [0, 1, 2] + in 0, *rest + rest + end.should == [1, 2] end it "matches an object with #deconstruct method which returns an array and each element in array matches element in pattern" do obj = Object.new - def obj.deconstruct; [0, 1] end - eval(<<~RUBY).should == true - case obj - in [Integer, Integer] - true - end - RUBY + def obj.deconstruct + [0, 1] + end + + case obj + in [Integer, Integer] + true + end.should == true end it "calls #deconstruct once for multiple patterns, caching the result" do @@ -697,314 +590,267 @@ describe "Pattern matching" do [0, 1] end - eval(<<~RUBY).should == true - case obj - in [1, 2] - false - in [0, 1] - true - end - RUBY + case obj + in [1, 2] + false + in [0, 1] + true + end.should == true ScratchPad.recorded.should == [:deconstruct] end it "calls #deconstruct even on objects that are already an array" do obj = [1, 2] + def obj.deconstruct ScratchPad << :deconstruct [3, 4] end - eval(<<~RUBY).should == true - case obj - in [3, 4] - true - else - false - end - RUBY + case obj + in [3, 4] + true + else + false + end.should == true ScratchPad.recorded.should == [:deconstruct] end it "does not match object if Constant === object returns false" do - eval(<<~RUBY).should == false - case [0, 1, 2] - in String[0, 1, 2] - true - else - false - end - RUBY + case [0, 1, 2] + in String[0, 1, 2] + true + else + false + end.should == false end it "checks Constant === object before calling #deconstruct" do c1 = Class.new obj = c1.new obj.should_not_receive(:deconstruct) - eval(<<~RUBY).should == false - case obj - in String[1] - true - else - false - end - RUBY + + case obj + in String[1] + true + else + false + end.should == false end it "does not match object without #deconstruct method" do obj = Object.new obj.should_receive(:respond_to?).with(:deconstruct) - eval(<<~RUBY).should == false - case obj - in Object[] - true - else - false - end - RUBY + case obj + in Object[] + true + else + false + end.should == false end it "raises TypeError if #deconstruct method does not return array" do obj = Object.new - def obj.deconstruct; "" end + + def obj.deconstruct + "" + end -> { - eval <<~RUBY - case obj - in Object[] - else - end - RUBY + case obj + in Object[] + else + end }.should raise_error(TypeError, /deconstruct must return Array/) end it "accepts a subclass of Array from #deconstruct" do obj = Object.new + def obj.deconstruct Class.new(Array).new([0, 1]) end - eval(<<~RUBY).should == true - case obj - in [1, 2] - false - in [0, 1] - true - end - RUBY + case obj + in [1, 2] + false + in [0, 1] + true + end.should == true end it "does not match object if elements of array returned by #deconstruct method does not match elements in pattern" do obj = Object.new - def obj.deconstruct; [1] end - eval(<<~RUBY).should == false - case obj - in Object[0] - true - else - false - end - RUBY + def obj.deconstruct + [1] + end + + case obj + in Object[0] + true + else + false + end.should == false end it "binds variables" do - eval(<<~RUBY).should == [0, 1, 2] - case [0, 1, 2] - in [a, b, c] - [a, b, c] - end - RUBY + case [0, 1, 2] + in [a, b, c] + [a, b, c] + end.should == [0, 1, 2] end it "supports splat operator *rest" do - eval(<<~RUBY).should == [1, 2] - case [0, 1, 2] - in [0, *rest] - rest - end - RUBY + case [0, 1, 2] + in [0, *rest] + rest + end.should == [1, 2] end it "does not match partially by default" do - eval(<<~RUBY).should == false - case [0, 1, 2, 3] - in [1, 2] - true - else - false - end - RUBY + case [0, 1, 2, 3] + in [1, 2] + true + else + false + end.should == false end it "does match partially from the array beginning if list + , syntax used" do - eval(<<~RUBY).should == true - case [0, 1, 2, 3] - in [0, 1,] - true - end - RUBY + case [0, 1, 2, 3] + in [0, 1, ] + true + end.should == true - eval(<<~RUBY).should == true - case [0, 1, 2, 3] - in 0, 1,; - true - end - RUBY + case [0, 1, 2, 3] + in 0, 1,; + true + end.should == true end it "matches [] with []" do - eval(<<~RUBY).should == true - case [] - in [] - true - end - RUBY + case [] + in [] + true + end.should == true end it "matches anything with *" do - eval(<<~RUBY).should == true - case [0, 1] - in *; - true - end - RUBY + case [0, 1] + in *; + true + end.should == true end it "can be used as a nested pattern" do - eval(<<~RUBY).should == true - case [[1], ["2"]] - in [[0] | nil, _] - false - in [[1], [1]] - false - in [[1], [2 | "2"]] - true - end - RUBY + case [[1], ["2"]] + in [[0] | nil, _] + false + in [[1], [1]] + false + in [[1], [2 | "2"]] + true + end.should == true - eval(<<~RUBY).should == true - case [1, 2] - in [0, _] | {a: 0} - false - in {a: 1, b: 2} | [1, 2] - true - end - RUBY + case [1, 2] + in [0, _] | {a: 0} + false + in {a: 1, b: 2} | [1, 2] + true + end.should == true end end describe "Hash pattern" do it "supports form Constant(id: pat, id: pat, ...)" do - eval(<<~RUBY).should == true - case {a: 0, b: 1} - in Hash(a: 0, b: 1) - true - end - RUBY + case {a: 0, b: 1} + in Hash(a: 0, b: 1) + true + end.should == true end it "supports form Constant[id: pat, id: pat, ...]" do - eval(<<~RUBY).should == true - case {a: 0, b: 1} - in Hash[a: 0, b: 1] - true - end - RUBY + case {a: 0, b: 1} + in Hash[a: 0, b: 1] + true + end.should == true end it "supports form {id: pat, id: pat, ...}" do - eval(<<~RUBY).should == true - case {a: 0, b: 1} - in {a: 0, b: 1} - true - end - RUBY + case {a: 0, b: 1} + in {a: 0, b: 1} + true + end.should == true end it "supports form id: pat, id: pat, ..." do - eval(<<~RUBY).should == true - case {a: 0, b: 1} - in a: 0, b: 1 - true - end - RUBY + case {a: 0, b: 1} + in a: 0, b: 1 + true + end.should == true - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in a: a, b: b - [a, b] - end - RUBY + case {a: 0, b: 1} + in a: a, b: b + [a, b] + end.should == [0, 1] - eval(<<~RUBY).should == { b: 1, c: 2 } - case {a: 0, b: 1, c: 2} - in a: 0, **rest - rest - end - RUBY + case {a: 0, b: 1, c: 2} + in a: 0, **rest + rest + end.should == {b: 1, c: 2} end it "supports a: which means a: a" do - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in Hash(a:, b:) - [a, b] - end - RUBY + case {a: 0, b: 1} + in Hash(a:, b:) + [a, b] + end.should == [0, 1] a = b = nil - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in Hash[a:, b:] - [a, b] - end - RUBY + + case {a: 0, b: 1} + in Hash[a:, b:] + [a, b] + end.should == [0, 1] a = b = nil - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in {a:, b:} - [a, b] - end - RUBY + + case {a: 0, b: 1} + in {a:, b:} + [a, b] + end.should == [0, 1] a = nil - eval(<<~RUBY).should == [0, {b: 1, c: 2}] - case {a: 0, b: 1, c: 2} - in {a:, **rest} - [a, rest] - end - RUBY + + case {a: 0, b: 1, c: 2} + in {a:, **rest} + [a, rest] + end.should == [0, {b: 1, c: 2}] a = b = nil - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in a:, b: - [a, b] - end - RUBY + + case {a: 0, b: 1} + in a:, b: + [a, b] + end.should == [0, 1] end it "can mix key (a:) and key-value (a: b) declarations" do - eval(<<~RUBY).should == [0, 1] - case {a: 0, b: 1} - in Hash(a:, b: x) - [a, x] - end - RUBY + case {a: 0, b: 1} + in Hash(a:, b: x) + [a, x] + end.should == [0, 1] end it "supports 'string': key literal" do - eval(<<~RUBY).should == true - case {a: 0} - in {"a": 0} - true - end - RUBY + case {a: 0} + in {"a": 0} + true + end.should == true end it "does not support non-symbol keys" do @@ -1018,8 +864,6 @@ describe "Pattern matching" do end it "does not support string interpolation in keys" do - x = "a" - -> { eval <<~'RUBY' case {a: 1} @@ -1041,14 +885,15 @@ describe "Pattern matching" do it "matches an object with #deconstruct_keys method which returns a Hash with equal keys and each value in Hash matches value in pattern" do obj = Object.new - def obj.deconstruct_keys(*); {a: 1} end - eval(<<~RUBY).should == true - case obj - in {a: 1} - true - end - RUBY + def obj.deconstruct_keys(*) + {a: 1} + end + + case obj + in {a: 1} + true + end.should == true end it "calls #deconstruct_keys per pattern" do @@ -1059,96 +904,92 @@ describe "Pattern matching" do {a: 1} end - eval(<<~RUBY).should == true - case obj - in {b: 1} - false - in {a: 1} - true - end - RUBY + case obj + in {b: 1} + false + in {a: 1} + true + end.should == true ScratchPad.recorded.should == [:deconstruct_keys, :deconstruct_keys] end it "does not match object if Constant === object returns false" do - eval(<<~RUBY).should == false - case {a: 1} - in String[a: 1] - true - else - false - end - RUBY + case {a: 1} + in String[a: 1] + true + else + false + end.should == false end it "checks Constant === object before calling #deconstruct_keys" do c1 = Class.new obj = c1.new obj.should_not_receive(:deconstruct_keys) - eval(<<~RUBY).should == false - case obj - in String(a: 1) - true - else - false - end - RUBY + + case obj + in String(a: 1) + true + else + false + end.should == false end it "does not match object without #deconstruct_keys method" do obj = Object.new obj.should_receive(:respond_to?).with(:deconstruct_keys) - eval(<<~RUBY).should == false - case obj - in Object[a: 1] - true - else - false - end - RUBY + case obj + in Object[a: 1] + true + else + false + end.should == false end it "does not match object if #deconstruct_keys method does not return Hash" do obj = Object.new - def obj.deconstruct_keys(*); "" end + + def obj.deconstruct_keys(*) + "" + end -> { - eval <<~RUBY - case obj - in Object[a: 1] - end - RUBY + case obj + in Object[a: 1] + end }.should raise_error(TypeError, /deconstruct_keys must return Hash/) end it "does not match object if #deconstruct_keys method returns Hash with non-symbol keys" do obj = Object.new - def obj.deconstruct_keys(*); {"a" => 1} end - eval(<<~RUBY).should == false - case obj - in Object[a: 1] - true - else - false - end - RUBY + def obj.deconstruct_keys(*) + {"a" => 1} + end + + case obj + in Object[a: 1] + true + else + false + end.should == false end it "does not match object if elements of Hash returned by #deconstruct_keys method does not match values in pattern" do obj = Object.new - def obj.deconstruct_keys(*); {a: 1} end - eval(<<~RUBY).should == false - case obj - in Object[a: 2] - true - else - false - end - RUBY + def obj.deconstruct_keys(*) + {a: 1} + end + + case obj + in Object[a: 2] + true + else + false + end.should == false end it "passes keys specified in pattern as arguments to #deconstruct_keys method" do @@ -1159,11 +1000,9 @@ describe "Pattern matching" do {a: 1, b: 2, c: 3} end - eval <<~RUBY - case obj - in Object[a: 1, b: 2, c: 3] - end - RUBY + case obj + in Object[a: 1, b: 2, c: 3] + end ScratchPad.recorded.sort.should == [[[:a, :b, :c]]] end @@ -1176,11 +1015,9 @@ describe "Pattern matching" do {a: 1, b: 2, c: 3} end - eval <<~RUBY - case obj - in Object[a: 1, b: 2, **] - end - RUBY + case obj + in Object[a: 1, b: 2, **] + end ScratchPad.recorded.sort.should == [[[:a, :b]]] end @@ -1193,131 +1030,105 @@ describe "Pattern matching" do {a: 1, b: 2} end - eval <<~RUBY - case obj - in Object[a: 1, **rest] - end - RUBY + case obj + in Object[a: 1, **rest] + end ScratchPad.recorded.should == [[nil]] end it "binds variables" do - eval(<<~RUBY).should == [0, 1, 2] - case {a: 0, b: 1, c: 2} - in {a: x, b: y, c: z} - [x, y, z] - end - RUBY + case {a: 0, b: 1, c: 2} + in {a: x, b: y, c: z} + [x, y, z] + end.should == [0, 1, 2] end it "supports double splat operator **rest" do - eval(<<~RUBY).should == {b: 1, c: 2} - case {a: 0, b: 1, c: 2} - in {a: 0, **rest} - rest - end - RUBY + case {a: 0, b: 1, c: 2} + in {a: 0, **rest} + rest + end.should == {b: 1, c: 2} end it "treats **nil like there should not be any other keys in a matched Hash" do - eval(<<~RUBY).should == true - case {a: 1, b: 2} - in {a: 1, b: 2, **nil} - true - end - RUBY + case {a: 1, b: 2} + in {a: 1, b: 2, **nil} + true + end.should == true - eval(<<~RUBY).should == false - case {a: 1, b: 2} - in {a: 1, **nil} - true - else - false - end - RUBY + case {a: 1, b: 2} + in {a: 1, **nil} + true + else + false + end.should == false end it "can match partially" do - eval(<<~RUBY).should == true - case {a: 1, b: 2} - in {a: 1} - true - end - RUBY + case {a: 1, b: 2} + in {a: 1} + true + end.should == true end it "matches {} with {}" do - eval(<<~RUBY).should == true - case {} - in {} - true - end - RUBY + case {} + in {} + true + end.should == true end it "in {} only matches empty hashes" do - eval(<<~RUBY).should == false - case {a: 1} - in {} - true - else - false - end - RUBY + case {a: 1} + in {} + true + else + false + end.should == false end it "in {**nil} only matches empty hashes" do - eval(<<~RUBY).should == true - case {} - in {**nil} - true - else - false - end - RUBY + case {} + in {**nil} + true + else + false + end.should == true - eval(<<~RUBY).should == false - case {a: 1} - in {**nil} - true - else - false - end - RUBY + case {a: 1} + in {**nil} + true + else + false + end.should == false end it "matches anything with **" do - eval(<<~RUBY).should == true - case {a: 1} - in **; - true - end - RUBY + case {a: 1} + in **; + true + end.should == true end it "can be used as a nested pattern" do - eval(<<~RUBY).should == true - case {a: {a: 1, b: 1}, b: {a: 1, b: 2}} - in {a: {a: 0}} - false - in {a: {a: 1}, b: {b: 1}} - false - in {a: {a: 1}, b: {b: 2}} - true - end - RUBY + case {a: {a: 1, b: 1}, b: {a: 1, b: 2}} + in {a: {a: 0}} + false + in {a: {a: 1}, b: {b: 1}} + false + in {a: {a: 1}, b: {b: 2} } + true + end.should == true - eval(<<~RUBY).should == true - case [{a: 1, b: [1]}, {a: 1, c: ["2"]}] - in [{a:, c:},] - false - in [{a: 1, b:}, {a: 1, c: [Integer]}] - false - in [_, {a: 1, c: [String]}] - true - end - RUBY + case [{a: 1, b: [1]}, {a: 1, c: ["2"]}] + in [{a:, c:}, ] + false + in [{a: 1, b:}, {a: 1, c: [Integer]}] + false + in [_, {a: 1, c: [String]}] + true + end.should == true end end @@ -1335,12 +1146,11 @@ describe "Pattern matching" do Module.new do using refinery - result = eval(<<~RUBY) + result = case [] in [0] true end - RUBY end result.should == true @@ -1359,12 +1169,11 @@ describe "Pattern matching" do Module.new do using refinery - result = eval(<<~RUBY) + result = case {} in a: 0 true end - RUBY end result.should == true @@ -1383,127 +1192,18 @@ describe "Pattern matching" do Module.new do using refinery - result = eval(<<~RUBY) + result = case {} in Array true end - RUBY end result.should == true end end +end - describe "Ruby 3.1 improvements" do - ruby_version_is "3.1" do - it "can omit parentheses in one line pattern matching" do - eval(<<~RUBY).should == [1, 2] - [1, 2] => a, b - [a, b] - RUBY - - eval(<<~RUBY).should == 1 - {a: 1} => a: - a - RUBY - end - - it "supports pinning instance variables" do - eval(<<~RUBY).should == true - @a = /a/ - case 'abc' - in ^@a - true - end - RUBY - end - - it "supports pinning class variables" do - result = nil - Module.new do - result = module_eval(<<~RUBY) - @@a = 0..10 - - case 2 - in ^@@a - true - end - RUBY - end - - result.should == true - end - - it "supports pinning global variables" do - eval(<<~RUBY).should == true - $a = /a/ - case 'abc' - in ^$a - true - end - RUBY - end - - it "supports pinning expressions" do - eval(<<~RUBY).should == true - case 'abc' - in ^(/a/) - true - end - RUBY - - eval(<<~RUBY).should == true - case 0 - in ^(0+0) - true - end - RUBY - end - - it "supports pinning expressions in array pattern" do - eval(<<~RUBY).should == true - case [3] - in [^(1+2)] - true - end - RUBY - end - - it "supports pinning expressions in hash pattern" do - eval(<<~RUBY).should == true - case {name: '2.6', released_at: Time.new(2018, 12, 25)} - in {released_at: ^(Time.new(2010)..Time.new(2020))} - true - end - RUBY - end - end - end - - describe "value in pattern" do - it "returns true if the pattern matches" do - eval("1 in 1").should == true - - eval("1 in Integer").should == true - - e = nil - eval("[1, 2] in [1, e]").should == true - e.should == 2 - - k = nil - eval("{k: 1} in {k:}").should == true - k.should == 1 - end - - it "returns false if the pattern does not match" do - eval("1 in 2").should == false - - eval("1 in Float").should == false - - eval("[1, 2] in [2, e]").should == false - - eval("{k: 1} in {k: 2}").should == false - end - end +ruby_version_is "3.1" do + require_relative 'pattern_matching/3.1' end diff --git a/spec/ruby/language/regexp/back-references_spec.rb b/spec/ruby/language/regexp/back-references_spec.rb index 26750c20c5..627c8daace 100644 --- a/spec/ruby/language/regexp/back-references_spec.rb +++ b/spec/ruby/language/regexp/back-references_spec.rb @@ -22,6 +22,15 @@ describe "Regexps with back-references" do $10.should == "0" end + it "returns nil for numbered variable with too large index" do + -> { + eval(<<~CODE).should == nil + "a" =~ /(.)/ + eval('$4294967296') + CODE + }.should complain(/warning: ('|`)\$4294967296' is too big for a number variable, always nil/) + end + it "will not clobber capture variables across threads" do cap1, cap2, cap3 = nil "foo" =~ /(o+)/ diff --git a/spec/ruby/language/retry_spec.rb b/spec/ruby/language/retry_spec.rb index ee5377946f..669d5f0ff5 100644 --- a/spec/ruby/language/retry_spec.rb +++ b/spec/ruby/language/retry_spec.rb @@ -31,8 +31,11 @@ describe "The retry statement" do results.should == [1, 2, 3, 1, 2, 4, 5, 6, 4, 5] end - it "raises a SyntaxError when used outside of a begin statement" do + it "raises a SyntaxError when used outside of a rescue statement" do -> { eval 'retry' }.should raise_error(SyntaxError) + -> { eval 'begin; retry; end' }.should raise_error(SyntaxError) + -> { eval 'def m; retry; end' }.should raise_error(SyntaxError) + -> { eval 'module RetrySpecs; retry; end' }.should raise_error(SyntaxError) end end diff --git a/spec/ruby/language/yield_spec.rb b/spec/ruby/language/yield_spec.rb index 5283517636..e125cf8e73 100644 --- a/spec/ruby/language/yield_spec.rb +++ b/spec/ruby/language/yield_spec.rb @@ -206,3 +206,15 @@ describe "Using yield in non-lambda block" do -> { eval(code) }.should raise_error(SyntaxError, /Invalid yield/) end end + +describe "Using yield in a module literal" do + it 'raises a SyntaxError' do + code = <<~RUBY + module YieldSpecs::ModuleWithYield + yield + end + RUBY + + -> { eval(code) }.should raise_error(SyntaxError, /Invalid yield/) + end +end diff --git a/spec/ruby/library/time/to_date_spec.rb b/spec/ruby/library/date/time/to_date_spec.rb index baeafe0847..f9132da289 100644 --- a/spec/ruby/library/time/to_date_spec.rb +++ b/spec/ruby/library/date/time/to_date_spec.rb @@ -1,5 +1,5 @@ -require_relative '../../spec_helper' +require_relative '../../../spec_helper' require 'time' describe "Time#to_date" do diff --git a/spec/ruby/library/time/to_datetime_spec.rb b/spec/ruby/library/datetime/time/to_datetime_spec.rb index 9c44f38e5c..1125dbe851 100644 --- a/spec/ruby/library/time/to_datetime_spec.rb +++ b/spec/ruby/library/datetime/time/to_datetime_spec.rb @@ -1,4 +1,4 @@ -require_relative '../../spec_helper' +require_relative '../../../spec_helper' require 'time' require 'date' date_version = defined?(Date::VERSION) ? Date::VERSION : '3.1.0' diff --git a/spec/ruby/library/io-wait/wait_readable_spec.rb b/spec/ruby/library/io-wait/wait_readable_spec.rb index 06ffbda5c8..df25fdb931 100644 --- a/spec/ruby/library/io-wait/wait_readable_spec.rb +++ b/spec/ruby/library/io-wait/wait_readable_spec.rb @@ -24,4 +24,23 @@ describe "IO#wait_readable" do it "waits for the IO to become readable with the given large timeout" do @io.wait_readable(365 * 24 * 60 * 60).should == @io end + + it "can be interrupted" do + rd, wr = IO.pipe + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + rd.wait_readable(10) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + ensure + rd.close + wr.close + end end diff --git a/spec/ruby/library/io-wait/wait_spec.rb b/spec/ruby/library/io-wait/wait_spec.rb index fc07c6a8d9..5952f127f7 100644 --- a/spec/ruby/library/io-wait/wait_spec.rb +++ b/spec/ruby/library/io-wait/wait_spec.rb @@ -1,5 +1,5 @@ require_relative '../../spec_helper' -require_relative 'fixtures/classes' +require_relative '../../fixtures/io' ruby_version_is ''...'3.2' do require 'io/wait' @@ -55,7 +55,7 @@ describe "IO#wait" do end it "waits for the WRITABLE event to be ready" do - written_bytes = IOWaitSpec.exhaust_write_buffer(@w) + written_bytes = IOSpec.exhaust_write_buffer(@w) @w.wait(IO::WRITABLE, 0).should == nil @r.read(written_bytes) @@ -67,7 +67,7 @@ describe "IO#wait" do end it "returns nil when the WRITABLE event is not ready during the timeout" do - IOWaitSpec.exhaust_write_buffer(@w) + IOSpec.exhaust_write_buffer(@w) @w.wait(IO::WRITABLE, 0).should == nil end @@ -92,7 +92,7 @@ describe "IO#wait" do end it "changes thread status to 'sleep' when waits for WRITABLE event" do - written_bytes = IOWaitSpec.exhaust_write_buffer(@w) + IOSpec.exhaust_write_buffer(@w) t = Thread.new { @w.wait(IO::WRITABLE, 10) } sleep 1 @@ -100,6 +100,37 @@ describe "IO#wait" do t.kill t.join # Thread#kill doesn't wait for the thread to end end + + it "can be interrupted when waiting for READABLE event" do + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @r.wait(IO::READABLE, 10) + end + + Thread.pass until t.stop? + t.kill + t.join # Thread#kill doesn't wait for the thread to end + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end + + it "can be interrupted when waiting for WRITABLE event" do + IOSpec.exhaust_write_buffer(@w) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @w.wait(IO::WRITABLE, 10) + end + + Thread.pass until t.stop? + t.kill + t.join # Thread#kill doesn't wait for the thread to end + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end end context "[timeout, mode] passed" do diff --git a/spec/ruby/library/io-wait/wait_writable_spec.rb b/spec/ruby/library/io-wait/wait_writable_spec.rb index 8c44780d39..8639dd717b 100644 --- a/spec/ruby/library/io-wait/wait_writable_spec.rb +++ b/spec/ruby/library/io-wait/wait_writable_spec.rb @@ -1,4 +1,5 @@ require_relative '../../spec_helper' +require_relative '../../fixtures/io' ruby_version_is ''...'3.2' do require 'io/wait' @@ -17,4 +18,24 @@ describe "IO#wait_writable" do # Represents one year and is larger than a 32-bit int STDOUT.wait_writable(365 * 24 * 60 * 60).should == STDOUT end + + it "can be interrupted" do + rd, wr = IO.pipe + IOSpec.exhaust_write_buffer(wr) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + wr.wait_writable(10) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + ensure + rd.close unless rd.closed? + wr.close unless wr.closed? + end end diff --git a/spec/ruby/library/rbconfig/rbconfig_spec.rb b/spec/ruby/library/rbconfig/rbconfig_spec.rb index b90cc90970..3e06219621 100644 --- a/spec/ruby/library/rbconfig/rbconfig_spec.rb +++ b/spec/ruby/library/rbconfig/rbconfig_spec.rb @@ -88,6 +88,30 @@ describe 'RbConfig::CONFIG' do end end end + + guard -> { %w[aarch64 arm64].include? RbConfig::CONFIG['host_cpu'] } do + it "['host_cpu'] returns CPU architecture properly for AArch64" do + platform_is :darwin do + RbConfig::CONFIG['host_cpu'].should == 'arm64' + end + + platform_is_not :darwin do + RbConfig::CONFIG['host_cpu'].should == 'aarch64' + end + end + end + + guard -> { platform_is(:linux) || platform_is(:darwin) } do + it "['host_os'] returns a proper OS name or platform" do + platform_is :darwin do + RbConfig::CONFIG['host_os'].should.match?(/darwin/) + end + + platform_is :linux do + RbConfig::CONFIG['host_os'].should.match?(/linux/) + end + end + end end describe "RbConfig::TOPDIR" do @@ -99,3 +123,34 @@ describe "RbConfig::TOPDIR" do end end end + +describe "RUBY_PLATFORM" do + it "RUBY_PLATFORM contains a proper CPU architecture" do + RUBY_PLATFORM.should.include? RbConfig::CONFIG['host_cpu'] + end + + guard -> { platform_is(:linux) || platform_is(:darwin) } do + it "RUBY_PLATFORM contains OS name" do + # don't use RbConfig::CONFIG['host_os'] as far as it could be slightly different, e.g. linux-gnu + platform_is(:linux) do + RUBY_PLATFORM.should.include? 'linux' + end + + platform_is(:darwin) do + RUBY_PLATFORM.should.include? 'darwin' + end + end + end +end + +describe "RUBY_DESCRIPTION" do + guard_not -> { RUBY_ENGINE == "ruby" && !RbConfig::TOPDIR } do + it "contains version" do + RUBY_DESCRIPTION.should.include? RUBY_VERSION + end + + it "contains RUBY_PLATFORM" do + RUBY_DESCRIPTION.should.include? RUBY_PLATFORM + end + end +end diff --git a/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb b/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb index 17c846054d..df42c116fb 100644 --- a/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb @@ -52,9 +52,19 @@ describe "Socket::BasicSocket#recv_nonblock" do @s2.send("data", 0, @s1.getsockname) IO.select([@s1], nil, nil, 2) - buf = +"foo" - @s1.recv_nonblock(5, 0, buf) - buf.should == "data" + buffer = +"foo" + @s1.recv_nonblock(5, 0, buffer).should.equal?(buffer) + buffer.should == "data" + end + + it "preserves the encoding of the given buffer" do + @s1.bind(Socket.pack_sockaddr_in(0, ip_address)) + @s2.send("data", 0, @s1.getsockname) + IO.select([@s1], nil, nil, 2) + + buffer = ''.encode(Encoding::ISO_8859_1) + @s1.recv_nonblock(5, 0, buffer) + buffer.encoding.should == Encoding::ISO_8859_1 end it "does not block if there's no data available" do diff --git a/spec/ruby/library/socket/basicsocket/recv_spec.rb b/spec/ruby/library/socket/basicsocket/recv_spec.rb index 9fe8c52f9a..e82a357d3d 100644 --- a/spec/ruby/library/socket/basicsocket/recv_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_spec.rb @@ -100,13 +100,29 @@ describe "BasicSocket#recv" do socket.write("data") client = @server.accept - buf = +"foo" + buffer = +"foo" begin - client.recv(4, 0, buf) + client.recv(4, 0, buffer).should.equal?(buffer) ensure client.close end - buf.should == "data" + buffer.should == "data" + + socket.close + end + + it "preserves the encoding of the given buffer" do + socket = TCPSocket.new('127.0.0.1', @port) + socket.write("data") + + client = @server.accept + buffer = ''.encode(Encoding::ISO_8859_1) + begin + client.recv(4, 0, buffer) + ensure + client.close + end + buffer.encoding.should == Encoding::ISO_8859_1 socket.close end diff --git a/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb b/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb index 94f58ac49f..5596f91bb8 100644 --- a/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb +++ b/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb @@ -52,6 +52,27 @@ describe 'Socket#recvfrom_nonblock' do end end + it "allows an output buffer as third argument" do + @client.write('hello') + + IO.select([@server]) + buffer = +'' + message, = @server.recvfrom_nonblock(5, 0, buffer) + + message.should.equal?(buffer) + buffer.should == 'hello' + end + + it "preserves the encoding of the given buffer" do + @client.write('hello') + + IO.select([@server]) + buffer = ''.encode(Encoding::ISO_8859_1) + @server.recvfrom_nonblock(5, 0, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + describe 'the returned data' do it 'is the same as the sent data' do 5.times do diff --git a/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb b/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb index 650a061221..b804099589 100644 --- a/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb +++ b/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb @@ -58,6 +58,15 @@ describe 'UDPSocket#recvfrom_nonblock' do buffer.should == 'h' end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + IO.select([@server]) + message, = @server.recvfrom_nonblock(1, 0, buffer) + + message.should.equal?(buffer) + buffer.encoding.should == Encoding::ISO_8859_1 + end + describe 'the returned Array' do before do IO.select([@server]) diff --git a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb index fedf74bb2f..d849fdc302 100644 --- a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb +++ b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb @@ -31,6 +31,29 @@ with_feature :unix_socket do sock.close end + it "allows an output buffer as third argument" do + buffer = +'' + + @client.send("foobar", 0) + sock = @server.accept + message, = sock.recvfrom(6, 0, buffer) + sock.close + + message.should.equal?(buffer) + buffer.should == "foobar" + end + + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + + @client.send("foobar", 0) + sock = @server.accept + sock.recvfrom(6, 0, buffer) + sock.close + + buffer.encoding.should == Encoding::ISO_8859_1 + end + it "uses different message options" do @client.send("foobar", Socket::MSG_PEEK) sock = @server.accept diff --git a/spec/ruby/library/stringio/shared/read.rb b/spec/ruby/library/stringio/shared/read.rb index e3840786d9..8ef6ec2734 100644 --- a/spec/ruby/library/stringio/shared/read.rb +++ b/spec/ruby/library/stringio/shared/read.rb @@ -11,10 +11,28 @@ describe :stringio_read, shared: true do end it "reads length bytes and writes them to the buffer String" do - @io.send(@method, 7, buffer = +"") + @io.send(@method, 7, buffer = +"").should.equal?(buffer) buffer.should == "example" end + ruby_version_is ""..."3.4" do + it "does not preserve the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.send(@method, 7, buffer) + + buffer.encoding.should_not == Encoding::ISO_8859_1 + end + end + + ruby_version_is "3.4" do + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.send(@method, 7, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + end + it "tries to convert the passed buffer Object to a String using #to_str" do obj = mock("to_str") obj.should_receive(:to_str).and_return(buffer = +"") diff --git a/spec/ruby/optional/capi/class_spec.rb b/spec/ruby/optional/capi/class_spec.rb index d0a9913570..a231245ebe 100644 --- a/spec/ruby/optional/capi/class_spec.rb +++ b/spec/ruby/optional/capi/class_spec.rb @@ -487,4 +487,16 @@ describe "C-API Class function" do @s.rb_class_real(0).should == 0 end end + + describe "rb_class_get_superclass" do + it "returns parent class for a provided class" do + a = Class.new + @s.rb_class_get_superclass(Class.new(a)).should == a + end + + it "returns false when there is no parent class" do + @s.rb_class_get_superclass(BasicObject).should == false + @s.rb_class_get_superclass(Module.new).should == false + end + end end diff --git a/spec/ruby/optional/capi/data_spec.rb b/spec/ruby/optional/capi/data_spec.rb index 18c769332e..d000eba6e3 100644 --- a/spec/ruby/optional/capi/data_spec.rb +++ b/spec/ruby/optional/capi/data_spec.rb @@ -1,52 +1,53 @@ require_relative 'spec_helper' +ruby_version_is ""..."3.4" do + load_extension("data") -load_extension("data") - -describe "CApiAllocSpecs (a class with an alloc func defined)" do - it "calls the alloc func" do - @s = CApiAllocSpecs.new - @s.wrapped_data.should == 42 # not defined in initialize - end -end - -describe "CApiWrappedStruct" do - before :each do - @s = CApiWrappedStructSpecs.new - end - - it "wraps with Data_Wrap_Struct and Data_Get_Struct returns data" do - a = @s.wrap_struct(1024) - @s.get_struct(a).should == 1024 + describe "CApiAllocSpecs (a class with an alloc func defined)" do + it "calls the alloc func" do + @s = CApiAllocSpecs.new + @s.wrapped_data.should == 42 # not defined in initialize + end end - describe "RDATA()" do - it "returns the struct data" do - a = @s.wrap_struct(1024) - @s.get_struct_rdata(a).should == 1024 + describe "CApiWrappedStruct" do + before :each do + @s = CApiWrappedStructSpecs.new end - it "allows changing the wrapped struct" do + it "wraps with Data_Wrap_Struct and Data_Get_Struct returns data" do a = @s.wrap_struct(1024) - @s.change_struct(a, 100) - @s.get_struct(a).should == 100 + @s.get_struct(a).should == 1024 end - it "raises a TypeError if the object does not wrap a struct" do - -> { @s.get_struct(Object.new) }.should raise_error(TypeError) + describe "RDATA()" do + it "returns the struct data" do + a = @s.wrap_struct(1024) + @s.get_struct_rdata(a).should == 1024 + end + + it "allows changing the wrapped struct" do + a = @s.wrap_struct(1024) + @s.change_struct(a, 100) + @s.get_struct(a).should == 100 + end + + it "raises a TypeError if the object does not wrap a struct" do + -> { @s.get_struct(Object.new) }.should raise_error(TypeError) + end end - end - describe "rb_check_type" do - it "does not raise an exception when checking data objects" do - a = @s.wrap_struct(1024) - @s.rb_check_type(a, a).should == true + describe "rb_check_type" do + it "does not raise an exception when checking data objects" do + a = @s.wrap_struct(1024) + @s.rb_check_type(a, a).should == true + end end - end - describe "DATA_PTR" do - it "returns the struct data" do - a = @s.wrap_struct(1024) - @s.get_struct_data_ptr(a).should == 1024 + describe "DATA_PTR" do + it "returns the struct data" do + a = @s.wrap_struct(1024) + @s.get_struct_data_ptr(a).should == 1024 + end end end end diff --git a/spec/ruby/optional/capi/ext/class_spec.c b/spec/ruby/optional/capi/ext/class_spec.c index f376534924..c13f02ecf2 100644 --- a/spec/ruby/optional/capi/ext/class_spec.c +++ b/spec/ruby/optional/capi/ext/class_spec.c @@ -79,6 +79,10 @@ static VALUE class_spec_rb_class_real(VALUE self, VALUE object) { } } +static VALUE class_spec_rb_class_get_superclass(VALUE self, VALUE klass) { + return rb_class_get_superclass(klass); +} + static VALUE class_spec_rb_class_superclass(VALUE self, VALUE klass) { return rb_class_superclass(klass); } @@ -160,6 +164,7 @@ void Init_class_spec(void) { rb_define_method(cls, "rb_class_new_instance_kw", class_spec_rb_class_new_instance_kw, 2); #endif rb_define_method(cls, "rb_class_real", class_spec_rb_class_real, 1); + rb_define_method(cls, "rb_class_get_superclass", class_spec_rb_class_get_superclass, 1); rb_define_method(cls, "rb_class_superclass", class_spec_rb_class_superclass, 1); rb_define_method(cls, "rb_cvar_defined", class_spec_cvar_defined, 2); rb_define_method(cls, "rb_cvar_get", class_spec_cvar_get, 2); diff --git a/spec/ruby/optional/capi/ext/data_spec.c b/spec/ruby/optional/capi/ext/data_spec.c index ef069ef0ba..efefe37c3a 100644 --- a/spec/ruby/optional/capi/ext/data_spec.c +++ b/spec/ruby/optional/capi/ext/data_spec.c @@ -3,6 +3,7 @@ #include <string.h> +#ifndef RUBY_VERSION_IS_3_4 #ifdef __cplusplus extern "C" { #endif @@ -70,8 +71,10 @@ VALUE sws_rb_check_type(VALUE self, VALUE obj, VALUE other) { rb_check_type(obj, TYPE(other)); return Qtrue; } +#endif void Init_data_spec(void) { +#ifndef RUBY_VERSION_IS_3_4 VALUE cls = rb_define_class("CApiAllocSpecs", rb_cObject); rb_define_alloc_func(cls, sdaf_alloc_func); rb_define_method(cls, "wrapped_data", sdaf_get_struct, 0); @@ -82,6 +85,7 @@ void Init_data_spec(void) { rb_define_method(cls, "get_struct_data_ptr", sws_get_struct_data_ptr, 1); rb_define_method(cls, "change_struct", sws_change_struct, 2); rb_define_method(cls, "rb_check_type", sws_rb_check_type, 2); +#endif } #ifdef __cplusplus diff --git a/spec/ruby/optional/capi/ext/gc_spec.c b/spec/ruby/optional/capi/ext/gc_spec.c index 1392bc6ee6..2637ad27ac 100644 --- a/spec/ruby/optional/capi/ext/gc_spec.c +++ b/spec/ruby/optional/capi/ext/gc_spec.c @@ -16,6 +16,8 @@ VALUE registered_after_rb_global_variable_bignum; VALUE registered_after_rb_global_variable_float; VALUE rb_gc_register_address_outside_init; +VALUE rb_gc_register_mark_object_not_referenced_float; + static VALUE registered_tagged_address(VALUE self) { return registered_tagged_value; } @@ -90,6 +92,10 @@ static VALUE gc_spec_rb_gc_register_mark_object(VALUE self, VALUE obj) { return Qnil; } +static VALUE gc_spec_rb_gc_register_mark_object_not_referenced_float(VALUE self) { + return rb_gc_register_mark_object_not_referenced_float; +} + void Init_gc_spec(void) { VALUE cls = rb_define_class("CApiGCSpecs", rb_cObject); @@ -115,6 +121,9 @@ void Init_gc_spec(void) { registered_after_rb_global_variable_float = DBL2NUM(6.28); rb_global_variable(®istered_after_rb_global_variable_float); + rb_gc_register_mark_object_not_referenced_float = DBL2NUM(1.61); + rb_gc_register_mark_object(rb_gc_register_mark_object_not_referenced_float); + rb_define_method(cls, "registered_tagged_address", registered_tagged_address, 0); rb_define_method(cls, "registered_reference_address", registered_reference_address, 0); rb_define_method(cls, "registered_before_rb_gc_register_address", get_registered_before_rb_gc_register_address, 0); @@ -131,6 +140,7 @@ void Init_gc_spec(void) { rb_define_method(cls, "rb_gc", gc_spec_rb_gc, 0); rb_define_method(cls, "rb_gc_adjust_memory_usage", gc_spec_rb_gc_adjust_memory_usage, 1); rb_define_method(cls, "rb_gc_register_mark_object", gc_spec_rb_gc_register_mark_object, 1); + rb_define_method(cls, "rb_gc_register_mark_object_not_referenced_float", gc_spec_rb_gc_register_mark_object_not_referenced_float, 0); rb_define_method(cls, "rb_gc_latest_gc_info", gc_spec_rb_gc_latest_gc_info, 1); } diff --git a/spec/ruby/optional/capi/ext/io_spec.c b/spec/ruby/optional/capi/ext/io_spec.c index 1a73331386..1b4d4fccce 100644 --- a/spec/ruby/optional/capi/ext/io_spec.c +++ b/spec/ruby/optional/capi/ext/io_spec.c @@ -330,13 +330,15 @@ static VALUE io_spec_errno_set(VALUE self, VALUE val) { } VALUE io_spec_mode_sync_flag(VALUE self, VALUE io) { + int mode; #ifdef RUBY_VERSION_IS_3_3 - if (rb_io_mode(io) & FMODE_SYNC) { + mode = rb_io_mode(io); #else rb_io_t *fp; GetOpenFile(io, fp); - if (fp->mode & FMODE_SYNC) { + mode = fp->mode; #endif + if (mode & FMODE_SYNC) { return Qtrue; } else { return Qfalse; diff --git a/spec/ruby/optional/capi/ext/object_spec.c b/spec/ruby/optional/capi/ext/object_spec.c index 8aa98cc5ce..aa60662e1e 100644 --- a/spec/ruby/optional/capi/ext/object_spec.c +++ b/spec/ruby/optional/capi/ext/object_spec.c @@ -390,22 +390,22 @@ static VALUE speced_allocator(VALUE klass) { return instance; } -static VALUE define_alloc_func(VALUE self, VALUE klass) { +static VALUE object_spec_rb_define_alloc_func(VALUE self, VALUE klass) { rb_define_alloc_func(klass, speced_allocator); return Qnil; } -static VALUE undef_alloc_func(VALUE self, VALUE klass) { +static VALUE object_spec_rb_undef_alloc_func(VALUE self, VALUE klass) { rb_undef_alloc_func(klass); return Qnil; } -static VALUE speced_allocator_p(VALUE self, VALUE klass) { +static VALUE object_spec_speced_allocator_p(VALUE self, VALUE klass) { rb_alloc_func_t allocator = rb_get_alloc_func(klass); return (allocator == speced_allocator) ? Qtrue : Qfalse; } -static VALUE custom_alloc_func_p(VALUE self, VALUE klass) { +static VALUE object_spec_custom_alloc_func_p(VALUE self, VALUE klass) { rb_alloc_func_t allocator = rb_get_alloc_func(klass); return allocator ? Qtrue : Qfalse; } @@ -485,10 +485,10 @@ void Init_object_spec(void) { rb_define_method(cls, "rb_ivar_defined", object_spec_rb_ivar_defined, 2); rb_define_method(cls, "rb_copy_generic_ivar", object_spec_rb_copy_generic_ivar, 2); rb_define_method(cls, "rb_free_generic_ivar", object_spec_rb_free_generic_ivar, 1); - rb_define_method(cls, "rb_define_alloc_func", define_alloc_func, 1); - rb_define_method(cls, "rb_undef_alloc_func", undef_alloc_func, 1); - rb_define_method(cls, "speced_allocator?", speced_allocator_p, 1); - rb_define_method(cls, "custom_alloc_func?", custom_alloc_func_p, 1); + rb_define_method(cls, "rb_define_alloc_func", object_spec_rb_define_alloc_func, 1); + rb_define_method(cls, "rb_undef_alloc_func", object_spec_rb_undef_alloc_func, 1); + rb_define_method(cls, "speced_allocator?", object_spec_speced_allocator_p, 1); + rb_define_method(cls, "custom_alloc_func?", object_spec_custom_alloc_func_p, 1); rb_define_method(cls, "not_implemented_method", rb_f_notimplement, -1); rb_define_method(cls, "rb_ivar_foreach", object_spec_rb_ivar_foreach, 1); } diff --git a/spec/ruby/optional/capi/ext/rubyspec.h b/spec/ruby/optional/capi/ext/rubyspec.h index 09135774af..1df274ead1 100644 --- a/spec/ruby/optional/capi/ext/rubyspec.h +++ b/spec/ruby/optional/capi/ext/rubyspec.h @@ -46,6 +46,10 @@ (RUBY_VERSION_MAJOR == (major) && RUBY_VERSION_MINOR == (minor) && RUBY_VERSION_TEENY < (teeny))) #define RUBY_VERSION_SINCE(major,minor,teeny) (!RUBY_VERSION_BEFORE(major, minor, teeny)) +#if RUBY_VERSION_SINCE(3, 4, 0) +#define RUBY_VERSION_IS_3_4 +#endif + #if RUBY_VERSION_SINCE(3, 3, 0) #define RUBY_VERSION_IS_3_3 #endif diff --git a/spec/ruby/optional/capi/ext/typed_data_spec.c b/spec/ruby/optional/capi/ext/typed_data_spec.c index 38889ecf5c..221f1c8ac4 100644 --- a/spec/ruby/optional/capi/ext/typed_data_spec.c +++ b/spec/ruby/optional/capi/ext/typed_data_spec.c @@ -106,6 +106,8 @@ VALUE sws_typed_wrap_struct(VALUE self, VALUE val) { return TypedData_Wrap_Struct(rb_cObject, &sample_typed_wrapped_struct_data_type, bar); } +#undef RUBY_UNTYPED_DATA_WARNING +#define RUBY_UNTYPED_DATA_WARNING 0 VALUE sws_untyped_wrap_struct(VALUE self, VALUE val) { int* data = (int*) malloc(sizeof(int)); *data = FIX2INT(val); diff --git a/spec/ruby/optional/capi/gc_spec.rb b/spec/ruby/optional/capi/gc_spec.rb index aaced56483..d9661328ab 100644 --- a/spec/ruby/optional/capi/gc_spec.rb +++ b/spec/ruby/optional/capi/gc_spec.rb @@ -108,6 +108,10 @@ describe "CApiGCSpecs" do it "can be called with an object" do @f.rb_gc_register_mark_object(Object.new).should be_nil end + + it "keeps the value alive even if the value is not referenced by any Ruby object" do + @f.rb_gc_register_mark_object_not_referenced_float.should == 1.61 + end end describe "rb_gc_latest_gc_info" do diff --git a/spec/ruby/optional/capi/io_spec.rb b/spec/ruby/optional/capi/io_spec.rb index 870abef3ea..bdec46f5e1 100644 --- a/spec/ruby/optional/capi/io_spec.rb +++ b/spec/ruby/optional/capi/io_spec.rb @@ -1,4 +1,5 @@ require_relative 'spec_helper' +require_relative '../../fixtures/io' load_extension('io') @@ -279,6 +280,22 @@ describe "C-API IO function" do it "raises an IOError if the IO is not initialized" do -> { @o.rb_io_maybe_wait_writable(0, IO.allocate, nil) }.should raise_error(IOError, "uninitialized stream") end + + it "can be interrupted" do + IOSpec.exhaust_write_buffer(@w_io) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @o.rb_io_maybe_wait_writable(0, @w_io, 10) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end end end @@ -355,6 +372,21 @@ describe "C-API IO function" do thr.join end + it "can be interrupted" do + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @o.rb_io_maybe_wait_readable(0, @r_io, 10, false) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end + it "raises an IOError if the IO is closed" do @r_io.close -> { @o.rb_io_maybe_wait_readable(0, @r_io, nil, false) }.should raise_error(IOError, "closed stream") @@ -438,6 +470,37 @@ describe "C-API IO function" do it "raises an IOError if the IO is not initialized" do -> { @o.rb_io_maybe_wait(0, IO.allocate, IO::WRITABLE, nil) }.should raise_error(IOError, "uninitialized stream") end + + it "can be interrupted when waiting for READABLE event" do + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @o.rb_io_maybe_wait(0, @r_io, IO::READABLE, 10) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end + + it "can be interrupted when waiting for WRITABLE event" do + IOSpec.exhaust_write_buffer(@w_io) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + t = Thread.new do + @o.rb_io_maybe_wait(0, @w_io, IO::WRITABLE, 10) + end + + Thread.pass until t.stop? + t.kill + t.join + + finish = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (finish - start).should < 9 + end end end diff --git a/spec/ruby/optional/capi/rbasic_spec.rb b/spec/ruby/optional/capi/rbasic_spec.rb index f3367e05ff..7b5b5b2fed 100644 --- a/spec/ruby/optional/capi/rbasic_spec.rb +++ b/spec/ruby/optional/capi/rbasic_spec.rb @@ -1,7 +1,9 @@ require_relative 'spec_helper' require_relative 'shared/rbasic' load_extension("rbasic") -load_extension("data") +ruby_version_is ""..."3.4" do + load_extension("data") +end load_extension("array") describe "RBasic support for regular objects" do @@ -12,33 +14,35 @@ describe "RBasic support for regular objects" do it_should_behave_like :rbasic end -describe "RBasic support for RData" do - before :all do - @specs = CApiRBasicRDataSpecs.new - @wrapping = CApiWrappedStructSpecs.new - @data = -> { [@wrapping.wrap_struct(1024), @wrapping.wrap_struct(1025)] } - end - it_should_behave_like :rbasic +ruby_version_is ""..."3.4" do + describe "RBasic support for RData" do + before :all do + @specs = CApiRBasicRDataSpecs.new + @wrapping = CApiWrappedStructSpecs.new + @data = -> { [@wrapping.wrap_struct(1024), @wrapping.wrap_struct(1025)] } + end + it_should_behave_like :rbasic - it "supports user flags" do - obj, _ = @data.call - initial = @specs.get_flags(obj) - @specs.set_flags(obj, 1 << 14 | 1 << 16 | initial).should == 1 << 14 | 1 << 16 | initial - @specs.get_flags(obj).should == 1 << 14 | 1 << 16 | initial - @specs.set_flags(obj, initial).should == initial - end + it "supports user flags" do + obj, _ = @data.call + initial = @specs.get_flags(obj) + @specs.set_flags(obj, 1 << 14 | 1 << 16 | initial).should == 1 << 14 | 1 << 16 | initial + @specs.get_flags(obj).should == 1 << 14 | 1 << 16 | initial + @specs.set_flags(obj, initial).should == initial + end - it "supports copying the flags from one object over to the other" do - obj1, obj2 = @data.call - initial = @specs.get_flags(obj1) - @specs.get_flags(obj2).should == initial - @specs.set_flags(obj1, 1 << 14 | 1 << 16 | initial) - @specs.get_flags(obj1).should == 1 << 14 | 1 << 16 | initial + it "supports copying the flags from one object over to the other" do + obj1, obj2 = @data.call + initial = @specs.get_flags(obj1) + @specs.get_flags(obj2).should == initial + @specs.set_flags(obj1, 1 << 14 | 1 << 16 | initial) + @specs.get_flags(obj1).should == 1 << 14 | 1 << 16 | initial - @specs.copy_flags(obj2, obj1) - @specs.get_flags(obj2).should == 1 << 14 | 1 << 16 | initial - @specs.set_flags(obj1, initial) - @specs.copy_flags(obj2, obj1) - @specs.get_flags(obj2).should == initial + @specs.copy_flags(obj2, obj1) + @specs.get_flags(obj2).should == 1 << 14 | 1 << 16 | initial + @specs.set_flags(obj1, initial) + @specs.copy_flags(obj2, obj1) + @specs.get_flags(obj2).should == initial + end end end diff --git a/spec/ruby/shared/kernel/at_exit.rb b/spec/ruby/shared/kernel/at_exit.rb index 16d41cb01c..29db79bb39 100644 --- a/spec/ruby/shared/kernel/at_exit.rb +++ b/spec/ruby/shared/kernel/at_exit.rb @@ -30,6 +30,12 @@ describe :kernel_at_exit, shared: true do result.lines.should.include?("The exception matches: true (message=foo)\n") end + it "gives access to an exception raised in a previous handler" do + code = "#{@method} { print '$!.message = ' + $!.message }; #{@method} { raise 'foo' }" + result = ruby_exe(code, args: "2>&1", exit_status: 1) + result.lines.should.include?("$!.message = foo") + end + it "both exceptions in a handler and in the main script are printed" do code = "#{@method} { raise 'at_exit_error' }; raise 'main_script_error'" result = ruby_exe(code, args: "2>&1", exit_status: 1) @@ -429,10 +429,6 @@ rb_str_format(int argc, const VALUE *argv, VALUE fmt) GETNUM(prec, precision); goto retry; - case '\n': - case '\0': - p--; - /* fall through */ case '%': if (flags != FNONE) { rb_raise(rb_eArgError, "invalid format character - %%"); @@ -89,6 +89,9 @@ VALUE rb_cSymbol; * another string (the shared root). * 3: STR_CHILLED (will be frozen in a future version) * The string appears frozen but can be mutated with a warning. + * 4: STR_PRECOMPUTED_HASH + * The string is embedded and has its precomputed hascode stored + * after the terminator. * 5: STR_SHARED_ROOT * Other strings may point to the contents of this string. When this * flag is set, STR_SHARED must not be set. @@ -116,6 +119,7 @@ VALUE rb_cSymbol; */ #define RUBY_MAX_CHAR_LEN 16 +#define STR_PRECOMPUTED_HASH FL_USER4 #define STR_SHARED_ROOT FL_USER5 #define STR_BORROWED FL_USER6 #define STR_TMPLOCK FL_USER7 @@ -240,6 +244,11 @@ rb_str_size_as_embedded(VALUE str) else { real_size = sizeof(struct RString); } + + if (FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)) { + real_size += sizeof(st_index_t); + } + return real_size; } @@ -257,6 +266,7 @@ static VALUE str_new(VALUE klass, const char *ptr, long len); static void str_make_independent_expand(VALUE str, long len, long expand, const int termlen); static inline void str_modifiable(VALUE str); static VALUE rb_str_downcase(int argc, VALUE *argv, VALUE str); +static inline VALUE str_alloc_embed(VALUE klass, size_t capa); static inline void str_make_independent(VALUE str) @@ -334,7 +344,7 @@ mustnot_wchar(VALUE str) static int fstring_cmp(VALUE a, VALUE b); -static VALUE register_fstring(VALUE str, bool copy); +static VALUE register_fstring(VALUE str, bool copy, bool precompute_hash); const struct st_hash_type rb_fstring_hash_type = { fstring_cmp, @@ -343,9 +353,42 @@ const struct st_hash_type rb_fstring_hash_type = { #define BARE_STRING_P(str) (!FL_ANY_RAW(str, FL_EXIVAR) && RBASIC_CLASS(str) == rb_cString) +static inline st_index_t +str_do_hash(VALUE str) +{ + st_index_t h = rb_memhash((const void *)RSTRING_PTR(str), RSTRING_LEN(str)); + int e = RSTRING_LEN(str) ? ENCODING_GET(str) : 0; + if (e && !is_ascii_string(str)) { + h = rb_hash_end(rb_hash_uint32(h, (uint32_t)e)); + } + return h; +} + +static VALUE +str_precompute_hash(VALUE str) +{ + RUBY_ASSERT(!FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)); + RUBY_ASSERT(STR_EMBED_P(str)); + +#if RUBY_DEBUG + size_t used_bytes = (RSTRING_LEN(str) + TERM_LEN(str)); + size_t free_bytes = str_embed_capa(str) - used_bytes; + RUBY_ASSERT(free_bytes >= sizeof(st_index_t)); +#endif + + typedef struct {char bytes[sizeof(st_index_t)];} unaligned_index; + union {st_index_t i; unaligned_index b;} u = {.i = str_do_hash(str)}; + *(unaligned_index *)(RSTRING_END(str) + TERM_LEN(str)) = u.b; + + FL_SET(str, STR_PRECOMPUTED_HASH); + + return str; +} + struct fstr_update_arg { VALUE fstr; bool copy; + bool precompute_hash; }; static int @@ -370,8 +413,23 @@ fstr_update_callback(st_data_t *key, st_data_t *value, st_data_t data, int exist else { if (FL_TEST_RAW(str, STR_FAKESTR)) { if (arg->copy) { - VALUE new_str = str_new(rb_cString, RSTRING(str)->as.heap.ptr, RSTRING(str)->len); - rb_enc_copy(new_str, str); + VALUE new_str; + long len = RSTRING_LEN(str); + long capa = len + sizeof(st_index_t); + int term_len = TERM_LEN(str); + + if (arg->precompute_hash && STR_EMBEDDABLE_P(capa, term_len)) { + new_str = str_alloc_embed(rb_cString, capa + term_len); + memcpy(RSTRING_PTR(new_str), RSTRING_PTR(str), len); + STR_SET_LEN(new_str, RSTRING_LEN(str)); + TERM_FILL(RSTRING_END(new_str), TERM_LEN(str)); + rb_enc_copy(new_str, str); + str_precompute_hash(new_str); + } + else { + new_str = str_new(rb_cString, RSTRING(str)->as.heap.ptr, RSTRING(str)->len); + rb_enc_copy(new_str, str); + } str = new_str; } else { @@ -428,7 +486,7 @@ rb_fstring(VALUE str) if (!FL_TEST_RAW(str, FL_FREEZE | STR_NOFREE | STR_CHILLED)) rb_str_resize(str, RSTRING_LEN(str)); - fstr = register_fstring(str, FALSE); + fstr = register_fstring(str, false, false); if (!bare) { str_replace_shared_without_enc(str, fstr); @@ -439,10 +497,12 @@ rb_fstring(VALUE str) } static VALUE -register_fstring(VALUE str, bool copy) +register_fstring(VALUE str, bool copy, bool precompute_hash) { - struct fstr_update_arg args; - args.copy = copy; + struct fstr_update_arg args = { + .copy = copy, + .precompute_hash = precompute_hash + }; RB_VM_LOCK_ENTER(); { @@ -500,14 +560,14 @@ VALUE rb_fstring_new(const char *ptr, long len) { struct RString fake_str; - return register_fstring(setup_fake_str(&fake_str, ptr, len, ENCINDEX_US_ASCII), FALSE); + return register_fstring(setup_fake_str(&fake_str, ptr, len, ENCINDEX_US_ASCII), false, false); } VALUE rb_fstring_enc_new(const char *ptr, long len, rb_encoding *enc) { struct RString fake_str; - return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), FALSE); + return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), false, false); } VALUE @@ -871,6 +931,7 @@ empty_str_alloc(VALUE klass) RUBY_DTRACE_CREATE_HOOK(STRING, 0); VALUE str = str_alloc_embed(klass, 0); memset(RSTRING(str)->as.embed.ary, 0, str_embed_capa(str)); + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); return str; } @@ -1492,6 +1553,7 @@ str_new_frozen_buffer(VALUE klass, VALUE orig, int copy_encoding) STR_SET_EMBED(str); memcpy(RSTRING_PTR(str), RSTRING_PTR(orig), RSTRING_LEN(orig)); STR_SET_LEN(str, RSTRING_LEN(orig)); + ENC_CODERANGE_SET(str, ENC_CODERANGE(orig)); TERM_FILL(RSTRING_END(str), TERM_LEN(orig)); } else { @@ -1836,12 +1898,6 @@ rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chil return new_str; } -bool -rb_str_chilled_p(VALUE str) -{ - return CHILLED_STRING_P(str); -} - /* * * call-seq: @@ -2453,6 +2509,9 @@ rb_check_lockedtmp(VALUE str) static inline void str_modifiable(VALUE str) { + if (CHILLED_STRING_P(str)) { + CHILLED_STRING_MUTATED(str); + } rb_check_lockedtmp(str); rb_check_frozen(str); } @@ -3053,7 +3112,7 @@ rb_str_freeze(VALUE str) static VALUE str_uplus(VALUE str) { - if (OBJ_FROZEN(str)) { + if (OBJ_FROZEN(str) || CHILLED_STRING_P(str)) { return rb_str_dup(str); } else { @@ -3136,7 +3195,11 @@ rb_str_set_len(VALUE str, long len) } int cr = ENC_CODERANGE(str); - if (cr == ENC_CODERANGE_UNKNOWN) { + if (len == 0) { + /* Empty string does not contain non-ASCII */ + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); + } + else if (cr == ENC_CODERANGE_UNKNOWN) { /* Leave unknown. */ } else if (len > RSTRING_LEN(str)) { @@ -3532,6 +3595,22 @@ rb_str_concat_multi(int argc, VALUE *argv, VALUE str) * s = 'foo' * s << 33 # => "foo!" * + * If that codepoint is not representable in the encoding of + * _string_, RangeError is raised. + * + * s = 'foo' + * s.encoding # => <Encoding:UTF-8> + * s << 0x00110000 # 1114112 out of char range (RangeError) + * s = 'foo'.encode('EUC-JP') + * s << 0x00800080 # invalid codepoint 0x800080 in EUC-JP (RangeError) + * + * If the encoding is US-ASCII and the codepoint is 0..0xff, _string_ + * is automatically promoted to ASCII-8BIT. + * + * s = 'foo'.encode('US-ASCII') + * s << 0xff + * s.encoding # => #<Encoding:BINARY (ASCII-8BIT)> + * * Related: String#concat, which takes multiple arguments. */ VALUE @@ -3655,12 +3734,15 @@ rb_str_prepend_multi(int argc, VALUE *argv, VALUE str) st_index_t rb_str_hash(VALUE str) { - st_index_t h = rb_memhash((const void *)RSTRING_PTR(str), RSTRING_LEN(str)); - int e = RSTRING_LEN(str) ? ENCODING_GET(str) : 0; - if (e && !is_ascii_string(str)) { - h = rb_hash_end(rb_hash_uint32(h, (uint32_t)e)); + if (FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)) { + typedef struct {char bytes[sizeof(st_index_t)];} unaligned_index; + st_index_t precomputed_hash = ((union {st_index_t i; unaligned_index b;} *)(RSTRING_END(str) + TERM_LEN(str)))->i; + + RUBY_ASSERT(precomputed_hash == str_do_hash(str)); + return precomputed_hash; } - return h; + + return str_do_hash(str); } int @@ -5084,7 +5166,9 @@ rb_str_upto_each(VALUE beg, VALUE end, int excl, int (*each)(VALUE, VALUE), VALU if (c > e || (excl && c == e)) return beg; for (;;) { - if ((*each)(rb_enc_str_new(&c, 1, enc), arg)) break; + VALUE str = rb_enc_str_new(&c, 1, enc); + ENC_CODERANGE_SET(str, RUBY_ENC_CODERANGE_7BIT); + if ((*each)(str, arg)) break; if (!excl && c == e) break; c++; if (excl && c == e) break; @@ -12130,7 +12214,7 @@ VALUE rb_interned_str(const char *ptr, long len) { struct RString fake_str; - return register_fstring(setup_fake_str(&fake_str, ptr, len, ENCINDEX_US_ASCII), TRUE); + return register_fstring(setup_fake_str(&fake_str, ptr, len, ENCINDEX_US_ASCII), true, false); } VALUE @@ -12147,7 +12231,18 @@ rb_enc_interned_str(const char *ptr, long len, rb_encoding *enc) } struct RString fake_str; - return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), TRUE); + return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), true, false); +} + +VALUE +rb_enc_literal_str(const char *ptr, long len, rb_encoding *enc) +{ + if (enc != NULL && UNLIKELY(rb_enc_autoload_p(enc))) { + rb_enc_autoload(enc); + } + + struct RString fake_str; + return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), true, true); } VALUE @@ -311,7 +311,7 @@ # - {Comparing}[rdoc-ref:String@Methods+for+Comparing] # - {Modifying a String}[rdoc-ref:String@Methods+for+Modifying+a+String] # - {Converting to New String}[rdoc-ref:String@Methods+for+Converting+to+New+String] -# - {Converting to Non-String}[rdoc-ref:String@Methods+for+Converting+to+Non--5CString] +# - {Converting to Non-String}[rdoc-ref:String@Methods+for+Converting+to+Non-String] # - {Iterating}[rdoc-ref:String@Methods+for+Iterating] # # === Methods for Creating a +String+ @@ -100,7 +100,12 @@ sym_type(VALUE sym) #define is_class_sym(sym) (sym_type(sym)==ID_CLASS) #define is_junk_sym(sym) (sym_type(sym)==ID_JUNK) -RUBY_FUNC_EXPORTED const uint_least32_t ruby_global_name_punct_bits[(0x7e - 0x20 + 31) / 32]; +#ifndef RIPPER +RUBY_FUNC_EXPORTED +#else +RUBY_EXTERN +#endif +const uint_least32_t ruby_global_name_punct_bits[(0x7e - 0x20 + 31) / 32]; static inline int is_global_name_punct(const int c) diff --git a/template/Makefile.in b/template/Makefile.in index d9a3cbc065..033ac56cb3 100644 --- a/template/Makefile.in +++ b/template/Makefile.in @@ -493,9 +493,8 @@ clean-local:: enc/encinit.c enc/encinit.$(OBJEXT) $(pkgconfig_DATA) \ ruby-runner.$(OBJEXT) ruby-runner.h \ || $(NULLCMD) - @$(RM) $(ALLOBJS:.$(OBJEXT)=.bc) - @$(RM) $(ALLOBJS:.$(OBJEXT)=.i) - @$(RM) $(ALLOBJS:.$(OBJEXT)=.s) + $(Q)find . ! -type d \( -name '*.bc' -o -name '*.[is]' \) -exec rm -f {} + || true + distclean-local:: $(Q)$(RM) \ @@ -663,3 +662,32 @@ yes-test-leaked-globals: yes-test-leaked-globals-precheck PLATFORM=$(hdrdir)/ruby/$(PLATFORM_DIR).h $(srcdir)/configure.ac \ $(COMMONOBJS) $(LIBRUBY_FOR_LEAKED_GLOBALS:yes=$(LIBRUBY_SO)) $(ACTIONS_ENDGROUP) + +test-syntax-suggest-precheck: $(TEST_RUNNABLE)-test-syntax-suggest-precheck +no-test-syntax-suggest-precheck: +yes-test-syntax-suggest-precheck: main + +test-syntax-suggest-prepare: $(TEST_RUNNABLE)-test-syntax-suggest-prepare +no-test-syntax-suggest-prepare: no-test-syntax-suggest-precheck +yes-test-syntax-suggest-prepare: yes-test-syntax-suggest-precheck + $(ACTIONS_GROUP) + $(XRUBY) -C "$(srcdir)" bin/gem install --no-document \ + --install-dir .bundle --conservative "rspec:~> 3" + $(ACTIONS_ENDGROUP) + +RSPECOPTS = +SYNTAX_SUGGEST_SPECS = +PREPARE_SYNTAX_SUGGEST = $(TEST_RUNNABLE)-test-syntax-suggest-prepare +test-syntax-suggest: $(TEST_RUNNABLE)-test-syntax-suggest +yes-test-syntax-suggest: $(PREPARE_SYNTAX_SUGGEST) + $(ACTIONS_GROUP) + $(XRUBY) -C $(srcdir) -Ispec/syntax_suggest:spec/lib .bundle/bin/rspec \ + --require rspec/expectations \ + --require spec_helper --require formatter_overrides --require spec_coverage \ + $(RSPECOPTS) spec/syntax_suggest/$(SYNTAX_SUGGEST_SPECS) + $(ACTIONS_ENDGROUP) +no-test-syntax-suggest: + +yesterday: + $(GIT) -C $(srcdir) reset --hard \ + `$(GIT) -C $(srcdir) log -1 --before=00:00+0900 --format=%H` diff --git a/template/prelude.c.tmpl b/template/prelude.c.tmpl index e17a75da79..af493dfaca 100644 --- a/template/prelude.c.tmpl +++ b/template/prelude.c.tmpl @@ -154,6 +154,18 @@ prelude_ast_value(VALUE name, VALUE code, int line) return ast_value; } +static void +pm_prelude_load(pm_parse_result_t *result, VALUE name, VALUE code, int line) +{ + pm_options_line_set(&result->options, line); + VALUE error = pm_parse_string(result, code, name); + + if (!NIL_P(error)) { + pm_parse_result_free(result); + rb_exc_raise(error); + } +} + % end % if @builtin_count > 0 #define PRELUDE_VAST(n, name_str, start_line) \ @@ -176,6 +188,28 @@ rb_builtin_ast_value(const char *feature_name, VALUE *name_str) return ast_value; } +bool +pm_builtin_ast_value(pm_parse_result_t *result, const char *feature_name, VALUE *name_str) +{ + const size_t prefix_len = rb_strlen_lit("<internal:"); + size_t namelen = strlen(feature_name); + +% @preludes.each_value do |i, prelude, lines, sub, start_line| +% if sub + if ( + (sizeof(prelude_name<%= i %>) - prefix_len - 2 == namelen) && + (strncmp(prelude_name<%= i %> + prefix_len, feature_name, namelen) == 0) + ) { + *name_str = PRELUDE_NAME(<%= i %>); + pm_prelude_load(result, *name_str, PRELUDE_CODE(<%= i %>), <%= start_line %>); + return true; + } +% end +% end + + return false; +} + % end % if @prelude_count > 0 COMPILER_WARNING_PUSH @@ -198,13 +232,22 @@ prelude_eval(VALUE code, VALUE name, int line) 0, /* int debug_level; */ }; - rb_ast_t *ast; - VALUE ast_value = prelude_ast_value(name, code, line); - ast = rb_ruby_ast_data_get(ast_value); - rb_iseq_eval(rb_iseq_new_with_opt(ast_value, name, name, Qnil, line, - NULL, 0, ISEQ_TYPE_TOP, &optimization, - Qnil)); - rb_ast_dispose(ast); + if (*rb_ruby_prism_ptr()) { + pm_parse_result_t result = { 0 }; + pm_prelude_load(&result, name, code, line); + rb_iseq_eval(pm_iseq_new_with_opt(&result.node, name, name, Qnil, line, + NULL, 0, ISEQ_TYPE_TOP, &optimization)); + pm_parse_result_free(&result); + } + else { + rb_ast_t *ast; + VALUE ast_value = prelude_ast_value(name, code, line); + ast = rb_ruby_ast_data_get(ast_value); + rb_iseq_eval(rb_iseq_new_with_opt(ast_value, name, name, Qnil, line, + NULL, 0, ISEQ_TYPE_TOP, &optimization, + Qnil)); + rb_ast_dispose(ast); + } } COMPILER_WARNING_POP diff --git a/test/-ext-/integer/test_my_integer.rb b/test/-ext-/integer/test_my_integer.rb index 1b6f8489f8..0dfa234921 100644 --- a/test/-ext-/integer/test_my_integer.rb +++ b/test/-ext-/integer/test_my_integer.rb @@ -8,19 +8,13 @@ class Test_MyInteger < Test::Unit::TestCase Bug::Integer::MyInteger.new.to_f end - begin - Bug::Integer::MyInteger.class_eval do - def to_f - end + int = Class.new(Bug::Integer::MyInteger) do + def to_f end + end - assert_nothing_raised do - Bug::Integer::MyInteger.new.to_f - end - ensure - Bug::Integer::MyInteger.class_eval do - remove_method :to_f - end + assert_nothing_raised do + int.new.to_f end end @@ -29,20 +23,14 @@ class Test_MyInteger < Test::Unit::TestCase Bug::Integer::MyInteger.new <=> 0 end - begin - Bug::Integer::MyInteger.class_eval do - def <=>(other) - 0 - end + int = Class.new(Bug::Integer::MyInteger) do + def <=>(other) + 0 end + end - assert_nothing_raised do - Bug::Integer::MyInteger.new <=> 0 - end - ensure - Bug::Integer::MyInteger.class_eval do - remove_method :<=> - end + assert_nothing_raised do + int.new <=> 0 end end end diff --git a/test/-ext-/string/test_chilled.rb b/test/-ext-/string/test_chilled.rb deleted file mode 100644 index dccab61ced..0000000000 --- a/test/-ext-/string/test_chilled.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'test/unit' -require '-test-/string' - -class Test_String_ChilledString < Test::Unit::TestCase - def test_rb_str_chilled_p - str = "" - assert_equal true, Bug::String.rb_str_chilled_p(str) - end - - def test_rb_str_chilled_p_frozen - str = "".freeze - assert_equal false, Bug::String.rb_str_chilled_p(str) - end - - def test_rb_str_chilled_p_mutable - str = "".dup - assert_equal false, Bug::String.rb_str_chilled_p(str) - end -end diff --git a/test/-ext-/thread/test_instrumentation_api.rb b/test/-ext-/thread/test_instrumentation_api.rb index 9a3b67fa10..663e41be53 100644 --- a/test/-ext-/thread/test_instrumentation_api.rb +++ b/test/-ext-/thread/test_instrumentation_api.rb @@ -54,7 +54,7 @@ class TestThreadInstrumentation < Test::Unit::TestCase thread&.join end - def test_muti_thread_timeline + def test_multi_thread_timeline threads = nil full_timeline = record do threads = threaded_cpu_bound_work(1.0) @@ -68,7 +68,7 @@ class TestThreadInstrumentation < Test::Unit::TestCase threads.each do |thread| timeline = timeline_for(thread, full_timeline) assert_consistent_timeline(timeline) - assert timeline.count(:suspended) > 1, "Expected threads to yield suspended at least once: #{timeline.inspect}" + assert_operator timeline.count(:suspended), :>=, 1, "Expected threads to yield suspended at least once: #{timeline.inspect}" end timeline = timeline_for(Thread.current, full_timeline) diff --git a/test/.excludes-prism/TestAssignment.rb b/test/.excludes-prism/TestAssignment.rb deleted file mode 100644 index 23a1b4c5e7..0000000000 --- a/test/.excludes-prism/TestAssignment.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_massign_order, "https://github.com/ruby/prism/issues/2370") diff --git a/test/.excludes-prism/TestAssignmentGen.rb b/test/.excludes-prism/TestAssignmentGen.rb deleted file mode 100644 index 5f3bb12ef7..0000000000 --- a/test/.excludes-prism/TestAssignmentGen.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_assignment, "https://github.com/ruby/prism/issues/2370") diff --git a/test/.excludes-prism/TestCoverage.rb b/test/.excludes-prism/TestCoverage.rb deleted file mode 100644 index 20f9972f89..0000000000 --- a/test/.excludes-prism/TestCoverage.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_eval, "unknown") diff --git a/test/.excludes-prism/TestIRB/RubyLexTest.rb b/test/.excludes-prism/TestIRB/RubyLexTest.rb deleted file mode 100644 index 2274ae62cf..0000000000 --- a/test/.excludes-prism/TestIRB/RubyLexTest.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_code_block_open_with_should_continue, "symbol encoding") diff --git a/test/.excludes-prism/TestISeq.rb b/test/.excludes-prism/TestISeq.rb deleted file mode 100644 index 3768d8fc05..0000000000 --- a/test/.excludes-prism/TestISeq.rb +++ /dev/null @@ -1,3 +0,0 @@ -exclude(:test_each_child, "https://github.com/ruby/prism/issues/2660") -exclude(:test_syntax_error_message, "Assertion checks against specific error format") -exclude(:test_trace_points, "https://github.com/ruby/prism/issues/2660") diff --git a/test/.excludes-prism/TestM17N.rb b/test/.excludes-prism/TestM17N.rb index e3c2d43b74..7f8c44d02a 100644 --- a/test/.excludes-prism/TestM17N.rb +++ b/test/.excludes-prism/TestM17N.rb @@ -1,2 +1 @@ -exclude(:test_regexp_usascii, "unknown") -exclude(:test_string_mixed_unicode, "unknown") +exclude(:test_regexp_usascii, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestMixedUnicodeEscape.rb b/test/.excludes-prism/TestMixedUnicodeEscape.rb index 09e3cc168b..7bf964ebf1 100644 --- a/test/.excludes-prism/TestMixedUnicodeEscape.rb +++ b/test/.excludes-prism/TestMixedUnicodeEscape.rb @@ -1 +1 @@ -exclude(:test_basic, "unknown") +exclude(:test_basic, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb deleted file mode 100644 index 4428e5a0e6..0000000000 --- a/test/.excludes-prism/TestParse.rb +++ /dev/null @@ -1,18 +0,0 @@ -exclude(:test_error_def_in_argument, "unknown") -exclude(:test_global_variable, "unknown") -exclude(:test_invalid_char, "unknown") -exclude(:test_location_of_invalid_token, "unknown") -exclude(:test_percent, "unknown") -exclude(:test_question, "unknown") -exclude(:test_shareable_constant_value_ignored, "unknown") -exclude(:test_shareable_constant_value_nested, "ractor support") -exclude(:test_shareable_constant_value_nonliteral, "ractor support") -exclude(:test_shareable_constant_value_simple, "ractor support") -exclude(:test_shareable_constant_value_unfrozen, "ractor support") -exclude(:test_shareable_constant_value_unshareable_literal, "ractor support") -exclude(:test_string, "unknown") -exclude(:test_truncated_source_line, "unknown") -exclude(:test_unexpected_eof, "unknown") -exclude(:test_unexpected_token_after_numeric, "unknown") -exclude(:test_unused_variable, "missing warning") -exclude(:test_void_value_in_rhs, "unknown") diff --git a/test/.excludes-prism/TestPatternMatching.rb b/test/.excludes-prism/TestPatternMatching.rb deleted file mode 100644 index cfd0c6bed9..0000000000 --- a/test/.excludes-prism/TestPatternMatching.rb +++ /dev/null @@ -1,2 +0,0 @@ -exclude(:test_hash_pattern, "useless literal warning missing") -exclude(:test_invalid_syntax, "[a:] is disallowed") diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb deleted file mode 100644 index 62e704fac5..0000000000 --- a/test/.excludes-prism/TestRegexp.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_unescape, "unknown") diff --git a/test/.excludes-prism/TestRequire.rb b/test/.excludes-prism/TestRequire.rb deleted file mode 100644 index a7f66c5d80..0000000000 --- a/test/.excludes-prism/TestRequire.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_require_nonascii_path_shift_jis, "encoding") diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 926f0c5a5e..853f23a3b9 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,5 +1 @@ -exclude(:test_debug_frozen_string_in_array_literal, "unknown") -exclude(:test_debug_frozen_string, "unknown") -exclude(:test_dregexp, "https://github.com/ruby/prism/issues/2664") -exclude(:test_hash_value_omission, "unknown") -exclude(:test_string, "https://github.com/ruby/prism/issues/2331") +exclude(:test_dregexp, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb deleted file mode 100644 index 9402546e68..0000000000 --- a/test/.excludes-prism/TestSyntax.rb +++ /dev/null @@ -1,19 +0,0 @@ -exclude(:test__END___cr, "unknown") -exclude(:test_argument_forwarding_with_super, "unknown") -exclude(:test_argument_forwarding, "unknown") -exclude(:test_dedented_heredoc_concatenation, "unknown") -exclude(:test_dedented_heredoc_continued_line, "unknown") -exclude(:test_duplicated_when, "unknown") -exclude(:test_error_message_encoding, "unknown") -exclude(:test_heredoc_cr, "unknown") -exclude(:test_heredoc_no_terminator, "unknown") -exclude(:test_it, "https://github.com/ruby/prism/issues/2323") -exclude(:test_keyword_invalid_name, "unknown") -exclude(:test_keyword_self_reference, "unknown") -exclude(:test_methoddef_endless_command, "unknown") -exclude(:test_numbered_parameter, "unknown") -exclude(:test_optional_self_reference, "unknown") -exclude(:test_syntax_error_at_newline, "unknown") -exclude(:test_unterminated_heredoc_cr, "unknown") -exclude(:test_warn_balanced, "unknown") -exclude(:test_warn_unreachable, "unknown") diff --git a/test/.excludes-prism/TestUnicodeEscape.rb b/test/.excludes-prism/TestUnicodeEscape.rb deleted file mode 100644 index add4911bc2..0000000000 --- a/test/.excludes-prism/TestUnicodeEscape.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_fail, "unknown") diff --git a/test/error_highlight/test_error_highlight.rb b/test/error_highlight/test_error_highlight.rb index 2095970af1..9af530b69f 100644 --- a/test/error_highlight/test_error_highlight.rb +++ b/test/error_highlight/test_error_highlight.rb @@ -1,9 +1,17 @@ require "test/unit" require "error_highlight" +require "did_you_mean" require "tempfile" class ErrorHighlightTest < Test::Unit::TestCase + # We can't revisit instruction sequences to find node ids if the prism + # compiler was used instead of the parse.y compiler. In that case, we'll omit + # some tests. + def self.compiling_with_prism? + RubyVM::InstructionSequence.compile("").to_a[4][:parser] == :prism + end + class DummyFormatter def self.message_for(corrections) "" @@ -868,7 +876,7 @@ uninitialized constant ErrorHighlightTest::NotDefined end end - if ErrorHighlight.const_get(:Spotter).const_get(:OPT_GETCONSTANT_PATH) + if ErrorHighlight.const_get(:Spotter).const_get(:OPT_GETCONSTANT_PATH) && !compiling_with_prism? def test_COLON2_5 # Unfortunately, we cannot identify which `NotDefined` caused the NameError assert_error_message(NameError, <<~END) do @@ -1334,6 +1342,7 @@ undefined method `foo' for #{ NIL_RECV_MESSAGE } def test_spot_with_node omit unless RubyVM::AbstractSyntaxTree.respond_to?(:node_id_for_backtrace_location) + omit if ErrorHighlightTest.compiling_with_prism? begin raise_name_error diff --git a/test/irb/command/test_disable_irb.rb b/test/irb/command/test_disable_irb.rb new file mode 100644 index 0000000000..14a20043d5 --- /dev/null +++ b/test/irb/command/test_disable_irb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: false +require 'irb' + +require_relative "../helper" + +module TestIRB + class DisableIRBTest < IntegrationTestCase + def test_disable_irb_disable_further_irb_breakpoints + write_ruby <<~'ruby' + puts "First line" + puts "Second line" + binding.irb + puts "Third line" + binding.irb + puts "Fourth line" + ruby + + output = run_ruby_file do + type "disable_irb" + end + + assert_match(/First line\r\n/, output) + assert_match(/Second line\r\n/, output) + assert_match(/Third line\r\n/, output) + assert_match(/Fourth line\r\n/, output) + end + end +end diff --git a/test/irb/test_init.rb b/test/irb/test_init.rb index f11d7398c8..3207c2898b 100644 --- a/test/irb/test_init.rb +++ b/test/irb/test_init.rb @@ -268,15 +268,95 @@ module TestIRB end end + class ConfigValidationTest < TestCase + def setup + @original_home = ENV["HOME"] + @original_irbrc = ENV["IRBRC"] + # To prevent the test from using the user's .irbrc file + ENV["HOME"] = @home = Dir.mktmpdir + IRB.instance_variable_set(:@existing_rc_name_generators, nil) + super + end + + def teardown + super + ENV["IRBRC"] = @original_irbrc + ENV["HOME"] = @original_home + File.unlink(@irbrc) + Dir.rmdir(@home) + end + + def test_irb_name_converts_non_string_values_to_string + assert_no_irb_validation_error(<<~'RUBY') + IRB.conf[:IRB_NAME] = :foo + RUBY + + assert_equal "foo", IRB.conf[:IRB_NAME] + end + + def test_irb_rc_name_only_takes_callable_objects + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:IRB_RC] should be a callable object. Got :foo.") + IRB.conf[:IRB_RC] = :foo + RUBY + end + + def test_back_trace_limit_only_accepts_integers + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got \"foo\".") + IRB.conf[:BACK_TRACE_LIMIT] = "foo" + RUBY + end + + def test_prompt_only_accepts_hash + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:PROMPT] should be a Hash. Got \"foo\".") + IRB.conf[:PROMPT] = "foo" + RUBY + end + + def test_eval_history_only_accepts_integers + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:EVAL_HISTORY] should be an integer. Got \"foo\".") + IRB.conf[:EVAL_HISTORY] = "foo" + RUBY + end + + private + + def assert_irb_validation_error(rc_content, error_message) + write_rc rc_content + + assert_raise_with_message(TypeError, error_message) do + IRB.setup(__FILE__) + end + end + + def assert_no_irb_validation_error(rc_content) + write_rc rc_content + + assert_nothing_raised do + IRB.setup(__FILE__) + end + end + + def write_rc(content) + @irbrc = Tempfile.new('irbrc') + @irbrc.write(content) + @irbrc.close + ENV['IRBRC'] = @irbrc.path + end + end + class InitIntegrationTest < IntegrationTestCase - def test_load_error_in_rc_file_is_warned - write_rc <<~'IRBRC' - require "file_that_does_not_exist" - IRBRC + def setup + super write_ruby <<~'RUBY' binding.irb RUBY + end + + def test_load_error_in_rc_file_is_warned + write_rc <<~'IRBRC' + require "file_that_does_not_exist" + IRBRC output = run_ruby_file do type "'foobar'" @@ -293,10 +373,6 @@ module TestIRB raise "I'm an error" IRBRC - write_ruby <<~'RUBY' - binding.irb - RUBY - output = run_ruby_file do type "'foobar'" type "exit" diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index 28be744088..3d8044c5a1 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -825,6 +825,13 @@ module TestIRB end class BacktraceFilteringTest < TestIRB::IntegrationTestCase + def setup + super + # These tests are sensitive to warnings, so we disable them + original_rubyopt = [ENV["RUBYOPT"], @envs["RUBYOPT"]].compact.join(" ") + @envs["RUBYOPT"] = original_rubyopt + " -W0" + end + def test_backtrace_filtering write_ruby <<~'RUBY' def foo diff --git a/test/logger/test_logperiod.rb b/test/logger/test_logperiod.rb index 6e6e5e9533..ee38d877c6 100644 --- a/test/logger/test_logperiod.rb +++ b/test/logger/test_logperiod.rb @@ -1,80 +1,67 @@ # coding: US-ASCII # frozen_string_literal: false -require 'logger' -require 'time' +require "logger" +require "time" class TestLogPeriod < Test::Unit::TestCase def test_next_rotate_time time = Time.parse("2019-07-18 13:52:02") - daily_result = Logger::Period.next_rotate_time(time, 'daily') - next_day = Time.parse("2019-07-19 00:00:00") - assert_equal(next_day, daily_result) + assert_next_rotate_time_words(time, "2019-07-19 00:00:00", ["daily", :daily]) + assert_next_rotate_time_words(time, "2019-07-21 00:00:00", ["weekly", :weekly]) + assert_next_rotate_time_words(time, "2019-08-01 00:00:00", ["monthly", :monthly]) - weekly_result = Logger::Period.next_rotate_time(time, 'weekly') - next_week = Time.parse("2019-07-21 00:00:00") - assert_equal(next_week, weekly_result) - - monthly_result = Logger::Period.next_rotate_time(time, 'monthly') - next_month = Time.parse("2019-08-1 00:00:00") - assert_equal(next_month, monthly_result) - - assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, 'invalid') } + assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, "invalid") } end def test_next_rotate_time_extreme_cases # First day of Month and Saturday time = Time.parse("2018-07-01 00:00:00") - daily_result = Logger::Period.next_rotate_time(time, 'daily') - next_day = Time.parse("2018-07-02 00:00:00") - assert_equal(next_day, daily_result) - - weekly_result = Logger::Period.next_rotate_time(time, 'weekly') - next_week = Time.parse("2018-07-08 00:00:00") - assert_equal(next_week, weekly_result) + assert_next_rotate_time_words(time, "2018-07-02 00:00:00", ["daily", :daily]) + assert_next_rotate_time_words(time, "2018-07-08 00:00:00", ["weekly", :weekly]) + assert_next_rotate_time_words(time, "2018-08-01 00:00:00", ["monthly", :monthly]) - monthly_result = Logger::Period.next_rotate_time(time, 'monthly') - next_month = Time.parse("2018-08-1 00:00:00") - assert_equal(next_month, monthly_result) - - assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, 'invalid') } + assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, "invalid") } end def test_previous_period_end time = Time.parse("2019-07-18 13:52:02") - daily_result = Logger::Period.previous_period_end(time, 'daily') - day_ago = Time.parse("2019-07-17 23:59:59") - assert_equal(day_ago, daily_result) - - weekly_result = Logger::Period.previous_period_end(time, 'weekly') - week_ago = Time.parse("2019-07-13 23:59:59") - assert_equal(week_ago, weekly_result) - - monthly_result = Logger::Period.previous_period_end(time, 'monthly') - month_ago = Time.parse("2019-06-30 23:59:59") - assert_equal(month_ago, monthly_result) + assert_previous_period_end_words(time, "2019-07-17 23:59:59", ["daily", :daily]) + assert_previous_period_end_words(time, "2019-07-13 23:59:59", ["weekly", :weekly]) + assert_previous_period_end_words(time, "2019-06-30 23:59:59", ["monthly", :monthly]) - assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, 'invalid') } + assert_raise(ArgumentError) { Logger::Period.previous_period_end(time, "invalid") } end def test_previous_period_end_extreme_cases # First day of Month and Saturday time = Time.parse("2018-07-01 00:00:00") + previous_date = "2018-06-30 23:59:59" - daily_result = Logger::Period.previous_period_end(time, 'daily') - day_ago = Time.parse("2018-06-30 23:59:59") - assert_equal(day_ago, daily_result) + assert_previous_period_end_words(time, previous_date, ["daily", :daily]) + assert_previous_period_end_words(time, previous_date, ["weekly", :weekly]) + assert_previous_period_end_words(time, previous_date, ["monthly", :monthly]) - weekly_result = Logger::Period.previous_period_end(time, 'weekly') - week_ago = Time.parse("2018-06-30 23:59:59") - assert_equal(week_ago, weekly_result) + assert_raise(ArgumentError) { Logger::Period.previous_period_end(time, "invalid") } + end + + private - monthly_result = Logger::Period.previous_period_end(time, 'monthly') - month_ago = Time.parse("2018-06-30 23:59:59") - assert_equal(month_ago, monthly_result) + def assert_next_rotate_time_words(time, next_date, words) + assert_time_words(:next_rotate_time, time, next_date, words) + end + + def assert_previous_period_end_words(time, previous_date, words) + assert_time_words(:previous_period_end, time, previous_date, words) + end - assert_raise(ArgumentError) { Logger::Period.next_rotate_time(time, 'invalid') } + def assert_time_words(method, time, date, words) + words.each do |word| + daily_result = Logger::Period.public_send(method, time, word) + expected_result = Time.parse(date) + assert_equal(expected_result, daily_result) + end end end diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index b798b897b4..17f73e4433 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -416,7 +416,7 @@ class TestObjSpace < Test::Unit::TestCase assert_equal('true', ObjectSpace.dump(true)) assert_equal('false', ObjectSpace.dump(false)) assert_equal('0', ObjectSpace.dump(0)) - assert_equal('{"type":"SYMBOL", "value":"foo"}', ObjectSpace.dump(:foo)) + assert_equal('{"type":"SYMBOL", "value":"test_dump_special_consts"}', ObjectSpace.dump(:test_dump_special_consts)) end def test_dump_singleton_class diff --git a/test/openssl/test_pkcs12.rb b/test/openssl/test_pkcs12.rb index e6b91b52af..faf26c9e3e 100644 --- a/test/openssl/test_pkcs12.rb +++ b/test/openssl/test_pkcs12.rb @@ -159,7 +159,6 @@ module OpenSSL DEFAULT_PBE_PKEYS, DEFAULT_PBE_CERTS, nil, - nil, 2048 ) @@ -178,6 +177,36 @@ module OpenSSL end end + def test_create_with_keytype + OpenSSL::PKCS12.create( + "omg", + "hello", + @mykey, + @mycert, + [], + DEFAULT_PBE_PKEYS, + DEFAULT_PBE_CERTS, + nil, + nil, + OpenSSL::PKCS12::KEY_SIG + ) + + assert_raise(ArgumentError) do + OpenSSL::PKCS12.create( + "omg", + "hello", + @mykey, + @mycert, + [], + DEFAULT_PBE_PKEYS, + DEFAULT_PBE_CERTS, + nil, + nil, + 2048 + ) + end + end + def test_new_with_no_keys # generated with: # openssl pkcs12 -certpbe PBE-SHA1-3DES -in <@mycert> -nokeys -export diff --git a/test/openssl/test_pkcs7.rb b/test/openssl/test_pkcs7.rb index 96f3f1f6be..c049ed444a 100644 --- a/test/openssl/test_pkcs7.rb +++ b/test/openssl/test_pkcs7.rb @@ -227,6 +227,12 @@ END assert_equal(p7.to_der, OpenSSL::PKCS7.read_smime(smime).to_der) end + def test_to_text + p7 = OpenSSL::PKCS7.new + p7.type = "signed" + assert_match(/signed/, p7.to_text) + end + def test_degenerate_pkcs7 ca_cert_pem = <<END -----BEGIN CERTIFICATE----- diff --git a/test/openssl/test_ts.rb b/test/openssl/test_ts.rb index 7cb1a1fe8e..ac0469ad56 100644 --- a/test/openssl/test_ts.rb +++ b/test/openssl/test_ts.rb @@ -323,6 +323,8 @@ _end_of_pem_ resp = fac.create_timestamp(ee_key, ts_cert_ee, req) assert_equal(OpenSSL::Timestamp::Response::GRANTED, resp.status) assert_equal("1.2.3.4.6", resp.token_info.policy_id) + + assert_match(/1\.2\.3\.4\.6/, resp.to_text) end def test_response_bad_purpose diff --git a/test/openssl/test_x509cert.rb b/test/openssl/test_x509cert.rb index 64805504de..12e9dd025c 100644 --- a/test/openssl/test_x509cert.rb +++ b/test/openssl/test_x509cert.rb @@ -322,6 +322,15 @@ class OpenSSL::TestX509Certificate < OpenSSL::TestCase end end + def test_tbs_precert_bytes + pend "LibreSSL < 3.5 does not have i2d_re_X509_tbs" if libressl? && !libressl?(3, 5, 0) + + cert = issue_cert(@ca, @rsa2048, 1, [], nil, nil) + seq = OpenSSL::ASN1.decode(cert.tbs_bytes) + + assert_equal 7, seq.value.size + end + private def certificate_error_returns_false diff --git a/test/prism/command_line_test.rb b/test/prism/api/command_line_test.rb index 4b04c36f3a..a313845ead 100644 --- a/test/prism/command_line_test.rb +++ b/test/prism/api/command_line_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class CommandLineTest < TestCase @@ -67,7 +67,7 @@ module Prism end def test_command_line_x_implicit - result = Prism.parse(<<~RUBY) + result = Prism.parse_statement(<<~RUBY) #!/bin/bash exit 1 @@ -75,18 +75,18 @@ module Prism 1 RUBY - assert_kind_of IntegerNode, result.value.statements.body.first + assert_kind_of IntegerNode, result end def test_command_line_x_explicit - result = Prism.parse(<<~RUBY, command_line: "x") + result = Prism.parse_statement(<<~RUBY, command_line: "x") exit 1 #!/usr/bin/env ruby 1 RUBY - assert_kind_of IntegerNode, result.value.statements.body.first + assert_kind_of IntegerNode, result end def test_command_line_x_implicit_fail diff --git a/test/prism/api/dump_test.rb b/test/prism/api/dump_test.rb new file mode 100644 index 0000000000..941088e159 --- /dev/null +++ b/test/prism/api/dump_test.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +return if ENV["PRISM_BUILD_MINIMAL"] + +require_relative "../test_helper" + +module Prism + class DumpTest < TestCase + Fixture.each do |fixture| + define_method(fixture.test_name) { assert_dump(fixture) } + end + + def test_dump + filepath = __FILE__ + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + + assert_equal Prism.lex(source, filepath: filepath).value, Prism.lex_file(filepath).value + assert_equal Prism.dump(source, filepath: filepath), Prism.dump_file(filepath) + + serialized = Prism.dump(source, filepath: filepath) + ast1 = Prism.load(source, serialized).value + ast2 = Prism.parse(source, filepath: filepath).value + ast3 = Prism.parse_file(filepath).value + + assert_equal_nodes ast1, ast2 + assert_equal_nodes ast2, ast3 + end + + def test_dump_file + assert_nothing_raised do + Prism.dump_file(__FILE__) + end + + error = assert_raise Errno::ENOENT do + Prism.dump_file("idontexist.rb") + end + + assert_equal "No such file or directory - idontexist.rb", error.message + + assert_raise TypeError do + Prism.dump_file(nil) + end + end + + private + + def assert_dump(fixture) + source = fixture.read + + result = Prism.parse(source, filepath: fixture.path) + dumped = Prism.dump(source, filepath: fixture.path) + + assert_equal_nodes(result.value, Prism.load(source, dumped).value) + end + end +end diff --git a/test/prism/parse_comments_test.rb b/test/prism/api/parse_comments_test.rb index 30086e3155..4dbcca1827 100644 --- a/test/prism/parse_comments_test.rb +++ b/test/prism/api/parse_comments_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class ParseCommentsTest < TestCase @@ -17,5 +17,17 @@ module Prism assert_kind_of Array, comments assert_equal 1, comments.length end + + def test_parse_file_comments_error + error = assert_raise Errno::ENOENT do + Prism.parse_file_comments("idontexist.rb") + end + + assert_equal "No such file or directory - idontexist.rb", error.message + + assert_raise TypeError do + Prism.parse_file_comments(nil) + end + end end end diff --git a/test/prism/parse_stream_test.rb b/test/prism/api/parse_stream_test.rb index 9e6347b92b..0edee74cc2 100644 --- a/test/prism/parse_stream_test.rb +++ b/test/prism/api/parse_stream_test.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" -require "stringio" +require_relative "../test_helper" module Prism class ParseStreamTest < TestCase @@ -10,7 +9,7 @@ module Prism result = Prism.parse_stream(io) assert result.success? - assert_kind_of Prism::CallNode, result.value.statements.body.first + assert_kind_of Prism::CallNode, result.statement end def test_multi_line @@ -18,8 +17,8 @@ module Prism result = Prism.parse_stream(io) assert result.success? - assert_kind_of Prism::CallNode, result.value.statements.body.first - assert_kind_of Prism::CallNode, result.value.statements.body.last + assert_kind_of Prism::CallNode, result.statement + assert_kind_of Prism::CallNode, result.statement end def test_multi_read @@ -27,7 +26,7 @@ module Prism result = Prism.parse_stream(io) assert result.success? - assert_kind_of Prism::CallNode, result.value.statements.body.first + assert_kind_of Prism::CallNode, result.statement end def test___END__ diff --git a/test/prism/api/parse_success_test.rb b/test/prism/api/parse_success_test.rb new file mode 100644 index 0000000000..2caaa5136e --- /dev/null +++ b/test/prism/api/parse_success_test.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class ParseSuccessTest < TestCase + def test_parse_success? + assert Prism.parse_success?("1") + refute Prism.parse_success?("<>") + end + + def test_parse_file_success? + assert Prism.parse_file_success?(__FILE__) + end + end +end diff --git a/test/prism/api/parse_test.rb b/test/prism/api/parse_test.rb new file mode 100644 index 0000000000..864d38461a --- /dev/null +++ b/test/prism/api/parse_test.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class ParseTest < TestCase + def test_parse_empty_string + result = Prism.parse("") + assert_equal [], result.value.statements.body + end + + def test_parse_takes_file_path + filepath = "filepath.rb" + result = Prism.parse("def foo; __FILE__; end", filepath: filepath) + + assert_equal filepath, find_source_file_node(result.value).filepath + end + + def test_parse_takes_line + line = 4 + result = Prism.parse("def foo\n __FILE__\nend", line: line) + + assert_equal line, result.value.location.start_line + assert_equal line + 1, find_source_file_node(result.value).location.start_line + + result = Prism.parse_lex("def foo\n __FILE__\nend", line: line) + assert_equal line, result.value.first.location.start_line + end + + def test_parse_takes_negative_lines + line = -2 + result = Prism.parse("def foo\n __FILE__\nend", line: line) + + assert_equal line, result.value.location.start_line + assert_equal line + 1, find_source_file_node(result.value).location.start_line + + result = Prism.parse_lex("def foo\n __FILE__\nend", line: line) + assert_equal line, result.value.first.location.start_line + end + + def test_parse_file + node = Prism.parse_file(__FILE__).value + assert_kind_of ProgramNode, node + + error = assert_raise Errno::ENOENT do + Prism.parse_file("idontexist.rb") + end + + assert_equal "No such file or directory - idontexist.rb", error.message + + assert_raise TypeError do + Prism.parse_file(nil) + end + end + + private + + def find_source_file_node(program) + queue = [program] + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.compact_child_nodes) + end + end + end +end diff --git a/test/prism/bom_test.rb b/test/prism/bom_test.rb index 1525caf458..890bc4b36c 100644 --- a/test/prism/bom_test.rb +++ b/test/prism/bom_test.rb @@ -2,7 +2,7 @@ # Don't bother checking this on these engines, this is such a specific Ripper # test. -return if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby" +return if RUBY_ENGINE != "ruby" require_relative "test_helper" diff --git a/test/prism/encoding/encodings_test.rb b/test/prism/encoding/encodings_test.rb new file mode 100644 index 0000000000..4ad2b465cc --- /dev/null +++ b/test/prism/encoding/encodings_test.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +return if RUBY_ENGINE != "ruby" + +require_relative "../test_helper" + +module Prism + class EncodingsTest < TestCase + class ConstantContext < BasicObject + def self.const_missing(const) + const + end + end + + class IdentifierContext < BasicObject + def method_missing(name, *) + name + end + end + + # These test that we're correctly parsing codepoints for each alias of each + # encoding that prism supports. + each_encoding do |encoding, range| + (encoding.names - %w[external internal filesystem locale]).each do |name| + define_method(:"test_encoding_#{name}") do + assert_encoding(encoding, name, range) + end + end + end + + private + + def assert_encoding_constant(name, character) + source = "# encoding: #{name}\n#{character}" + expected = ConstantContext.new.instance_eval(source) + + result = Prism.parse(source) + assert result.success? + + actual = result.value.statements.body.last + assert_kind_of ConstantReadNode, actual + assert_equal expected, actual.name + end + + def assert_encoding_identifier(name, character) + source = "# encoding: #{name}\n#{character}" + expected = IdentifierContext.new.instance_eval(source) + + result = Prism.parse(source) + assert result.success? + + actual = result.value.statements.body.last + assert_kind_of CallNode, actual + assert_equal expected, actual.name + end + + # Check that we can properly parse every codepoint in the given encoding. + def assert_encoding(encoding, name, range) + # I'm not entirely sure, but I believe these codepoints are incorrect in + # their parsing in CRuby. They all report as matching `[[:lower:]]` but + # then they are parsed as constants. This is because CRuby determines if + # an identifier is a constant or not by case folding it down to lowercase + # and checking if there is a difference. And even though they report + # themselves as lowercase, their case fold is different. I have reported + # this bug upstream. + case encoding + when Encoding::UTF_8, Encoding::UTF_8_MAC, Encoding::UTF8_DoCoMo, Encoding::UTF8_KDDI, Encoding::UTF8_SoftBank, Encoding::CESU_8 + range = range.to_a - [ + 0x01c5, 0x01c8, 0x01cb, 0x01f2, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, + 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, + 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, + 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fbc, 0x1fcc, 0x1ffc, + ] + when Encoding::Windows_1253 + range = range.to_a - [0xb5] + end + + range.each do |codepoint| + character = codepoint.chr(encoding) + + if character.match?(/[[:alpha:]]/) + if character.match?(/[[:upper:]]/) + assert_encoding_constant(name, character) + else + assert_encoding_identifier(name, character) + end + elsif character.match?(/[[:alnum:]]/) + assert_encoding_identifier(name, "_#{character}") + else + next if ["/", "{"].include?(character) + + source = "# encoding: #{name}\n/(?##{character})/\n" + assert Prism.parse_success?(source), "Expected #{source.inspect} to parse successfully." + end + rescue RangeError + source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}" + assert Prism.parse_failure?(source) + end + end + end +end diff --git a/test/prism/encoding/regular_expression_encoding_test.rb b/test/prism/encoding/regular_expression_encoding_test.rb new file mode 100644 index 0000000000..5d062fe59a --- /dev/null +++ b/test/prism/encoding/regular_expression_encoding_test.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +return unless defined?(RubyVM::InstructionSequence) +return if RubyVM::InstructionSequence.compile("").to_a[4][:parser] == :prism + +require_relative "../test_helper" + +module Prism + class RegularExpressionEncodingTest < TestCase + each_encoding do |encoding, _| + define_method(:"test_regular_expression_encoding_flags_#{encoding.name}") do + assert_regular_expression_encoding_flags(encoding, ["/a/", "/ą/", "//"]) + end + + escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"] + escapes = escapes.concat(escapes.product(escapes).map(&:join)) + + define_method(:"test_regular_expression_escape_encoding_flags_#{encoding.name}") do + assert_regular_expression_encoding_flags(encoding, escapes.map { |e| "/#{e}/" }) + end + + ["n", "u", "e", "s"].each do |modifier| + define_method(:"test_regular_expression_encoding_modifiers_/#{modifier}_#{encoding.name}") do + regexp_sources = ["abc", "garçon", "\\x80", "gar\\xC3\\xA7on", "gar\\u{E7}on", "abc\\u{FFFFFF}", "\\x80\\u{80}" ] + + assert_regular_expression_encoding_flags( + encoding, + regexp_sources.product(["n", "u", "e", "s"]).map { |r, modifier| "/#{r}/#{modifier}" } + ) + end + end + end + + private + + def assert_regular_expression_encoding_flags(encoding, regexps) + regexps.each do |regexp| + regexp_modifier_used = regexp.end_with?("/u") || regexp.end_with?("/e") || regexp.end_with?("/s") || regexp.end_with?("/n") + source = "# encoding: #{encoding.name}\n#{regexp}" + + encoding_errors = ["invalid multibyte char", "escaped non ASCII character in UTF-8 regexp", "differs from source encoding"] + skipped_errors = ["invalid multibyte escape", "incompatible character encoding", "UTF-8 character in non UTF-8 regexp", "invalid Unicode range", "invalid Unicode list"] + + # TODO (nirvdrum 21-Feb-2024): Prism currently does not handle Regexp validation unless modifiers are used. So, skip processing those errors for now: https://github.com/ruby/prism/issues/2104 + unless regexp_modifier_used + skipped_errors += encoding_errors + encoding_errors.clear + end + + expected = + begin + eval(source).encoding + rescue SyntaxError => error + if encoding_errors.find { |e| error.message.include?(e) } + error.message.split("\n").map { |m| m[/: (.+?)$/, 1] } + elsif skipped_errors.find { |e| error.message.include?(e) } + next + else + raise + end + end + + actual = + Prism.parse(source).then do |result| + if result.success? + regexp = result.statement + + actual_encoding = if regexp.forced_utf8_encoding? + Encoding::UTF_8 + elsif regexp.forced_binary_encoding? + Encoding::ASCII_8BIT + elsif regexp.forced_us_ascii_encoding? + Encoding::US_ASCII + elsif regexp.ascii_8bit? + Encoding::ASCII_8BIT + elsif regexp.utf_8? + Encoding::UTF_8 + elsif regexp.euc_jp? + Encoding::EUC_JP + elsif regexp.windows_31j? + Encoding::Windows_31J + else + encoding + end + + if regexp.utf_8? && actual_encoding != Encoding::UTF_8 + raise "expected regexp encoding to be UTF-8 due to '/u' modifier, but got #{actual_encoding.name}" + elsif regexp.ascii_8bit? && (actual_encoding != Encoding::ASCII_8BIT && actual_encoding != Encoding::US_ASCII) + raise "expected regexp encoding to be ASCII-8BIT or US-ASCII due to '/n' modifier, but got #{actual_encoding.name}" + elsif regexp.euc_jp? && actual_encoding != Encoding::EUC_JP + raise "expected regexp encoding to be EUC-JP due to '/e' modifier, but got #{actual_encoding.name}" + elsif regexp.windows_31j? && actual_encoding != Encoding::Windows_31J + raise "expected regexp encoding to be Windows-31J due to '/s' modifier, but got #{actual_encoding.name}" + end + + if regexp.utf_8? && regexp.forced_utf8_encoding? + raise "the forced_utf8 flag should not be set when the UTF-8 modifier (/u) is used" + elsif regexp.ascii_8bit? && regexp.forced_binary_encoding? + raise "the forced_ascii_8bit flag should not be set when the UTF-8 modifier (/u) is used" + end + + actual_encoding + else + errors = result.errors.map(&:message) + + if errors.last&.include?("UTF-8 mixed within") + nil + else + errors + end + end + end + + # TODO (nirvdrum 22-Feb-2024): Remove this workaround once Prism better maps CRuby's error messages. + # This class of error message is tricky. The part not being compared is a representation of the regexp. + # Depending on the source encoding and any encoding modifiers being used, CRuby alters how the regexp is represented. + # Sometimes it's an MBC string. Other times it uses hexadecimal character escapes. And in other cases it uses + # the long-form Unicode escape sequences. This short-circuit checks that the error message is mostly correct. + if expected.is_a?(Array) && actual.is_a?(Array) + if expected.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") && + actual.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") + expected.last.clear + actual.last.clear + end + end + + assert_equal expected, actual + end + end + end +end diff --git a/test/prism/encoding/string_encoding_test.rb b/test/prism/encoding/string_encoding_test.rb new file mode 100644 index 0000000000..6f9d86df3b --- /dev/null +++ b/test/prism/encoding/string_encoding_test.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class StringEncodingTest < TestCase + each_encoding do |encoding, _| + define_method(:"test_#{encoding.name}") do + assert_encoding(encoding) + end + end + + def test_coding + actual = Prism.parse_statement("# coding: utf-8\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_coding_with_whitespace + actual = Prism.parse_statement("# coding \t \r \v : \t \v \r ascii-8bit \n'string'").unescaped.encoding + assert_equal Encoding::ASCII_8BIT, actual + end + + def test_emacs_style + actual = Prism.parse_statement("# -*- coding: utf-8 -*-\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_utf_8_unix + actual = Prism.parse_statement("# coding: utf-8-unix\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_utf_8_dos + actual = Prism.parse_statement("# coding: utf-8-dos\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_utf_8_mac + actual = Prism.parse_statement("# coding: utf-8-mac\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_utf_8_star + actual = Prism.parse_statement("# coding: utf-8-*\n'string'").unescaped.encoding + assert_equal Encoding::UTF_8, actual + end + + def test_first_lexed_token + encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding + assert_equal Encoding::ASCII_8BIT, encoding + end + + if !ENV["PRISM_BUILD_MINIMAL"] + # This test may be a little confusing. Basically when we use our strpbrk, + # it takes into account the encoding of the file. + def test_strpbrk_multibyte + result = Prism.parse(<<~RUBY) + # encoding: Shift_JIS + %w[\x81\x5c] + RUBY + + assert(result.errors.empty?) + assert_equal( + (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), + result.statement.elements.first.unescaped + ) + end + + def test_slice_encoding + slice = Prism.parse("# encoding: Shift_JIS\nア").value.slice + assert_equal (+"ア").force_encoding(Encoding::SHIFT_JIS), slice + assert_equal Encoding::SHIFT_JIS, slice.encoding + end + + def test_multibyte_escapes + [ + ["'", "'"], + ["\"", "\""], + ["`", "`"], + ["/", "/"], + ["<<'HERE'\n", "\nHERE"], + ["<<-HERE\n", "\nHERE"] + ].each do |opening, closing| + assert Prism.parse_success?("# encoding: shift_jis\n'\\\x82\xA0'\n") + end + end + end + + private + + def assert_encoding(encoding) + escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"] + escapes = escapes.concat(escapes.product(escapes).map(&:join)) + + escapes.each do |escaped| + source = "# encoding: #{encoding.name}\n\"#{escaped}\"" + + expected = + begin + eval(source).encoding + rescue SyntaxError => error + if error.message.include?("UTF-8 mixed within") + error.message[/UTF-8 mixed within .+? source/] + else + raise + end + end + + actual = + Prism.parse(source).then do |result| + if result.success? + string = result.statement + + if string.forced_utf8_encoding? + Encoding::UTF_8 + elsif string.forced_binary_encoding? + Encoding::ASCII_8BIT + else + encoding + end + else + error = result.errors.first + + if error.message.include?("mixed") + error.message + else + raise error.message + end + end + end + + assert_equal expected, actual + end + end + end +end diff --git a/test/prism/encoding/symbol_encoding_test.rb b/test/prism/encoding/symbol_encoding_test.rb new file mode 100644 index 0000000000..20c998a58b --- /dev/null +++ b/test/prism/encoding/symbol_encoding_test.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +return if RUBY_ENGINE != "ruby" + +require_relative "../test_helper" + +module Prism + class SymbolEncodingTest < TestCase + each_encoding do |encoding, _| + define_method(:"test_symbols_#{encoding.name}") do + assert_symbols(encoding) + end + + define_method(:"test_escapes_#{encoding.name}") do + assert_escapes(encoding) + end + end + + private + + def expected_encoding(source) + eval(source).encoding + end + + def actual_encoding(source, encoding) + result = Prism.parse(source) + + if result.success? + symbol = result.statement + + if symbol.forced_utf8_encoding? + Encoding::UTF_8 + elsif symbol.forced_binary_encoding? + Encoding::ASCII_8BIT + elsif symbol.forced_us_ascii_encoding? + Encoding::US_ASCII + else + encoding + end + else + raise SyntaxError.new(result.errors.map(&:message).join("\n")) + end + end + + def assert_symbols(encoding) + [:a, :ą, :+].each do |symbol| + source = "# encoding: #{encoding.name}\n#{symbol.inspect}" + + expected = + begin + expected_encoding(source) + rescue SyntaxError => error + if error.message.include?("invalid multibyte") + "invalid multibyte" + else + raise + end + end + + actual = + begin + actual_encoding(source, encoding) + rescue SyntaxError => error + if error.message.include?("invalid multibyte") + "invalid multibyte" + else + raise + end + end + + assert_equal expected, actual + end + end + + def assert_escapes(encoding) + escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"] + escapes = escapes.concat(escapes.product(escapes).map(&:join)) + + escapes.each do |escaped| + source = "# encoding: #{encoding.name}\n:\"#{escaped}\"" + + expected = + begin + expected_encoding(source) + rescue SyntaxError => error + if error.message.include?("UTF-8 mixed within") + error.message[/UTF-8 mixed within .+? source/] + else + raise + end + end + + actual = + begin + actual_encoding(source, encoding) + rescue SyntaxError => error + if error.message.include?("mixed") + error.message.split("\n", 2).first + else + raise + end + end + + assert_equal expected, actual + end + end + end +end diff --git a/test/prism/encoding_test.rb b/test/prism/encoding_test.rb deleted file mode 100644 index 2aee473ddf..0000000000 --- a/test/prism/encoding_test.rb +++ /dev/null @@ -1,577 +0,0 @@ -# frozen_string_literal: true - -return if RUBY_ENGINE != "ruby" - -require_relative "test_helper" - -module Prism - class EncodingTest < TestCase - codepoints_1byte = 0...0x100 - encodings = { - Encoding::ASCII_8BIT => codepoints_1byte, - Encoding::US_ASCII => codepoints_1byte - } - - if !ENV["PRISM_BUILD_MINIMAL"] - encodings[Encoding::Windows_1253] = codepoints_1byte - end - - # By default we don't test every codepoint in these encodings because it - # takes a very long time. - if ENV["PRISM_TEST_ALL_ENCODINGS"] - codepoints_2bytes = 0...0x10000 - codepoints_unicode = (0...0x110000) - - codepoints_eucjp = [ - *(0...0x10000), - *(0...0x10000).map { |bytes| bytes | 0x8F0000 } - ] - - codepoints_emacs_mule = [ - *(0...0x80), - *((0x81...0x90).flat_map { |byte1| (0x90...0x100).map { |byte2| byte1 << 8 | byte2 } }), - *((0x90...0x9C).flat_map { |byte1| (0xA0...0x100).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| byte1 << 16 | byte2 << 8 | byte3 } } }), - *((0xF0...0xF5).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| (0xA0...0x100).flat_map { |byte4| 0x9C << 24 | byte3 << 16 | byte3 << 8 | byte4 } } }), - ] - - codepoints_gb18030 = [ - *(0...0x80), - *((0x81..0xFE).flat_map { |byte1| (0x40...0x100).map { |byte2| byte1 << 8 | byte2 } }), - *((0x81..0xFE).flat_map { |byte1| (0x30...0x40).flat_map { |byte2| (0x81..0xFE).flat_map { |byte3| (0x2F...0x41).map { |byte4| byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } }), - ] - - codepoints_euc_tw = [ - *(0..0x7F), - *(0xA1..0xFF).flat_map { |byte1| (0xA1..0xFF).map { |byte2| (byte1 << 8) | byte2 } }, - *(0xA1..0xB0).flat_map { |byte2| (0xA1..0xFF).flat_map { |byte3| (0xA1..0xFF).flat_map { |byte4| 0x8E << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } - ] - - encodings.merge!( - Encoding::CP850 => codepoints_1byte, - Encoding::CP852 => codepoints_1byte, - Encoding::CP855 => codepoints_1byte, - Encoding::GB1988 => codepoints_1byte, - Encoding::IBM437 => codepoints_1byte, - Encoding::IBM720 => codepoints_1byte, - Encoding::IBM737 => codepoints_1byte, - Encoding::IBM775 => codepoints_1byte, - Encoding::IBM852 => codepoints_1byte, - Encoding::IBM855 => codepoints_1byte, - Encoding::IBM857 => codepoints_1byte, - Encoding::IBM860 => codepoints_1byte, - Encoding::IBM861 => codepoints_1byte, - Encoding::IBM862 => codepoints_1byte, - Encoding::IBM863 => codepoints_1byte, - Encoding::IBM864 => codepoints_1byte, - Encoding::IBM865 => codepoints_1byte, - Encoding::IBM866 => codepoints_1byte, - Encoding::IBM869 => codepoints_1byte, - Encoding::ISO_8859_1 => codepoints_1byte, - Encoding::ISO_8859_2 => codepoints_1byte, - Encoding::ISO_8859_3 => codepoints_1byte, - Encoding::ISO_8859_4 => codepoints_1byte, - Encoding::ISO_8859_5 => codepoints_1byte, - Encoding::ISO_8859_6 => codepoints_1byte, - Encoding::ISO_8859_7 => codepoints_1byte, - Encoding::ISO_8859_8 => codepoints_1byte, - Encoding::ISO_8859_9 => codepoints_1byte, - Encoding::ISO_8859_10 => codepoints_1byte, - Encoding::ISO_8859_11 => codepoints_1byte, - Encoding::ISO_8859_13 => codepoints_1byte, - Encoding::ISO_8859_14 => codepoints_1byte, - Encoding::ISO_8859_15 => codepoints_1byte, - Encoding::ISO_8859_16 => codepoints_1byte, - Encoding::KOI8_R => codepoints_1byte, - Encoding::KOI8_U => codepoints_1byte, - Encoding::MACCENTEURO => codepoints_1byte, - Encoding::MACCROATIAN => codepoints_1byte, - Encoding::MACCYRILLIC => codepoints_1byte, - Encoding::MACGREEK => codepoints_1byte, - Encoding::MACICELAND => codepoints_1byte, - Encoding::MACROMAN => codepoints_1byte, - Encoding::MACROMANIA => codepoints_1byte, - Encoding::MACTHAI => codepoints_1byte, - Encoding::MACTURKISH => codepoints_1byte, - Encoding::MACUKRAINE => codepoints_1byte, - Encoding::TIS_620 => codepoints_1byte, - Encoding::Windows_1250 => codepoints_1byte, - Encoding::Windows_1251 => codepoints_1byte, - Encoding::Windows_1252 => codepoints_1byte, - Encoding::Windows_1254 => codepoints_1byte, - Encoding::Windows_1255 => codepoints_1byte, - Encoding::Windows_1256 => codepoints_1byte, - Encoding::Windows_1257 => codepoints_1byte, - Encoding::Windows_1258 => codepoints_1byte, - Encoding::Windows_874 => codepoints_1byte, - Encoding::Big5 => codepoints_2bytes, - Encoding::Big5_HKSCS => codepoints_2bytes, - Encoding::Big5_UAO => codepoints_2bytes, - Encoding::CP949 => codepoints_2bytes, - Encoding::CP950 => codepoints_2bytes, - Encoding::CP951 => codepoints_2bytes, - Encoding::EUC_KR => codepoints_2bytes, - Encoding::GBK => codepoints_2bytes, - Encoding::GB12345 => codepoints_2bytes, - Encoding::GB2312 => codepoints_2bytes, - Encoding::MACJAPANESE => codepoints_2bytes, - Encoding::Shift_JIS => codepoints_2bytes, - Encoding::SJIS_DoCoMo => codepoints_2bytes, - Encoding::SJIS_KDDI => codepoints_2bytes, - Encoding::SJIS_SoftBank => codepoints_2bytes, - Encoding::Windows_31J => codepoints_2bytes, - Encoding::UTF_8 => codepoints_unicode, - Encoding::UTF8_MAC => codepoints_unicode, - Encoding::UTF8_DoCoMo => codepoints_unicode, - Encoding::UTF8_KDDI => codepoints_unicode, - Encoding::UTF8_SoftBank => codepoints_unicode, - Encoding::CESU_8 => codepoints_unicode, - Encoding::CP51932 => codepoints_eucjp, - Encoding::EUC_JP => codepoints_eucjp, - Encoding::EUCJP_MS => codepoints_eucjp, - Encoding::EUC_JIS_2004 => codepoints_eucjp, - Encoding::EMACS_MULE => codepoints_emacs_mule, - Encoding::STATELESS_ISO_2022_JP => codepoints_emacs_mule, - Encoding::STATELESS_ISO_2022_JP_KDDI => codepoints_emacs_mule, - Encoding::GB18030 => codepoints_gb18030, - Encoding::EUC_TW => codepoints_euc_tw - ) - end - - # These test that we're correctly parsing codepoints for each alias of each - # encoding that prism supports. - encodings.each do |encoding, range| - (encoding.names - %w[external internal filesystem locale]).each do |name| - define_method(:"test_encoding_#{name}") do - assert_encoding(encoding, name, range) - end - end - end - - # These test that we're correctly setting the flags on strings for each - # encoding that prism supports. - escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"] - escapes = escapes.concat(escapes.product(escapes).map(&:join)) - symbols = [:a, :ą, :+] - regexps = [/a/, /ą/, //] - - encodings.each_key do |encoding| - define_method(:"test_encoding_flags_#{encoding.name}") do - assert_encoding_flags(encoding, escapes) - end - - define_method(:"test_symbol_encoding_flags_#{encoding.name}") do - assert_symbol_encoding_flags(encoding, symbols) - end - - define_method(:"test_symbol_character_escape_encoding_flags_#{encoding.name}") do - assert_symbol_character_escape_encoding_flags(encoding, escapes) - end - - define_method(:"test_regular_expression_encoding_flags_#{encoding.name}") do - assert_regular_expression_encoding_flags(encoding, regexps.map(&:inspect)) - end - - define_method(:"test_regular_expression_escape_encoding_flags_#{encoding.name}") do - assert_regular_expression_encoding_flags(encoding, escapes.map { |e| "/#{e}/" }) - end - end - - encoding_modifiers = { ascii_8bit: "n", utf_8: "u", euc_jp: "e", windows_31j: "s" } - regexp_sources = ["abc", "garçon", "\\x80", "gar\\xC3\\xA7on", "gar\\u{E7}on", "abc\\u{FFFFFF}", "\\x80\\u{80}" ] - - encoding_modifiers.each_value do |modifier| - encodings.each_key do |encoding| - define_method(:"test_regular_expression_encoding_modifiers_/#{modifier}_#{encoding.name}") do - assert_regular_expression_encoding_flags( - encoding, - regexp_sources.product(encoding_modifiers.values).map { |r, modifier| "/#{r}/#{modifier}" } - ) - end - end - end - - def test_coding - result = Prism.parse("# coding: utf-8\n'string'") - actual = result.value.statements.body.first.unescaped.encoding - assert_equal Encoding.find("utf-8"), actual - end - - def test_coding_with_whitespace - result = Prism.parse("# coding \t \r \v : \t \v \r ascii-8bit \n'string'") - actual = result.value.statements.body.first.unescaped.encoding - assert_equal Encoding.find("ascii-8bit"), actual - end - - def test_emacs_style - result = Prism.parse("# -*- coding: utf-8 -*-\n'string'") - actual = result.value.statements.body.first.unescaped.encoding - assert_equal Encoding.find("utf-8"), actual - end - - def test_utf_8_variations - %w[ - utf-8-unix - utf-8-dos - utf-8-mac - utf-8-* - ].each do |encoding| - result = Prism.parse("# coding: #{encoding}\n'string'") - actual = result.value.statements.body.first.unescaped.encoding - assert_equal Encoding.find("utf-8"), actual - end - end - - def test_first_lexed_token - encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding - assert_equal Encoding.find("ascii-8bit"), encoding - end - - if !ENV["PRISM_BUILD_MINIMAL"] - # This test may be a little confusing. Basically when we use our strpbrk, - # it takes into account the encoding of the file. - def test_strpbrk_multibyte - result = Prism.parse(<<~RUBY) - # encoding: Shift_JIS - %w[\x81\x5c] - RUBY - - assert(result.errors.empty?) - assert_equal( - (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), - result.value.statements.body.first.elements.first.unescaped - ) - end - - def test_slice_encoding - slice = Prism.parse("# encoding: Shift_JIS\nア").value.slice - assert_equal (+"ア").force_encoding(Encoding::SHIFT_JIS), slice - assert_equal Encoding::SHIFT_JIS, slice.encoding - end - - def test_multibyte_escapes - [ - ["'", "'"], - ["\"", "\""], - ["`", "`"], - ["/", "/"], - ["<<'HERE'\n", "\nHERE"], - ["<<-HERE\n", "\nHERE"] - ].each do |opening, closing| - assert Prism.parse_success?("# encoding: shift_jis\n'\\\x82\xA0'\n") - end - end - end - - private - - class ConstantContext < BasicObject - def self.const_missing(const) - const - end - end - - def constant_context - ConstantContext.new - end - - class IdentifierContext < BasicObject - def method_missing(name, *) - name - end - end - - def identifier_context - IdentifierContext.new - end - - def assert_encoding_constant(name, character) - source = "# encoding: #{name}\n#{character}" - expected = constant_context.instance_eval(source) - - result = Prism.parse(source) - assert result.success? - - actual = result.value.statements.body.last - assert_kind_of ConstantReadNode, actual - assert_equal expected, actual.name - end - - def assert_encoding_identifier(name, character) - source = "# encoding: #{name}\n#{character}" - expected = identifier_context.instance_eval(source) - - result = Prism.parse(source) - assert result.success? - - actual = result.value.statements.body.last - assert_kind_of CallNode, actual - assert_equal expected, actual.name - end - - # Check that we can properly parse every codepoint in the given encoding. - def assert_encoding(encoding, name, range) - # I'm not entirely sure, but I believe these codepoints are incorrect in - # their parsing in CRuby. They all report as matching `[[:lower:]]` but - # then they are parsed as constants. This is because CRuby determines if - # an identifier is a constant or not by case folding it down to lowercase - # and checking if there is a difference. And even though they report - # themselves as lowercase, their case fold is different. I have reported - # this bug upstream. - case encoding - when Encoding::UTF_8, Encoding::UTF_8_MAC, Encoding::UTF8_DoCoMo, Encoding::UTF8_KDDI, Encoding::UTF8_SoftBank, Encoding::CESU_8 - range = range.to_a - [ - 0x01c5, 0x01c8, 0x01cb, 0x01f2, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, - 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, - 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, - 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fbc, 0x1fcc, 0x1ffc, - ] - when Encoding::Windows_1253 - range = range.to_a - [0xb5] - end - - range.each do |codepoint| - character = codepoint.chr(encoding) - - if character.match?(/[[:alpha:]]/) - if character.match?(/[[:upper:]]/) - assert_encoding_constant(name, character) - else - assert_encoding_identifier(name, character) - end - elsif character.match?(/[[:alnum:]]/) - assert_encoding_identifier(name, "_#{character}") - else - next if ["/", "{"].include?(character) - - source = "# encoding: #{name}\n/(?##{character})/\n" - assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully." - end - rescue RangeError - source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}" - refute Prism.parse(source).success? - end - end - - def assert_encoding_flags(encoding, escapes) - escapes.each do |escaped| - source = "# encoding: #{encoding.name}\n\"#{escaped}\"" - - expected = - begin - eval(source).encoding - rescue SyntaxError => error - if error.message.include?("UTF-8 mixed within") - error.message[/: (.+?)\n/, 1] - else - raise - end - end - - actual = - Prism.parse(source).then do |result| - if result.success? - string = result.value.statements.body.first - - if string.forced_utf8_encoding? - Encoding::UTF_8 - elsif string.forced_binary_encoding? - Encoding::ASCII_8BIT - else - encoding - end - else - error = result.errors.first - - if error.message.include?("mixed") - error.message - else - raise error.message - end - end - end - - assert_equal expected, actual - end - end - - # Test Symbol literals without any interpolation or escape sequences. - def assert_symbol_encoding_flags(encoding, symbols) - symbols.each do |symbol| - source = "# encoding: #{encoding.name}\n#{symbol.inspect}" - - expected = - begin - eval(source).encoding - rescue SyntaxError => error - unless error.message.include?("invalid multibyte char") - raise - end - end - - actual = - Prism.parse(source).then do |result| - if result.success? - symbol = result.value.statements.body.first - - if symbol.forced_utf8_encoding? - Encoding::UTF_8 - elsif symbol.forced_binary_encoding? - Encoding::ASCII_8BIT - elsif symbol.forced_us_ascii_encoding? - Encoding::US_ASCII - else - encoding - end - else - error = result.errors.last - - unless error.message.include?("invalid symbol") - raise error.message - end - end - end - - assert_equal expected, actual - end - end - - def assert_symbol_character_escape_encoding_flags(encoding, escapes) - escapes.each do |escaped| - source = "# encoding: #{encoding.name}\n:\"#{escaped}\"" - - expected = - begin - eval(source).encoding - rescue SyntaxError => error - if error.message.include?("UTF-8 mixed within") - error.message[/: (.+?)\n/, 1] - else - raise - end - end - - actual = - Prism.parse(source).then do |result| - if result.success? - symbol = result.value.statements.body.first - - if symbol.forced_utf8_encoding? - Encoding::UTF_8 - elsif symbol.forced_binary_encoding? - Encoding::ASCII_8BIT - elsif symbol.forced_us_ascii_encoding? - Encoding::US_ASCII - else - encoding - end - else - error = result.errors.first - - if error.message.include?("mixed") - error.message - else - raise error.message - end - end - end - - assert_equal expected, actual - end - end - - def assert_regular_expression_encoding_flags(encoding, regexps) - regexps.each do |regexp| - regexp_modifier_used = regexp.end_with?("/u") || regexp.end_with?("/e") || regexp.end_with?("/s") || regexp.end_with?("/n") - source = "# encoding: #{encoding.name}\n#{regexp}" - - encoding_errors = ["invalid multibyte char", "escaped non ASCII character in UTF-8 regexp", "differs from source encoding"] - skipped_errors = ["invalid multibyte escape", "incompatible character encoding", "UTF-8 character in non UTF-8 regexp", "invalid Unicode range", "invalid Unicode list"] - - # TODO (nirvdrum 21-Feb-2024): Prism currently does not handle Regexp validation unless modifiers are used. So, skip processing those errors for now: https://github.com/ruby/prism/issues/2104 - unless regexp_modifier_used - skipped_errors += encoding_errors - encoding_errors.clear - end - - expected = - begin - eval(source).encoding - rescue SyntaxError => error - if encoding_errors.find { |e| error.message.include?(e) } - error.message.split("\n").map { |m| m[/: (.+?)$/, 1] } - elsif skipped_errors.find { |e| error.message.include?(e) } - next - else - raise - end - end - - actual = - Prism.parse(source).then do |result| - if result.success? - regexp = result.value.statements.body.first - - actual_encoding = if regexp.forced_utf8_encoding? - Encoding::UTF_8 - elsif regexp.forced_binary_encoding? - Encoding::ASCII_8BIT - elsif regexp.forced_us_ascii_encoding? - Encoding::US_ASCII - elsif regexp.ascii_8bit? - Encoding::ASCII_8BIT - elsif regexp.utf_8? - Encoding::UTF_8 - elsif regexp.euc_jp? - Encoding::EUC_JP - elsif regexp.windows_31j? - Encoding::Windows_31J - else - encoding - end - - if regexp.utf_8? && actual_encoding != Encoding::UTF_8 - raise "expected regexp encoding to be UTF-8 due to '/u' modifier, but got #{actual_encoding.name}" - elsif regexp.ascii_8bit? && (actual_encoding != Encoding::ASCII_8BIT && actual_encoding != Encoding::US_ASCII) - raise "expected regexp encoding to be ASCII-8BIT or US-ASCII due to '/n' modifier, but got #{actual_encoding.name}" - elsif regexp.euc_jp? && actual_encoding != Encoding::EUC_JP - raise "expected regexp encoding to be EUC-JP due to '/e' modifier, but got #{actual_encoding.name}" - elsif regexp.windows_31j? && actual_encoding != Encoding::Windows_31J - raise "expected regexp encoding to be Windows-31J due to '/s' modifier, but got #{actual_encoding.name}" - end - - if regexp.utf_8? && regexp.forced_utf8_encoding? - raise "the forced_utf8 flag should not be set when the UTF-8 modifier (/u) is used" - elsif regexp.ascii_8bit? && regexp.forced_binary_encoding? - raise "the forced_ascii_8bit flag should not be set when the UTF-8 modifier (/u) is used" - end - - actual_encoding - else - errors = result.errors.map(&:message) - - if errors.last&.include?("UTF-8 mixed within") - nil - else - errors - end - end - end - - # TODO (nirvdrum 22-Feb-2024): Remove this workaround once Prism better maps CRuby's error messages. - # This class of error message is tricky. The part not being compared is a representation of the regexp. - # Depending on the source encoding and any encoding modifiers being used, CRuby alters how the regexp is represented. - # Sometimes it's an MBC string. Other times it uses hexadecimal character escapes. And in other cases it uses - # the long-form Unicode escape sequences. This short-circuit checks that the error message is mostly correct. - if expected.is_a?(Array) && actual.is_a?(Array) - if expected.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") && - actual.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") - expected.last.clear - actual.last.clear - end - end - - assert_equal expected, actual - end - end - end -end diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 280bf63a24..c995748083 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -99,7 +99,7 @@ module Prism ) assert_errors expected, "BEGIN { 1 + }", [ - ["expected an expression after the operator", 10..11], + ["unexpected '}'; expected an expression after the operator", 12..13], ["unexpected '}', assuming it is closing the parent 'BEGIN' block", 12..13] ] end @@ -195,8 +195,7 @@ module Prism def test_unterminated_parenthesized_expression assert_errors expression('(1 + 2'), '(1 + 2', [ - ["unexpected end of file, expecting end-of-input", 6..6], - ["unexpected end of file, assuming it is closing the parent top level context", 6..6], + ["unexpected end-of-input, assuming it is closing the parent top level context", 6..6], ["expected a matching `)`", 6..6] ] end @@ -209,21 +208,21 @@ module Prism def test_unterminated_argument_expression assert_errors expression('a %'), 'a %', [ - ["invalid `%` token", 2..3], - ["expected an expression after the operator", 2..3], - ["unexpected end of file, assuming it is closing the parent top level context", 3..3] + ["unterminated quoted string meets end of file", 2..3], + ["unexpected end-of-input; expected an expression after the operator", 3..3], + ["unexpected end-of-input, assuming it is closing the parent top level context", 3..3] ] end def test_unterminated_interpolated_symbol assert_error_messages ":\"#", [ - "expected a closing delimiter for the interpolated symbol" + "unterminated symbol; expected a closing delimiter for the interpolated symbol" ] end def test_cr_without_lf_in_percent_expression assert_errors expression("%\r"), "%\r", [ - ["invalid `%` token", 0..2], + ["unterminated string meets end of file", 2..2], ] end @@ -365,7 +364,7 @@ module Prism assert_error_messages "x.each { x end", [ "unexpected 'end', expecting end-of-input", "unexpected 'end', ignoring it", - "unexpected end of file, assuming it is closing the parent top level context", + "unexpected end-of-input, assuming it is closing the parent top level context", "expected a block beginning with `{` to end with `}`" ] end @@ -716,12 +715,13 @@ module Prism assert_errors expected, '"\u{000z}"', [ ["invalid Unicode escape sequence", 7..7], + ["unterminated Unicode escape", 7..7] ] end def test_unterminated_unicode_brackets_should_be_a_syntax_error assert_errors expression('?\\u{3'), '?\\u{3', [ - ["invalid Unicode escape sequence; needs closing `}`", 1..5], + ["unterminated Unicode escape", 1..5], ] end @@ -864,7 +864,7 @@ module Prism :foo, Location(), nil, - ParametersNode([], [], nil, [], [], ForwardingParameterNode(), nil), + ParametersNode([], [], nil, [ForwardingParameterNode()], [], ForwardingParameterNode(), nil), nil, [], Location(), @@ -1241,13 +1241,12 @@ module Prism expected = CallNode(0, receiver, Location(), :foo, Location(), nil, nil, nil, nil) assert_errors expected, "<<~FOO.foo\n", [ - ["unterminated heredoc; can't find string \"FOO\"", 3..6] + ["unterminated heredoc; can't find string \"FOO\" anywhere before EOF", 3..6] ] end def test_invalid_message_name - result = Prism.parse("+.@foo,+=foo") - assert_equal :"", result.value.statements.body.first.write_name + assert_equal :"", Prism.parse_statement("+.@foo,+=foo").write_name end def test_invalid_operator_write_fcall @@ -1356,14 +1355,14 @@ module Prism if RUBY_VERSION >= "3.0" def test_writing_numbered_parameter - assert_errors expression("-> { _1 = 0 }"), "-> { _1 = 0 }", [ - ["_1 is reserved for numbered parameters", 5..7] + assert_error_messages "-> { _1 = 0 }", [ + "_1 is reserved for numbered parameters" ] end def test_targeting_numbered_parameter - assert_errors expression("-> { _1, = 0 }"), "-> { _1, = 0 }", [ - ["_1 is reserved for numbered parameters", 5..7] + assert_error_messages "-> { _1, = 0 }", [ + "_1 is reserved for numbered parameters" ] end @@ -1377,7 +1376,7 @@ module Prism def test_double_scope_numbered_parameters source = "-> { _1 + -> { _2 } }" - errors = [["numbered parameter is already used in outer scope", 15..17]] + errors = [["numbered parameter is already used in outer block", 15..17]] assert_errors expression(source), source, errors end @@ -1401,7 +1400,7 @@ module Prism end def test_alnum_delimiters - error_messages = ["invalid `%` token"] + error_messages = ["unknown type of %string"] assert_error_messages "%qXfooX", error_messages assert_error_messages "%QXfooX", error_messages @@ -1466,7 +1465,7 @@ module Prism def test_forwarding_arg_after_keyword_rest source = "def f(**,...);end" assert_errors expression(source), source, [ - ["unexpected `...` in parameters", 9..12], + ["unexpected parameter order", 9..12] ] end @@ -1482,8 +1481,7 @@ module Prism assert_errors expression(source), source, [ ["expected a `do` keyword or a `{` to open the lambda block", 3..3], - ["unexpected end of file, expecting end-of-input", 7..7], - ["unexpected end of file, assuming it is closing the parent top level context", 7..7], + ["unexpected end-of-input, assuming it is closing the parent top level context", 7..7], ["expected a lambda block beginning with `do` to end with `end`", 7..7] ] end @@ -1541,7 +1539,7 @@ module Prism assert_errors expression(source), source, [ ["expected a predicate expression for the `while` statement", 22..22], - ["unexpected end of file, assuming it is closing the parent top level context", 22..22], + ["unexpected end-of-input, assuming it is closing the parent top level context", 22..22], ["expected an `end` to close the `while` statement", 22..22] ] end @@ -1631,6 +1629,41 @@ module Prism ] end + def test_void_value_expression_in_begin_statement + source = <<~RUBY + x = return 1 + x = return, 1 + x = 1, return + x, y = return + x = begin return ensure end + x = begin ensure return end + x = begin return ensure return end + x = begin return; rescue; return end + x = begin return; rescue; return; else return end + x = begin; return; rescue; retry; end + RUBY + + message = 'unexpected void value expression' + assert_errors expression(source), source, [ + [message, 4..12], + [message, 17..23], + [message, 34..40], + [message, 48..54], + [message, 65..71], + [message, 100..106], + [message, 121..127], + [message, 156..162], + [message, 222..228], + [message, 244..250], + ] + + refute_error_messages("x = begin return; rescue; end") + refute_error_messages("x = begin return; rescue; return; else end") + refute_error_messages("x = begin; rescue; retry; end") + refute_error_messages("x = begin 1; rescue; retry; ensure; end") + refute_error_messages("x = begin 1; rescue; return; end") + end + def test_void_value_expression_in_def source = <<~RUBY def (return).x @@ -1942,10 +1975,10 @@ module Prism RUBY assert_errors expression(source), source, [ - ["unexpected '..', expecting end-of-input", 3..5], - ["unexpected '..', ignoring it", 3..5], - ["unexpected '..', expecting end-of-input", 10..12], - ["unexpected '..', ignoring it", 10..12] + ["unexpected .., expecting end-of-input", 3..5], + ["unexpected .., ignoring it", 3..5], + ["unexpected .., expecting end-of-input", 10..12], + ["unexpected .., ignoring it", 10..12] ] end @@ -1957,12 +1990,18 @@ module Prism proc { |foo: foo| } RUBY - assert_errors expression(source), source, [ - ["circular argument reference - bar", 8..11], - ["circular argument reference - bar", 32..35], - ["circular argument reference - foo", 55..58], - ["circular argument reference - foo", 76..79] - ] + assert_errors( + expression(source), + source, + [ + ["circular argument reference - bar", 8..11], + ["circular argument reference - bar", 32..35], + ["circular argument reference - foo", 55..58], + ["circular argument reference - foo", 76..79] + ], + check_valid_syntax: false, + version: "3.3.0" + ) refute_error_messages("def foo(bar: bar = 1); end") end @@ -2082,7 +2121,7 @@ module Prism def test_forwarding_arg_and_block source = 'def foo(...) = foo(...) { }' assert_errors expression(source), source, [ - ['both a block argument and a forwarding argument; only one block is allowed', 24..27] + ['both block arg and actual block given; only one block is allowed', 24..27] ] end @@ -2211,10 +2250,10 @@ module Prism private - def assert_errors(expected, source, errors, check_valid_syntax: true) + def assert_errors(expected, source, errors, check_valid_syntax: true, **options) refute_valid_syntax(source) if check_valid_syntax - result = Prism.parse(source) + result = Prism.parse(source, **options) node = result.value.statements.body.last assert_equal_nodes(expected, node, compare_location: false) diff --git a/test/prism/fixtures/break.txt b/test/prism/fixtures/break.txt index 82fa45bdb4..5532322c5c 100644 --- a/test/prism/fixtures/break.txt +++ b/test/prism/fixtures/break.txt @@ -23,3 +23,7 @@ tap { break(1) } foo { break 42 } == 42 foo { |a| break } == 42 + +while _ && break; end + +until _ && break; end diff --git a/test/prism/fixtures/seattlerb/pct_Q_backslash_nl.txt b/test/prism/fixtures/seattlerb/pct_Q_backslash_nl.txt index d88e1fd21c..4420560d2b 100644 --- a/test/prism/fixtures/seattlerb/pct_Q_backslash_nl.txt +++ b/test/prism/fixtures/seattlerb/pct_Q_backslash_nl.txt @@ -1,2 +1,2 @@ -%Q{ \ +%q{ \ } diff --git a/test/prism/fixtures/whitequark/method_definition_in_while_cond.txt b/test/prism/fixtures/whitequark/method_definition_in_while_cond.txt new file mode 100644 index 0000000000..6ec38906a1 --- /dev/null +++ b/test/prism/fixtures/whitequark/method_definition_in_while_cond.txt @@ -0,0 +1,7 @@ +while def foo a = tap do end; end; break; end + +while def foo; tap do end; end; break; end + +while def self.foo a = tap do end; end; break; end + +while def self.foo; tap do end; end; break; end diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb new file mode 100644 index 0000000000..7225b4ac66 --- /dev/null +++ b/test/prism/fixtures_test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +return if RUBY_VERSION < "3.2.0" + +require_relative "test_helper" + +module Prism + class FixturesTest < TestCase + except = [] + + # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace + # characters in the heredoc start. + # Example: <<~' EOF' or <<-' EOF' + # https://bugs.ruby-lang.org/issues/19539 + except << "heredocs_leading_whitespace.txt" if RUBY_VERSION < "3.3.0" + + Fixture.each(except: except) do |fixture| + define_method(fixture.test_name) { assert_valid_syntax(fixture.read) } + end + end +end diff --git a/test/prism/format_errors_test.rb b/test/prism/format_errors_test.rb deleted file mode 100644 index a1edbef2e8..0000000000 --- a/test/prism/format_errors_test.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -return if Prism::BACKEND == :FFI - -module Prism - class FormatErrorsTest < TestCase - def test_format_errors - assert_equal <<~ERROR, Debug.format_errors("<>", false) - > 1 | <> - | ^ unexpected '<', ignoring it - | ^ unexpected '>', ignoring it - ERROR - - assert_equal <<~'ERROR', Debug.format_errors('"%W"\u"', false) - > 1 | "%W"\u" - | ^ unexpected backslash, ignoring it - | ^ unexpected local variable or method, expecting end-of-input - | ^ unterminated string meets end of file - ERROR - end - end -end diff --git a/test/prism/fuzzer_test.rb b/test/prism/fuzzer_test.rb index 511210e7ee..4927478bdc 100644 --- a/test/prism/fuzzer_test.rb +++ b/test/prism/fuzzer_test.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -return if ENV["PRISM_BUILD_MINIMAL"] - require_relative "test_helper" module Prism @@ -9,7 +7,7 @@ module Prism # invalid memory access. class FuzzerTest < TestCase def self.snippet(name, source) - define_method(:"test_fuzzer_#{name}") { Prism.dump(source) } + define_method(:"test_fuzzer_#{name}") { Prism.profile(source) } end snippet "incomplete global variable", "$" @@ -39,29 +37,31 @@ module Prism snippet "escaped unicode at end of file 8", '"\\u33' snippet "escaped unicode at end of file 9", '"\\u333' snippet "float suffix at end of file", "1e" + snippet "parameter name that is zero length", "a { |b;" snippet "statements node with multiple heredocs", <<~EOF for <<A + <<B A B EOF + snippet "create a binary call node with arg before receiver", <<~EOF <<-A.g/{/ A /, ""\\ EOF + snippet "regular expression with start and end out of order", <<~RUBY <<-A.g//, A /{/, ''\\ RUBY + snippet "interpolated regular expression with start and end out of order", <<~RUBY <<-A.g/{/, A a /{/, ''\\ RUBY - - snippet "parameter name that is zero length", "a { |b;" end end diff --git a/test/prism/heredoc_dedent_test.rb b/test/prism/heredoc_dedent_test.rb index 9fbc4d936a..4e7a3c0a14 100644 --- a/test/prism/heredoc_dedent_test.rb +++ b/test/prism/heredoc_dedent_test.rb @@ -4,24 +4,131 @@ require_relative "test_helper" module Prism class HeredocDedentTest < TestCase - filepath = File.expand_path("fixtures/tilde_heredocs.txt", __dir__) + def test_content_dedented_interpolation_content + assert_heredoc_dedent( + " a\n" "1\n" " a\n", + "<<~EOF\n" " a\n" "\#{1}\n" " a\n" "EOF\n" + ) + end + + def test_content + assert_heredoc_dedent( + "a\n", + "<<~EOF\n" " a\n" "EOF\n" + ) + end + + def test_tabs_dedent_spaces + assert_heredoc_dedent( + "\ta\n" "b\n" "\t\tc\n", + "<<~EOF\n" "\ta\n" " b\n" "\t\tc\n" "EOF\n" + ) + end + + def test_interpolation_then_content + assert_heredoc_dedent( + "1 a\n", + "<<~EOF\n" " \#{1} a\n" "EOF\n" + ) + end + + def test_content_then_interpolation + assert_heredoc_dedent( + "a 1\n", + "<<~EOF\n" " a \#{1}\n" "EOF\n" + ) + end + + def test_content_dedented_interpolation + assert_heredoc_dedent( + " a\n" "1\n", + "<<~EOF\n" " a\n" " \#{1}\n" "EOF\n" + ) + end + + def test_content_interpolation + assert_heredoc_dedent( + "a\n" "1\n", + "<<~EOF\n" " a\n" " \#{1}\n" "EOF\n" + ) + end - File.read(filepath).split(/(?=\n)\n(?=<)/).each_with_index do |heredoc, index| - # The first example in this file has incorrect dedent calculated by - # TruffleRuby so we skip it. - next if index == 0 && RUBY_ENGINE == "truffleruby" + def test_content_content + assert_heredoc_dedent( + "a\n" "b\n", + "<<~EOF\n" " a\n" " b\n" "EOF\n" + ) + end - define_method "test_heredoc_#{index}" do - node = Prism.parse(heredoc).value.statements.body.first + def test_content_indented_content + assert_heredoc_dedent( + "a\n" " b\n", + "<<~EOF\n" " a\n" " b\n" "EOF\n" + ) + end - if node.is_a?(StringNode) - actual = node.unescaped - else - actual = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : "1" }.join - end + def test_content_dedented_content + assert_heredoc_dedent( + "\ta\n" "b\n", + "<<~EOF\n" "\t\t\ta\n" "\t\tb\n" "EOF\n" + ) + end - assert_equal(eval(heredoc), actual, "Expected heredocs to match.") + def test_single_quote + assert_heredoc_dedent( + "a \#{1}\n", + "<<~'EOF'\n" "a \#{1}\n" "EOF\n" + ) + end + + def test_mixed_indentation + assert_heredoc_dedent( + "a\n" " b\n", + "<<~EOF\n" "\ta\n" "\t b\n" "EOF\n" + ) + end + + def test_indented_content_content + assert_heredoc_dedent( + " a\n" "b\n", + "<<~EOF\n" "\t a\n" "\tb\n" "EOF\n" + ) + end + + def test_indent_size + assert_heredoc_dedent( + "a\n" " b\n", + "<<~EOF\n" "\ta\n" " b\n" "EOF\n" + ) + end + + def test_blank_lines + assert_heredoc_dedent( + "a\n" "\n" "b\n", + "<<~EOF\n" " a\n" "\n" " b\n" "EOF\n" + ) + end + + def test_many_blank_lines + assert_heredoc_dedent( + "a\n" "\n" "\n" "\n" "\n" "b\n", + "<<~EOF\n" " a\n" "\n" "\n" "\n" "\n" " b\n" "EOF\n" + ) + end + + private + + def assert_heredoc_dedent(expected, source) + node = Prism.parse_statement(source) + + if node.is_a?(StringNode) + actual = node.unescaped + else + actual = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : "1" }.join end + + assert_equal(expected, actual) + assert_equal(eval(source), actual) end end end diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb new file mode 100644 index 0000000000..7eac677ef7 --- /dev/null +++ b/test/prism/lex_test.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +return if !(RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.2.0") + +require_relative "test_helper" + +module Prism + class LexTest < TestCase + except = [ + # It seems like there are some oddities with nested heredocs and ripper. + # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. + "seattlerb/heredoc_nested.txt", + "whitequark/dedenting_heredoc.txt", + # Ripper seems to have a bug that the regex portions before and after + # the heredoc are combined into a single token. See + # https://bugs.ruby-lang.org/issues/19838. + "spanning_heredoc.txt", + "spanning_heredoc_newlines.txt" + ] + + if RUBY_VERSION < "3.3.0" + # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if + # we're on an earlier version. + except << "seattlerb/pct_w_heredoc_interp_nested.txt" + + # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace + # characters in the heredoc start. + # Example: <<~' EOF' or <<-' EOF' + # https://bugs.ruby-lang.org/issues/19539 + except << "heredocs_leading_whitespace.txt" + end + + Fixture.each(except: except) do |fixture| + define_method(fixture.test_name) { assert_lex(fixture) } + end + + def test_lex_file + assert_nothing_raised do + Prism.lex_file(__FILE__) + end + + error = assert_raise Errno::ENOENT do + Prism.lex_file("idontexist.rb") + end + + assert_equal "No such file or directory - idontexist.rb", error.message + + assert_raise TypeError do + Prism.lex_file(nil) + end + end + + def test_parse_lex + node, tokens = Prism.parse_lex("def foo; end").value + + assert_kind_of ProgramNode, node + assert_equal 5, tokens.length + end + + def test_parse_lex_file + node, tokens = Prism.parse_lex_file(__FILE__).value + + assert_kind_of ProgramNode, node + refute_empty tokens + + error = assert_raise Errno::ENOENT do + Prism.parse_lex_file("idontexist.rb") + end + + assert_equal "No such file or directory - idontexist.rb", error.message + + assert_raise TypeError do + Prism.parse_lex_file(nil) + end + end + + private + + def assert_lex(fixture) + source = fixture.read + + result = Prism.lex_compat(source) + assert_equal [], result.errors + + Prism.lex_ripper(source).zip(result.value).each do |(ripper, prism)| + assert_equal ripper, prism + end + end + end +end diff --git a/test/prism/library_symbols_test.rb b/test/prism/library_symbols_test.rb index b10a367c18..44f225478b 100644 --- a/test/prism/library_symbols_test.rb +++ b/test/prism/library_symbols_test.rb @@ -3,8 +3,6 @@ require_relative "test_helper" return if RUBY_PLATFORM !~ /linux/ - -# TODO: determine why these symbols are incorrect on ppc64le return if RUBY_PLATFORM =~ /powerpc64le/ module Prism diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb index 0eb73f1b9c..27fdfc90ef 100644 --- a/test/prism/locals_test.rb +++ b/test/prism/locals_test.rb @@ -17,14 +17,14 @@ require_relative "test_helper" module Prism class LocalsTest < TestCase - base = File.join(__dir__, "fixtures") - Dir["**/*.txt", base: base].each do |relative| + except = [ # Skip this fixture because it has a different number of locals because # CRuby is eliminating dead code. - next if relative == "whitequark/ruby_bug_10653.txt" + "whitequark/ruby_bug_10653.txt" + ] - filepath = File.join(base, relative) - define_method("test_#{relative}") { assert_locals(filepath) } + Fixture.each(except: except) do |fixture| + define_method(fixture.test_name) { assert_locals(fixture) } end def setup @@ -38,21 +38,188 @@ module Prism private - def assert_locals(filepath) - source = File.read(filepath) + def assert_locals(fixture) + source = fixture.read - expected = Debug.cruby_locals(source) - actual = Debug.prism_locals(source) + expected = cruby_locals(source) + actual = prism_locals(source) assert_equal(expected, actual) end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity + # A wrapper around a RubyVM::InstructionSequence that provides a more + # convenient interface for accessing parts of the iseq. + class ISeq + attr_reader :parts + + def initialize(parts) + @parts = parts + end + + def type + parts[0] + end + + def local_table + parts[10] + end + + def instructions + parts[13] + end + + def each_child + instructions.each do |instruction| + # Only look at arrays. Other instructions are line numbers or + # tracepoint events. + next unless instruction.is_a?(Array) + + instruction.each do |opnd| + # Only look at arrays. Other operands are literals. + next unless opnd.is_a?(Array) + + # Only look at instruction sequences. Other operands are literals. + next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat" + + yield ISeq.new(opnd) + end + end + end + end + + # Used to hold the place of a local that will be in the local table but + # cannot be accessed directly from the source code. For example, the + # iteration variable in a for loop or the positional parameter on a method + # definition that is destructured. + AnonymousLocal = Object.new + + # For the given source, compiles with CRuby and returns a list of all of the + # sets of local variables that were encountered. + def cruby_locals(source) + locals = [] #: Array[Array[Symbol | Integer]] + stack = [ISeq.new(ignore_warnings { RubyVM::InstructionSequence.compile(source) }.to_a)] + + while (iseq = stack.pop) + names = [*iseq.local_table] + names.map!.with_index do |name, index| + # When an anonymous local variable is present in the iseq's local + # table, it is represented as the stack offset from the top. + # However, when these are dumped to binary and read back in, they + # are replaced with the symbol :#arg_rest. To consistently handle + # this, we replace them here with their index. + if name == :"#arg_rest" + names.length - index + 1 + else + name + end + end + + locals << names + iseq.each_child { |child| stack << child } + end + + locals + end + + # For the given source, parses with prism and returns a list of all of the + # sets of local variables that were encountered. + def prism_locals(source) + locals = [] #: Array[Array[Symbol | Integer]] + stack = [Prism.parse(source).value] #: Array[Prism::node] + + while (node = stack.pop) + case node + when BlockNode, DefNode, LambdaNode + names = node.locals + params = + if node.is_a?(DefNode) + node.parameters + elsif node.parameters.is_a?(NumberedParametersNode) + nil + else + node.parameters&.parameters + end + + # prism places parameters in the same order that they appear in the + # source. CRuby places them in the order that they need to appear + # according to their own internal calling convention. We mimic that + # order here so that we can compare properly. + if params + sorted = [ + *params.requireds.map do |required| + if required.is_a?(RequiredParameterNode) + required.name + else + AnonymousLocal + end + end, + *params.optionals.map(&:name), + *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)), + *params.posts.map do |post| + if post.is_a?(RequiredParameterNode) + post.name + else + AnonymousLocal + end + end, + *params.keywords.grep(RequiredKeywordParameterNode).map(&:name), + *params.keywords.grep(OptionalKeywordParameterNode).map(&:name), + ] + + sorted << AnonymousLocal if params.keywords.any? + + if params.keyword_rest.is_a?(ForwardingParameterNode) + sorted.push(:*, :**, :&, :"...") + elsif params.keyword_rest.is_a?(KeywordRestParameterNode) + sorted << (params.keyword_rest.name || :**) + end + + # Recurse down the parameter tree to find any destructured + # parameters and add them after the other parameters. + param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse + while (param = param_stack.pop) + case param + when MultiTargetNode + param_stack.concat(param.rights.reverse) + param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name) + param_stack.concat(param.lefts.reverse) + when RequiredParameterNode + sorted << param.name + when SplatNode + sorted << param.expression.name + end + end + + if params.block + sorted << (params.block.name || :&) + end + + names = sorted.concat(names - sorted) + end + + names.map!.with_index do |name, index| + if name == AnonymousLocal + names.length - index + 1 + else + name + end + end + + locals << names + when ClassNode, ModuleNode, ProgramNode, SingletonClassNode + locals << node.locals + when ForNode + locals << [2] + when PostExecutionNode + locals.push([], []) + when InterpolatedRegularExpressionNode + locals << [] if node.once? + end + + stack.concat(node.compact_child_nodes) + end + + locals end end end diff --git a/test/prism/magic_comment_test.rb b/test/prism/magic_comment_test.rb index 9e2e92af92..14653fb0f8 100644 --- a/test/prism/magic_comment_test.rb +++ b/test/prism/magic_comment_test.rb @@ -2,32 +2,109 @@ require_relative "test_helper" -return if RUBY_ENGINE != "ruby" - module Prism class MagicCommentTest < TestCase - examples = [ - "# encoding: ascii", - "# coding: ascii", - "# eNcOdInG: ascii", - "# CoDiNg: ascii", - "# \s\t\v encoding \s\t\v : \s\t\v ascii \s\t\v", - "# -*- encoding: ascii -*-", - "# -*- coding: ascii -*-", - "# -*- eNcOdInG: ascii -*-", - "# -*- CoDiNg: ascii -*-", - "# -*- \s\t\v encoding \s\t\v : \s\t\v ascii \s\t\v -*-", - "# -*- foo: bar; encoding: ascii -*-", - "# coding \t \r \v : \t \v \r ascii-8bit", - "# vim: filetype=ruby, fileencoding=windows-31j, tabsize=3, shiftwidth=3" - ] - - examples.each.with_index(1) do |example, index| - define_method(:"test_magic_comment_#{index}") do - expected = RubyVM::InstructionSequence.compile(%Q{#{example}\n""}).eval.encoding - actual = Prism.parse(example).encoding + if RUBY_ENGINE == "ruby" + class MagicCommentRipper < Ripper + attr_reader :magic_comments + + def initialize(*) + super + @magic_comments = [] + end + + def on_magic_comment(key, value) + @magic_comments << [key, value] + super + end + end + + Fixture.each do |fixture| + define_method(fixture.test_name) { assert_magic_comments(fixture) } + end + end + + def test_encoding + assert_magic_encoding(Encoding::US_ASCII, "# encoding: ascii") + end + + def test_coding + assert_magic_encoding(Encoding::US_ASCII, "# coding: ascii") + end + + def test_eNcOdInG + assert_magic_encoding(Encoding::US_ASCII, "# eNcOdInG: ascii") + end + + def test_CoDiNg + assert_magic_encoding(Encoding::US_ASCII, "# CoDiNg: ascii") + end + + def test_encoding_whitespace + assert_magic_encoding(Encoding::US_ASCII, "# \s\t\v encoding \s\t\v : \s\t\v ascii \s\t\v") + end + + def test_emacs_encoding + assert_magic_encoding(Encoding::US_ASCII, "# -*- encoding: ascii -*-") + end + + def test_emacs_coding + assert_magic_encoding(Encoding::US_ASCII, "# -*- coding: ascii -*-") + end + + def test_emacs_eNcOdInG + assert_magic_encoding(Encoding::US_ASCII, "# -*- eNcOdInG: ascii -*-") + end + + def test_emacs_CoDiNg + assert_magic_encoding(Encoding::US_ASCII, "# -*- CoDiNg: ascii -*-") + end + + def test_emacs_whitespace + assert_magic_encoding(Encoding::US_ASCII, "# -*- \s\t\v encoding \s\t\v : \s\t\v ascii \s\t\v -*-") + end + + def test_emacs_multiple + assert_magic_encoding(Encoding::US_ASCII, "# -*- foo: bar; encoding: ascii -*-") + end + + def test_coding_whitespace + assert_magic_encoding(Encoding::ASCII_8BIT, "# coding \t \r \v : \t \v \r ascii-8bit") + end + + def test_vim + assert_magic_encoding(Encoding::Windows_31J, "# vim: filetype=ruby, fileencoding=windows-31j, tabsize=3, shiftwidth=3") + end + + private + + def assert_magic_encoding(expected, line) + source = %Q{#{line}\n""} + actual = Prism.parse(source).encoding + + # Compare against our expectation. + assert_equal expected, actual + + # Compare against Ruby's expectation. + if defined?(RubyVM::InstructionSequence) + expected = RubyVM::InstructionSequence.compile(source).eval.encoding assert_equal expected, actual end end + + def assert_magic_comments(fixture) + source = fixture.read + + # Check that we get the correct number of magic comments when lexing with + # ripper. + expected = MagicCommentRipper.new(source).tap(&:parse).magic_comments + actual = Prism.parse(source).magic_comments + + assert_equal expected.length, actual.length + expected.zip(actual).each do |(expected_key, expected_value), magic_comment| + assert_equal expected_key, magic_comment.key + assert_equal expected_value, magic_comment.value + end + end end end diff --git a/test/prism/memsize_test.rb b/test/prism/memsize_test.rb deleted file mode 100644 index d7e1448dbc..0000000000 --- a/test/prism/memsize_test.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -return if Prism::BACKEND == :FFI - -module Prism - class MemsizeTest < TestCase - def test_memsize - result = Debug.memsize("2 + 3") - - assert_equal 5, result[:length] - assert_kind_of Integer, result[:memsize] - assert_equal 6, result[:node_count] - end - end -end diff --git a/test/prism/newline_offsets_test.rb b/test/prism/newline_offsets_test.rb new file mode 100644 index 0000000000..99b808b1df --- /dev/null +++ b/test/prism/newline_offsets_test.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module Prism + class NewlineOffsetsTest < TestCase + Fixture.each do |fixture| + define_method(fixture.test_name) { assert_newline_offsets(fixture) } + end + + private + + def assert_newline_offsets(fixture) + source = fixture.read + + expected = [0] + source.b.scan("\n") { expected << $~.offset(0)[0] + 1 } + + assert_equal expected, Prism.parse(source).source.offsets + end + end +end diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index d31fb89bc6..03d7df4c97 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -6,18 +6,23 @@ return unless defined?(RubyVM::InstructionSequence) module Prism class NewlineTest < TestCase - base = File.expand_path("../", __FILE__) - filepaths = Dir["*.rb", base: base] - %w[ - encoding_test.rb + skips = %w[ errors_test.rb - parser_test.rb + locals_test.rb regexp_test.rb - static_literals_test.rb + test_helper.rb unescape_test.rb + encoding/regular_expression_encoding_test.rb + encoding/string_encoding_test.rb + result/static_literals_test.rb + result/warnings_test.rb + ruby/parser_test.rb + ruby/ruby_parser_test.rb ] - filepaths.each do |relative| - define_method("test_newline_flags_#{relative}") do + base = __dir__ + (Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base] - skips).each do |relative| + define_method(:"test_#{relative}") do assert_newlines(base, relative) end end @@ -63,14 +68,6 @@ module Prism assert_equal expected, actual end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity - end - def rubyvm_lines(source) queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] lines = [] diff --git a/test/prism/onigmo_test.rb b/test/prism/onigmo_test.rb new file mode 100644 index 0000000000..03f44c4e4c --- /dev/null +++ b/test/prism/onigmo_test.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +return if RUBY_ENGINE != "ruby" + +require_relative "test_helper" + +begin + require "onigmo" +rescue LoadError + # In CRuby's CI, we're not going to test against the parser gem because we + # don't want to have to install it. So in this case we'll just skip this test. + return +end + +module Prism + class OnigmoTest < TestCase + def test_ONIGERR_PARSE_DEPTH_LIMIT_OVER + assert_error(%Q{#{"(" * 4096}a#{")" * 4096}}, "parse depth limit over") + end + + def test_ONIGERR_EMPTY_CHAR_CLASS + assert_error("[]", "empty char-class") + end + + def test_ONIGERR_TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED + assert_error("*", "target of repeat operator is not specified") + assert_error("+", "target of repeat operator is not specified") + assert_error("?", "target of repeat operator is not specified") + end + + def test_ONIGERR_EMPTY_GROUP_NAME + assert_error("(?<>)", "group name is empty") + end + + def test_ONIGERR_END_PATTERN_WITH_UNMATCHED_PARENTHESIS + assert_error("(", "end pattern with unmatched parenthesis") + assert_error("(|", "end pattern with unmatched parenthesis") + assert_error("(?<", "end pattern with unmatched parenthesis") + end + + def test_ONIGERR_END_PATTERN_IN_GROUP + assert_error("(?", "end pattern in group") + assert_error("(?#", "end pattern in group") + end + + def test_ONIGERR_UNDEFINED_GROUP_OPTION + assert_error("(?P", "undefined group option") + end + + def test_ONIGERR_UNMATCHED_CLOSE_PARENTHESIS + assert_error(")", "unmatched close parenthesis") + end + + private + + def assert_error(source, message) + result = Prism.parse("/#{source}/") + + assert result.failure?, "Expected #{source.inspect} to error" + assert_equal message, result.errors.first.message + + error = assert_raise(ArgumentError) { Onigmo.parse(source) } + assert_equal message, error.message + end + end +end diff --git a/test/prism/parse_test.rb b/test/prism/parse_test.rb deleted file mode 100644 index afb53e0668..0000000000 --- a/test/prism/parse_test.rb +++ /dev/null @@ -1,371 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -module Prism - class ParseTest < TestCase - # A subclass of Ripper that extracts out magic comments. - class MagicCommentRipper < Ripper - attr_reader :magic_comments - - def initialize(*) - super - @magic_comments = [] - end - - def on_magic_comment(key, value) - @magic_comments << [key, value] - super - end - end - - # When we pretty-print the trees to compare against the snapshots, we want to - # be certain that we print with the same external encoding. This is because - # methods like Symbol#inspect take into account external encoding and it could - # change how the snapshot is generated. On machines with certain settings - # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're - # going to force it to be UTF-8 to keep the snapshots consistent. - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end - - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end - - def test_empty_string - result = Prism.parse("") - assert_equal [], result.value.statements.body - end - - def test_parse_takes_file_path - filepath = "filepath.rb" - result = Prism.parse("def foo; __FILE__; end", filepath: filepath) - - assert_equal filepath, find_source_file_node(result.value).filepath - end - - def test_parse_takes_line - line = 4 - result = Prism.parse("def foo\n __FILE__\nend", line: line) - - assert_equal line, result.value.location.start_line - assert_equal line + 1, find_source_file_node(result.value).location.start_line - - result = Prism.parse_lex("def foo\n __FILE__\nend", line: line) - assert_equal line, result.value.first.location.start_line - end - - def test_parse_takes_negative_lines - line = -2 - result = Prism.parse("def foo\n __FILE__\nend", line: line) - - assert_equal line, result.value.location.start_line - assert_equal line + 1, find_source_file_node(result.value).location.start_line - - result = Prism.parse_lex("def foo\n __FILE__\nend", line: line) - assert_equal line, result.value.first.location.start_line - end - - def test_parse_lex - node, tokens = Prism.parse_lex("def foo; end").value - - assert_kind_of ProgramNode, node - assert_equal 5, tokens.length - end - - if !ENV["PRISM_BUILD_MINIMAL"] - def test_dump_file - assert_nothing_raised do - Prism.dump_file(__FILE__) - end - - error = assert_raise Errno::ENOENT do - Prism.dump_file("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.dump_file(nil) - end - end - end - - def test_lex_file - assert_nothing_raised do - Prism.lex_file(__FILE__) - end - - error = assert_raise Errno::ENOENT do - Prism.lex_file("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.lex_file(nil) - end - end - - def test_parse_lex_file - node, tokens = Prism.parse_lex_file(__FILE__).value - - assert_kind_of ProgramNode, node - refute_empty tokens - - error = assert_raise Errno::ENOENT do - Prism.parse_lex_file("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.parse_lex_file(nil) - end - end - - def test_parse_file - node = Prism.parse_file(__FILE__).value - assert_kind_of ProgramNode, node - - error = assert_raise Errno::ENOENT do - Prism.parse_file("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.parse_file(nil) - end - end - - def test_parse_file_success - assert_predicate Prism.parse_file_comments(__FILE__), :any? - - error = assert_raise Errno::ENOENT do - Prism.parse_file_comments("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.parse_file_comments(nil) - end - end - - def test_parse_file_comments - assert_predicate Prism.parse_file_comments(__FILE__), :any? - - error = assert_raise Errno::ENOENT do - Prism.parse_file_comments("idontexist.rb") - end - - assert_equal "No such file or directory - idontexist.rb", error.message - - assert_raise TypeError do - Prism.parse_file_comments(nil) - end - end - - # To accurately compare against Ripper, we need to make sure that we're - # running on CRuby 3.2+. - ripper_enabled = RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.2.0" - - # The FOCUS environment variable allows you to specify one particular fixture - # to test, instead of all of them. - base = File.join(__dir__, "fixtures") - relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] - - relatives.each do |relative| - # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" - next if RUBY_ENGINE == "truffleruby" and %w[emoji_method_calls.txt seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) - - filepath = File.join(base, relative) - snapshot = File.expand_path(File.join("snapshots", relative), __dir__) - - directory = File.dirname(snapshot) - FileUtils.mkdir_p(directory) unless File.directory?(directory) - - ripper_should_match = ripper_enabled - check_valid_syntax = RUBY_VERSION >= "3.2.0" - - case relative - when "seattlerb/pct_w_heredoc_interp_nested.txt" - # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if - # we're on an earlier version. - ripper_should_match = false if RUBY_VERSION < "3.3.0" - when "seattlerb/heredoc_nested.txt", "whitequark/dedenting_heredoc.txt" - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - ripper_should_match = false - when "spanning_heredoc.txt", "spanning_heredoc_newlines.txt" - # Ripper seems to have a bug that the regex portions before and after - # the heredoc are combined into a single token. See - # https://bugs.ruby-lang.org/issues/19838. - ripper_should_match = false - when "heredocs_leading_whitespace.txt" - # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace - # characters in the heredoc start. - # Example: <<~' EOF' or <<-' EOF' - # https://bugs.ruby-lang.org/issues/19539 - if RUBY_VERSION < "3.3.0" - ripper_should_match = false - check_valid_syntax = false - end - end - - define_method "test_filepath_#{relative}" do - # First, read the source from the filepath. Use binmode to avoid - # converting CRLF on Windows, and explicitly set the external encoding - # to UTF-8 to override the binmode default. - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - - # Make sure that the given source is valid syntax, otherwise we have an - # invalid fixture. - assert_valid_syntax(source) if check_valid_syntax - - # Next, assert that there were no errors during parsing. - result = Prism.parse(source, filepath: relative) - assert_empty result.errors - - # Next, pretty print the source. - printed = PP.pp(result.value, +"", 79) - - if File.exist?(snapshot) - saved = File.read(snapshot) - - # If the snapshot file exists, but the printed value does not match the - # snapshot, then update the snapshot file. - if printed != saved - File.write(snapshot, printed) - warn("Updated snapshot at #{snapshot}.") - end - - # If the snapshot file exists, then assert that the printed value - # matches the snapshot. - assert_equal(saved, printed) - else - # If the snapshot file does not yet exist, then write it out now. - File.write(snapshot, printed) - warn("Created snapshot at #{snapshot}.") - end - - if !ENV["PRISM_BUILD_MINIMAL"] - # Next, assert that the value can be serialized and deserialized - # without changing the shape of the tree. - assert_equal_nodes(result.value, Prism.load(source, Prism.dump(source, filepath: relative)).value) - end - - # Next, check that the location ranges of each node in the tree are a - # superset of their respective child nodes. - assert_non_overlapping_locations(result.value) - - # Next, assert that the newlines are in the expected places. - expected_newlines = [0] - source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } - assert_equal expected_newlines, Debug.newlines(source) - - if ripper_should_match - # Finally, assert that we can lex the source and get the same tokens as - # Ripper. - lex_result = Prism.lex_compat(source) - assert_equal [], lex_result.errors - tokens = lex_result.value - - begin - Prism.lex_ripper(source).zip(tokens).each do |(ripper, prism)| - assert_equal ripper, prism - end - rescue SyntaxError - raise ArgumentError, "Test file has invalid syntax #{filepath}" - end - - # Next, check that we get the correct number of magic comments when - # lexing with ripper. - expected = MagicCommentRipper.new(source).tap(&:parse).magic_comments - actual = result.magic_comments - - assert_equal expected.length, actual.length - expected.zip(actual).each do |(expected_key, expected_value), magic_comment| - assert_equal expected_key, magic_comment.key - assert_equal expected_value, magic_comment.value - end - end - end - end - - Dir["*.txt", base: base].each do |relative| - next if relative == "newline_terminated.txt" || relative == "spanning_heredoc_newlines.txt" - - # We test every snippet (separated by \n\n) in isolation - # to ensure the parser does not try to read bytes further than the end of each snippet - define_method "test_individual_snippets_#{relative}" do - filepath = File.join(base, relative) - - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - - file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| - snippet = snippet.rstrip - result = Prism.parse(snippet, filepath: relative) - assert_empty result.errors - - if !ENV["PRISM_BUILD_MINIMAL"] - assert_equal_nodes(result.value, Prism.load(snippet, Prism.dump(snippet, filepath: relative)).value) - end - end - end - end - - private - - # Check that the location ranges of each node in the tree are a superset of - # their respective child nodes. - def assert_non_overlapping_locations(node) - queue = [node] - - while (current = queue.shift) - # We only want to compare parent/child location overlap in the case that - # we are not looking at a heredoc. That's because heredoc locations are - # special in that they only use the declaration of the heredoc. - compare = !(current.is_a?(StringNode) || - current.is_a?(XStringNode) || - current.is_a?(InterpolatedStringNode) || - current.is_a?(InterpolatedXStringNode)) || - !current.opening&.start_with?("<<") - - current.child_nodes.each do |child| - # child_nodes can return nil values, so we need to skip those. - next unless child - - # Now that we know we have a child node, add that to the queue. - queue << child - - if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset - end - end - end - end - - def find_source_file_node(program) - queue = [program] - while (node = queue.shift) - return node if node.is_a?(SourceFileNode) - queue.concat(node.compact_child_nodes) - end - end - - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity - end - end -end diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb deleted file mode 100644 index 79b65cf75b..0000000000 --- a/test/prism/parser_test.rb +++ /dev/null @@ -1,186 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -begin - verbose, $VERBOSE = $VERBOSE, nil - require "parser/ruby33" - require "prism/translation/parser33" -rescue LoadError - # In CRuby's CI, we're not going to test against the parser gem because we - # don't want to have to install it. So in this case we'll just skip this test. - return -ensure - $VERBOSE = verbose -end - -# First, opt in to every AST feature. -Parser::Builders::Default.modernize - -# Modify the source map == check so that it doesn't check against the node -# itself so we don't get into a recursive loop. -Parser::Source::Map.prepend( - Module.new { - def ==(other) - self.class == other.class && - (instance_variables - %i[@node]).map do |ivar| - instance_variable_get(ivar) == other.instance_variable_get(ivar) - end.reduce(:&) - end - } -) - -# Next, ensure that we're comparing the nodes and also comparing the source -# ranges so that we're getting all of the necessary information. -Parser::AST::Node.prepend( - Module.new { - def ==(other) - super && (location == other.location) - end - } -) - -module Prism - class ParserTest < TestCase - base = File.join(__dir__, "fixtures") - - # These files are erroring because of the parser gem being wrong. - skip_incorrect = [ - "embdoc_no_newline_at_end.txt" - ] - - # These files are either failing to parse or failing to translate, so we'll - # skip them for now. - skip_all = skip_incorrect | [ - "dash_heredocs.txt", - "dos_endings.txt", - "heredocs_with_ignored_newlines.txt", - "regex.txt", - "regex_char_width.txt", - "spanning_heredoc.txt", - "spanning_heredoc_newlines.txt", - "unescaping.txt" - ] - - # Not sure why these files are failing on JRuby, but skipping them for now. - if RUBY_ENGINE == "jruby" - skip_all.push("emoji_method_calls.txt", "symbols.txt") - end - - # These files are failing to translate their lexer output into the lexer - # output expected by the parser gem, so we'll skip them for now. - skip_tokens = [ - "comments.txt", - "heredoc_with_comment.txt", - "indented_file_end.txt", - "methods.txt", - "strings.txt", - "tilde_heredocs.txt", - "xstring_with_backslash.txt" - ] - - Dir["*.txt", base: base].each do |name| - next if skip_all.include?(name) - - define_method("test_#{name}") do - assert_equal_parses(File.join(base, name), compare_tokens: !skip_tokens.include?(name)) - end - end - - private - - def assert_equal_parses(filepath, compare_tokens: true) - buffer = Parser::Source::Buffer.new(filepath, 1) - buffer.source = File.read(filepath) - - parser = Parser::Ruby33.new - parser.diagnostics.consumer = ->(*) {} - parser.diagnostics.all_errors_are_fatal = true - - expected_ast, expected_comments, expected_tokens = - begin - parser.tokenize(buffer) - rescue ArgumentError, Parser::SyntaxError - return - end - - actual_ast, actual_comments, actual_tokens = - Prism::Translation::Parser33.new.tokenize(buffer) - - assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) } - assert_equal_tokens(expected_tokens, actual_tokens) if compare_tokens - assert_equal_comments(expected_comments, actual_comments) - end - - def assert_equal_asts_message(expected_ast, actual_ast) - queue = [[expected_ast, actual_ast]] - - while (left, right = queue.shift) - if left.type != right.type - return "expected: #{left.type}\nactual: #{right.type}" - end - - if left.location != right.location - return "expected:\n#{left.inspect}\n#{left.location.inspect}\nactual:\n#{right.inspect}\n#{right.location.inspect}" - end - - if left.type == :str && left.children[0] != right.children[0] - return "expected: #{left.inspect}\nactual: #{right.inspect}" - end - - left.children.zip(right.children).each do |left_child, right_child| - queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node) - end - end - - "expected: #{expected_ast.inspect}\nactual: #{actual_ast.inspect}" - end - - def assert_equal_tokens(expected_tokens, actual_tokens) - if expected_tokens != actual_tokens - expected_index = 0 - actual_index = 0 - - while expected_index < expected_tokens.length - expected_token = expected_tokens[expected_index] - actual_token = actual_tokens[actual_index] - - expected_index += 1 - actual_index += 1 - - # The parser gem always has a space before a string end in list - # literals, but we don't. So we'll skip over the space. - if expected_token[0] == :tSPACE && actual_token[0] == :tSTRING_END - expected_index += 1 - next - end - - # There are a lot of tokens that have very specific meaning according - # to the context of the parser. We don't expose that information in - # prism, so we need to normalize these tokens a bit. - case actual_token[0] - when :kDO - actual_token[0] = expected_token[0] if %i[kDO_BLOCK kDO_LAMBDA].include?(expected_token[0]) - when :tLPAREN - actual_token[0] = expected_token[0] if expected_token[0] == :tLPAREN2 - when :tPOW - actual_token[0] = expected_token[0] if expected_token[0] == :tDSTAR - end - - # Now we can assert that the tokens are actually equal. - assert_equal expected_token, actual_token, -> { - "expected: #{expected_token.inspect}\n" \ - "actual: #{actual_token.inspect}" - } - end - end - end - - def assert_equal_comments(expected_comments, actual_comments) - assert_equal expected_comments, actual_comments, -> { - "expected: #{expected_comments.inspect}\n" \ - "actual: #{actual_comments.inspect}" - } - end - end -end diff --git a/test/prism/regexp_test.rb b/test/prism/regexp_test.rb index 0a5fc2b4fc..297020fc72 100644 --- a/test/prism/regexp_test.rb +++ b/test/prism/regexp_test.rb @@ -2,78 +2,84 @@ require_relative "test_helper" -return if Prism::BACKEND == :FFI - module Prism class RegexpTest < TestCase - ############################################################################## + ############################################################################ # These tests test the actual use case of extracting named capture groups - ############################################################################## + ############################################################################ def test_named_captures_with_arrows - assert_equal(["foo"], named_captures("(?<foo>bar)")) + assert_equal([:foo], named_captures("(?<foo>bar)")) end def test_named_captures_with_single_quotes - assert_equal(["foo"], named_captures("(?'foo'bar)")) + assert_equal([:foo], named_captures("(?'foo'bar)")) end def test_nested_named_captures_with_arrows - assert_equal(["foo", "bar"], named_captures("(?<foo>(?<bar>baz))")) + assert_equal([:foo, :bar], named_captures("(?<foo>(?<bar>baz))")) end def test_nested_named_captures_with_single_quotes - assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) + assert_equal([:foo, :bar], named_captures("(?'foo'(?'bar'baz))")) end def test_allows_duplicate_named_captures - assert_equal(["foo", "foo"], named_captures("(?<foo>bar)(?<foo>baz)")) + assert_equal([:foo], named_captures("(?<foo>bar)(?<foo>baz)")) end def test_named_capture_inside_fake_range_quantifier - assert_equal(["foo"], named_captures("foo{1, (?<foo>2)}")) + assert_equal([:foo], named_captures("foo{1, (?<foo>2)}")) + end + + def test_fake_named_captures_inside_character_sets + assert_equal([], named_captures("[a-z(?<foo>)]")) end - ############################################################################## + def test_fake_named_capture_inside_character_set_with_escaped_ending + assert_equal([], named_captures("[a-z\\](?<foo>)]")) + end + + ############################################################################ # These tests test the rest of the AST. They are not exhaustive, but they # should cover the most common cases. We test these to make sure we don't # accidentally regress and stop being able to extract named captures. - ############################################################################## + ############################################################################ def test_alternation - refute_nil(named_captures("foo|bar")) + assert_valid_regexp("foo|bar") end def test_anchors - refute_nil(named_captures("^foo$")) + assert_valid_regexp("^foo$") end def test_any - refute_nil(named_captures(".")) + assert_valid_regexp(".") end def test_posix_character_classes - refute_nil(named_captures("[[:digit:]]")) + assert_valid_regexp("[[:digit:]]") end def test_negated_posix_character_classes - refute_nil(named_captures("[[:^digit:]]")) + assert_valid_regexp("[[:^digit:]]") end def test_invalid_posix_character_classes_should_fall_back_to_regular_classes - refute_nil(named_captures("[[:foo]]")) + assert_valid_regexp("[[:foo]]") end def test_character_sets - refute_nil(named_captures("[abc]")) + assert_valid_regexp("[abc]") end def test_nested_character_sets - refute_nil(named_captures("[[abc]]")) + assert_valid_regexp("[[abc]]") end def test_nested_character_sets_with_operators - refute_nil(named_captures("[[abc] && [def]]")) + assert_valid_regexp("[[abc] && [def]]") end def test_named_capture_inside_nested_character_set @@ -81,120 +87,108 @@ module Prism end def test_negated_character_sets - refute_nil(named_captures("[^abc]")) + assert_valid_regexp("[^abc]") end def test_character_ranges - refute_nil(named_captures("[a-z]")) + assert_valid_regexp("[a-z]") end def test_negated_character_ranges - refute_nil(named_captures("[^a-z]")) - end - - def test_fake_named_captures_inside_character_sets - assert_equal([], named_captures("[a-z(?<foo>)]")) - end - - def test_fake_named_capture_inside_character_set_with_escaped_ending - assert_equal([], named_captures("[a-z\\](?<foo>)]")) + assert_valid_regexp("[^a-z]") end def test_comments - refute_nil(named_captures("(?#foo)")) + assert_valid_regexp("(?#foo)") end def test_comments_with_escaped_parentheses - refute_nil(named_captures("(?#foo\\)\\))")) + assert_valid_regexp("(?#foo\\)\\))") end def test_non_capturing_groups - refute_nil(named_captures("(?:foo)")) + assert_valid_regexp("(?:foo)") end def test_positive_lookaheads - refute_nil(named_captures("(?=foo)")) + assert_valid_regexp("(?=foo)") end def test_negative_lookaheads - refute_nil(named_captures("(?!foo)")) + assert_valid_regexp("(?!foo)") end def test_positive_lookbehinds - refute_nil(named_captures("(?<=foo)")) + assert_valid_regexp("(?<=foo)") end def test_negative_lookbehinds - refute_nil(named_captures("(?<!foo)")) + assert_valid_regexp("(?<!foo)") end def test_atomic_groups - refute_nil(named_captures("(?>foo)")) + assert_valid_regexp("(?>foo)") end def test_absence_operator - refute_nil(named_captures("(?~foo)")) + assert_valid_regexp("(?~foo)") end def test_conditional_expression_with_index - refute_nil(named_captures("(?(1)foo)")) + assert_valid_regexp("(?(1)foo)") end def test_conditional_expression_with_name - refute_nil(named_captures("(?(foo)bar)")) + assert_valid_regexp("(?(foo)bar)") end def test_conditional_expression_with_group - refute_nil(named_captures("(?(<foo>)bar)")) + assert_valid_regexp("(?(<foo>)bar)") end def test_options_on_groups - refute_nil(named_captures("(?imxdau:foo)")) - end - - def test_options_on_groups_with_invalid_options - assert_nil(named_captures("(?z:bar)")) + assert_valid_regexp("(?imxdau:foo)") end def test_options_on_groups_getting_turned_off - refute_nil(named_captures("(?-imx:foo)")) + assert_valid_regexp("(?-imx:foo)") end def test_options_on_groups_some_getting_turned_on_some_getting_turned_off - refute_nil(named_captures("(?im-x:foo)")) + assert_valid_regexp("(?im-x:foo)") end def test_star_quantifier - refute_nil(named_captures("foo*")) + assert_valid_regexp("foo*") end def test_plus_quantifier - refute_nil(named_captures("foo+")) + assert_valid_regexp("foo+") end def test_question_mark_quantifier - refute_nil(named_captures("foo?")) + assert_valid_regexp("foo?") end def test_endless_range_quantifier - refute_nil(named_captures("foo{1,}")) + assert_valid_regexp("foo{1,}") end def test_beginless_range_quantifier - refute_nil(named_captures("foo{,1}")) + assert_valid_regexp("foo{,1}") end def test_range_quantifier - refute_nil(named_captures("foo{1,2}")) + assert_valid_regexp("foo{1,2}") end def test_fake_range_quantifier_because_of_spaces - refute_nil(named_captures("foo{1, 2}")) + assert_valid_regexp("foo{1, 2}") end - ############################################################################## + ############################################################################ # These test that flag values are correct. - ############################################################################## + ############################################################################ def test_flag_ignorecase assert_equal(Regexp::IGNORECASE, options("i")) @@ -229,26 +223,30 @@ module Prism def test_last_encoding_option_wins regex = "/foo/nu" - option = Prism.parse(regex).value.statements.body.first.options + option = Prism.parse_statement(regex).options assert_equal Regexp::FIXEDENCODING, option regex = "/foo/un" - option = Prism.parse(regex).value.statements.body.first.options + option = Prism.parse_statement(regex).options assert_equal Regexp::NOENCODING, option end private + def assert_valid_regexp(source) + assert Prism.parse_success?("/#{source}/ =~ \"\"") + end + def named_captures(source) - Debug.named_captures(source) + Prism.parse("/#{source}/ =~ \"\"").value.locals end def options(flags) options = ["/foo/#{flags}", "/foo\#{1}/#{flags}"].map do |source| - Prism.parse(source).value.statements.body.first.options + Prism.parse_statement(source).options end # Check that we get the same set of options from both regular expressions diff --git a/test/prism/attribute_write_test.rb b/test/prism/result/attribute_write_test.rb index bd83d72da3..8f2e352738 100644 --- a/test/prism/attribute_write_test.rb +++ b/test/prism/result/attribute_write_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class AttributeWriteTest < TestCase @@ -41,18 +41,14 @@ module Prism private - def parse(source) - Prism.parse(source).value.statements.body.first - end - def assert_attribute_write(source) - call = parse(source) + call = Prism.parse_statement(source) assert(call.attribute_write?) assert_equal(1, eval(source)) end def refute_attribute_write(source) - call = parse(source) + call = Prism.parse_statement(source) refute(call.attribute_write?) refute_equal(1, eval(source)) end diff --git a/test/prism/comments_test.rb b/test/prism/result/comments_test.rb index b99c00268c..178623a75f 100644 --- a/test/prism/comments_test.rb +++ b/test/prism/result/comments_test.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class CommentsTest < TestCase def test_comment_inline source = "# comment" - assert_equal [0], Debug.newlines(source) + assert_equal [0], Prism.parse(source).source.offsets assert_comment( source, diff --git a/test/prism/constant_path_node_test.rb b/test/prism/result/constant_path_node_test.rb index dffb55c0ff..75925600ca 100644 --- a/test/prism/constant_path_node_test.rb +++ b/test/prism/result/constant_path_node_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class ConstantPathNodeTest < TestCase @@ -11,7 +11,7 @@ module Prism Qux RUBY - constant_path = Prism.parse(source).value.statements.body.first + constant_path = Prism.parse_statement(source) assert_equal("Foo::Bar::Baz::Qux", constant_path.full_name) end @@ -22,7 +22,7 @@ module Prism Qux RUBY - constant_path = Prism.parse(source).value.statements.body.first + constant_path = Prism.parse_statement(source) assert_raise(ConstantPathNode::DynamicPartsInConstantPathError) do constant_path.full_name end @@ -35,7 +35,7 @@ module Prism Qux RUBY - constant_path = Prism.parse(source).value.statements.body.first + constant_path = Prism.parse_statement(source) assert_raise(ConstantPathNode::DynamicPartsInConstantPathError) do constant_path.full_name @@ -49,7 +49,7 @@ module Prism Qux, Something = [1, 2] RUBY - node = Prism.parse(source).value.statements.body.first + node = Prism.parse_statement(source) assert_equal("Foo::Bar::Baz::Qux", node.lefts.first.full_name) end @@ -60,7 +60,7 @@ module Prism Qux, Something = [1, 2] RUBY - node = Prism.parse(source).value.statements.body.first + node = Prism.parse_statement(source) assert_equal("::Foo::Bar::Baz::Qux", node.lefts.first.full_name) end @@ -69,7 +69,7 @@ module Prism self::Foo, Bar = [1, 2] RUBY - constant_target = Prism.parse(source).value.statements.body.first + constant_target = Prism.parse_statement(source) dynamic, static = constant_target.lefts assert_raise(ConstantPathNode::DynamicPartsInConstantPathError) do @@ -84,7 +84,7 @@ module Prism Bar RUBY - constant = Prism.parse(source).value.statements.body.first + constant = Prism.parse_statement(source) assert_equal("Bar", constant.full_name) end end diff --git a/test/prism/result/equality_test.rb b/test/prism/result/equality_test.rb new file mode 100644 index 0000000000..4f6e665a88 --- /dev/null +++ b/test/prism/result/equality_test.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class EqualityTest < TestCase + def test_equality + assert_operator Prism.parse_statement("1"), :===, Prism.parse_statement("1") + assert_operator Prism.parse("1").value, :===, Prism.parse("1").value + + complex_source = "class Something; @var = something.else { _1 }; end" + assert_operator Prism.parse_statement(complex_source), :===, Prism.parse_statement(complex_source) + + refute_operator Prism.parse_statement("1"), :===, Prism.parse_statement("2") + refute_operator Prism.parse_statement("1"), :===, Prism.parse_statement("0x1") + + complex_source_1 = "class Something; @var = something.else { _1 }; end" + complex_source_2 = "class Something; @var = something.else { _2 }; end" + refute_operator Prism.parse_statement(complex_source_1), :===, Prism.parse_statement(complex_source_2) + end + end +end diff --git a/test/prism/result/heredoc_test.rb b/test/prism/result/heredoc_test.rb new file mode 100644 index 0000000000..7913c04a88 --- /dev/null +++ b/test/prism/result/heredoc_test.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class HeredocTest < TestCase + def test_heredoc? + refute Prism.parse_statement("\"foo\"").heredoc? + refute Prism.parse_statement("\"foo \#{1}\"").heredoc? + refute Prism.parse_statement("`foo`").heredoc? + refute Prism.parse_statement("`foo \#{1}`").heredoc? + + assert Prism.parse_statement("<<~HERE\nfoo\nHERE\n").heredoc? + assert Prism.parse_statement("<<~HERE\nfoo \#{1}\nHERE\n").heredoc? + assert Prism.parse_statement("<<~`HERE`\nfoo\nHERE\n").heredoc? + assert Prism.parse_statement("<<~`HERE`\nfoo \#{1}\nHERE\n").heredoc? + end + end +end diff --git a/test/prism/index_write_test.rb b/test/prism/result/index_write_test.rb index cf90eb082f..0d5383b601 100644 --- a/test/prism/index_write_test.rb +++ b/test/prism/result/index_write_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class IndexWriteTest < TestCase diff --git a/test/prism/result/integer_base_flags_test.rb b/test/prism/result/integer_base_flags_test.rb new file mode 100644 index 0000000000..ef15fb437c --- /dev/null +++ b/test/prism/result/integer_base_flags_test.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class IntegerBaseFlagsTest < TestCase + # Through some bit hackery, we want to allow consumers to use the integer + # base flags as the base itself. It has a nice property that the current + # alignment provides them in the correct order. So here we test that our + # assumption holds so that it doesn't change out from under us. + # + # In C, this would look something like: + # + # ((flags & ~DECIMAL) << 1) || 10 + # + # We have to do some other work in Ruby because 0 is truthy and ~ on an + # integer doesn't have a fixed width. + def test_flags + assert_equal 2, base("0b1") + assert_equal 8, base("0o1") + assert_equal 10, base("0d1") + assert_equal 16, base("0x1") + end + + private + + def base(source) + node = Prism.parse_statement(source) + value = (node.send(:flags) & (0b1111 - IntegerBaseFlags::DECIMAL)) << 1 + value == 0 ? 10 : value + end + end +end diff --git a/test/prism/integer_parse_test.rb b/test/prism/result/integer_parse_test.rb index f42e817e79..7b5ce98bb6 100644 --- a/test/prism/integer_parse_test.rb +++ b/test/prism/result/integer_parse_test.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" - -return if Prism::BACKEND == :FFI +require_relative "../test_helper" module Prism class IntegerParseTest < TestCase @@ -37,9 +35,7 @@ module Prism private def assert_integer_parse(expected, source = expected.to_s) - integer, string = Debug.integer_parse(source) - assert_equal expected, integer - assert_equal expected.to_s, string + assert_equal expected, Prism.parse_statement(source).value end end end diff --git a/test/prism/result/numeric_value_test.rb b/test/prism/result/numeric_value_test.rb new file mode 100644 index 0000000000..5c89230a1f --- /dev/null +++ b/test/prism/result/numeric_value_test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class NumericValueTest < TestCase + def test_numeric_value + assert_equal 123, Prism.parse_statement("123").value + assert_equal 3.14, Prism.parse_statement("3.14").value + assert_equal 42i, Prism.parse_statement("42i").value + assert_equal 42.1ri, Prism.parse_statement("42.1ri").value + assert_equal 3.14i, Prism.parse_statement("3.14i").value + assert_equal 42r, Prism.parse_statement("42r").value + assert_equal 0.5r, Prism.parse_statement("0.5r").value + assert_equal 42ri, Prism.parse_statement("42ri").value + assert_equal 0.5ri, Prism.parse_statement("0.5ri").value + assert_equal 0xFFr, Prism.parse_statement("0xFFr").value + assert_equal 0xFFri, Prism.parse_statement("0xFFri").value + end + end +end diff --git a/test/prism/result/overlap_test.rb b/test/prism/result/overlap_test.rb new file mode 100644 index 0000000000..155bc870d3 --- /dev/null +++ b/test/prism/result/overlap_test.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class OverlapTest < TestCase + Fixture.each do |fixture| + define_method(fixture.test_name) { assert_overlap(fixture) } + end + + private + + # Check that the location ranges of each node in the tree are a superset of + # their respective child nodes. + def assert_overlap(fixture) + queue = [Prism.parse_file(fixture.full_path).value] + + while (current = queue.shift) + # We only want to compare parent/child location overlap in the case that + # we are not looking at a heredoc. That's because heredoc locations are + # special in that they only use the declaration of the heredoc. + compare = !(current.is_a?(StringNode) || + current.is_a?(XStringNode) || + current.is_a?(InterpolatedStringNode) || + current.is_a?(InterpolatedXStringNode)) || + !current.opening&.start_with?("<<") + + current.child_nodes.each do |child| + # child_nodes can return nil values, so we need to skip those. + next unless child + + # Now that we know we have a child node, add that to the queue. + queue << child + + if compare + assert_operator current.location.start_offset, :<=, child.location.start_offset + assert_operator current.location.end_offset, :>=, child.location.end_offset + end + end + end + end + end +end diff --git a/test/prism/redundant_return_test.rb b/test/prism/result/redundant_return_test.rb index c668169245..3b20aeba00 100644 --- a/test/prism/redundant_return_test.rb +++ b/test/prism/result/redundant_return_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class RedundantReturnTest < TestCase diff --git a/test/prism/result/regular_expression_options_test.rb b/test/prism/result/regular_expression_options_test.rb new file mode 100644 index 0000000000..ff6e20526f --- /dev/null +++ b/test/prism/result/regular_expression_options_test.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class RegularExpressionOptionsTest < TestCase + def test_options + assert_equal "", Prism.parse_statement("__FILE__").filepath + assert_equal "foo.rb", Prism.parse_statement("__FILE__", filepath: "foo.rb").filepath + + assert_equal 1, Prism.parse_statement("foo").location.start_line + assert_equal 10, Prism.parse_statement("foo", line: 10).location.start_line + + refute Prism.parse_statement("\"foo\"").frozen? + assert Prism.parse_statement("\"foo\"", frozen_string_literal: true).frozen? + refute Prism.parse_statement("\"foo\"", frozen_string_literal: false).frozen? + + assert_kind_of CallNode, Prism.parse_statement("foo") + assert_kind_of LocalVariableReadNode, Prism.parse_statement("foo", scopes: [[:foo]]) + assert_equal 1, Prism.parse_statement("foo", scopes: [[:foo], []]).depth + + assert_equal [:foo], Prism.parse("foo", scopes: [[:foo]]).value.locals + end + end +end diff --git a/test/prism/location_test.rb b/test/prism/result/source_location_test.rb index 0724995671..ca74b36e6f 100644 --- a/test/prism/location_test.rb +++ b/test/prism/result/source_location_test.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism - class LocationTest < TestCase + class SourceLocationTest < TestCase def test_AliasGlobalVariableNode assert_location(AliasGlobalVariableNode, "alias $foo $bar") end @@ -175,14 +175,6 @@ module Prism assert_location(CallNode, "foo bar baz") assert_location(CallNode, "foo bar('baz')") - - assert_location(CallNode, "-> { it }", 5...7, version: "3.3.0") do |node| - node.body.body.first - end - - assert_location(LocalVariableReadNode, "-> { it }", 5...7, version: "3.4.0") do |node| - node.body.body.first - end end def test_CallAndWriteNode @@ -543,6 +535,24 @@ module Prism assert_location(InterpolatedXStringNode, '`foo #{bar} baz`') end + def test_ItLocalVariableReadNode + assert_location(ItLocalVariableReadNode, "-> { it }", 5...7) do |node| + node.body.body.first + end + + assert_location(ItLocalVariableReadNode, "foo { it }", 6...8) do |node| + node.block.body.body.first + end + + assert_location(CallNode, "-> { it }", 5...7, version: "3.3.0") do |node| + node.body.body.first + end + + assert_location(ItLocalVariableReadNode, "-> { it }", 5...7, version: "3.4.0") do |node| + node.body.body.first + end + end + def test_ItParametersNode assert_location(ItParametersNode, "-> { it }", &:parameters) end @@ -583,12 +593,6 @@ module Prism def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) - assert_location(LocalVariableReadNode, "-> { it }", 5...7) do |node| - node.body.body.first - end - assert_location(LocalVariableReadNode, "foo { it }", 6...8) do |node| - node.block.body.body.first - end end def test_LocalVariableTargetNode @@ -917,7 +921,7 @@ module Prism def test_all_tested expected = Prism.constants.grep(/.Node$/).sort - %i[MissingNode ProgramNode] - actual = LocationTest.instance_methods(false).grep(/.Node$/).map { |name| name[5..].to_sym }.sort + actual = SourceLocationTest.instance_methods(false).grep(/.Node$/).map { |name| name[5..].to_sym }.sort assert_equal expected, actual end diff --git a/test/prism/static_inspect_test.rb b/test/prism/result/static_inspect_test.rb index 8df2fd241e..cf8cef3298 100644 --- a/test/prism/static_inspect_test.rb +++ b/test/prism/result/static_inspect_test.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" - -return if Prism::BACKEND == :FFI +require_relative "../test_helper" module Prism class StaticInspectTest < TestCase @@ -84,7 +82,8 @@ module Prism private def static_inspect(source, **options) - Debug.static_inspect(source, **options) + warnings = Prism.parse("{ #{source} => 1, #{source} => 1 }", **options).warnings + warnings.last.message[/^key (.+) is duplicated and overwritten on line \d/, 1] end end end diff --git a/test/prism/static_literals_test.rb b/test/prism/result/static_literals_test.rb index 31c802bf90..dcfc692897 100644 --- a/test/prism/static_literals_test.rb +++ b/test/prism/result/static_literals_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class StaticLiteralsTest < TestCase diff --git a/test/prism/warnings_test.rb b/test/prism/result/warnings_test.rb index d01db01a0e..ea062d4221 100644 --- a/test/prism/warnings_test.rb +++ b/test/prism/result/warnings_test.rb @@ -2,8 +2,7 @@ return if RUBY_VERSION < "3.1" -require_relative "test_helper" -require "stringio" +require_relative "../test_helper" module Prism class WarningsTest < TestCase @@ -23,6 +22,23 @@ module Prism assert_warning("a /b/", "wrap regexp in parentheses") end + def test_binary_operator + [ + [:**, "argument prefix"], + [:*, "argument prefix"], + [:<<, "here document"], + [:&, "argument prefix"], + [:+, "unary operator"], + [:-, "unary operator"], + [:/, "regexp literal"], + [:%, "string literal"] + ].each do |(operator, warning)| + assert_warning("puts 1 #{operator}0", warning) + assert_warning("puts :a #{operator}0", warning) + assert_warning("m = 1; puts m #{operator}0", warning) + end + end + def test_equal_in_conditional assert_warning("if a = 1; end; a = a", "should be ==") end @@ -48,7 +64,7 @@ module Prism end def test_duplicated_when_clause - assert_warning("case 1; when 1, 1; end", "clause with line") + assert_warning("case 1; when 1, 1; end", "when' clause") end def test_float_out_of_range @@ -64,6 +80,15 @@ module Prism assert_warning("if true\nelsif\nfalse; end", "end of line") end + def test_shareable_constant_value + assert_warning("foo # shareable_constant_value: none", "ignored") + assert_warning("\v # shareable_constant_value: none", "ignored") + + refute_warning("# shareable_constant_value: none") + refute_warning(" # shareable_constant_value: none") + refute_warning("\t\t# shareable_constant_value: none") + end + def test_string_in_predicate assert_warning("if 'foo'; end", "string") assert_warning("if \"\#{foo}\"; end", "string") diff --git a/test/prism/compiler_test.rb b/test/prism/ruby/compiler_test.rb index 9a326eb8d6..35ccfd5950 100644 --- a/test/prism/compiler_test.rb +++ b/test/prism/ruby/compiler_test.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # typed: ignore -require_relative "test_helper" +require_relative "../test_helper" module Prism class CompilerTest < TestCase diff --git a/test/prism/desugar_compiler_test.rb b/test/prism/ruby/desugar_compiler_test.rb index 1a1d580d2d..fe9a25e030 100644 --- a/test/prism/desugar_compiler_test.rb +++ b/test/prism/ruby/desugar_compiler_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class DesugarCompilerTest < TestCase diff --git a/test/prism/dispatcher_test.rb b/test/prism/ruby/dispatcher_test.rb index 0d8a6d35e9..1b6d7f4117 100644 --- a/test/prism/dispatcher_test.rb +++ b/test/prism/ruby/dispatcher_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class DispatcherTest < TestCase diff --git a/test/prism/ruby/location_test.rb b/test/prism/ruby/location_test.rb new file mode 100644 index 0000000000..fc80a5b875 --- /dev/null +++ b/test/prism/ruby/location_test.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class LocationTest < TestCase + def test_join + call = Prism.parse_statement("1234 + 567") + receiver = call.receiver + argument = call.arguments.arguments.first + + joined = receiver.location.join(argument.location) + assert_equal 0, joined.start_offset + assert_equal 10, joined.length + + assert_raise(RuntimeError, "Incompatible locations") do + argument.location.join(receiver.location) + end + + other_argument = Prism.parse_statement("1234 + 567").arguments.arguments.first + + assert_raise(RuntimeError, "Incompatible sources") do + other_argument.location.join(receiver.location) + end + + assert_raise(RuntimeError, "Incompatible sources") do + receiver.location.join(other_argument.location) + end + end + + def test_character_offsets + program = Prism.parse("😀 + 😀\n😍 ||= 😍").value + + # first 😀 + location = program.statements.body.first.receiver.location + assert_equal 0, location.start_character_offset + assert_equal 1, location.end_character_offset + assert_equal 0, location.start_character_column + assert_equal 1, location.end_character_column + + # second 😀 + location = program.statements.body.first.arguments.arguments.first.location + assert_equal 4, location.start_character_offset + assert_equal 5, location.end_character_offset + assert_equal 4, location.start_character_column + assert_equal 5, location.end_character_column + + # first 😍 + location = program.statements.body.last.name_loc + assert_equal 6, location.start_character_offset + assert_equal 7, location.end_character_offset + assert_equal 0, location.start_character_column + assert_equal 1, location.end_character_column + + # second 😍 + location = program.statements.body.last.value.location + assert_equal 12, location.start_character_offset + assert_equal 13, location.end_character_offset + assert_equal 6, location.start_character_column + assert_equal 7, location.end_character_column + end + + def test_code_units + program = Prism.parse("😀 + 😀\n😍 ||= 😍").value + + # first 😀 + location = program.statements.body.first.receiver.location + + assert_equal 0, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 0, location.start_code_units_offset(Encoding::UTF_16LE) + assert_equal 0, location.start_code_units_offset(Encoding::UTF_32LE) + + assert_equal 1, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 2, location.end_code_units_offset(Encoding::UTF_16LE) + assert_equal 1, location.end_code_units_offset(Encoding::UTF_32LE) + + assert_equal 0, location.start_code_units_column(Encoding::UTF_8) + assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) + assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) + + assert_equal 1, location.end_code_units_column(Encoding::UTF_8) + assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) + assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) + + # second 😀 + location = program.statements.body.first.arguments.arguments.first.location + + assert_equal 4, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 5, location.start_code_units_offset(Encoding::UTF_16LE) + assert_equal 4, location.start_code_units_offset(Encoding::UTF_32LE) + + assert_equal 5, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 7, location.end_code_units_offset(Encoding::UTF_16LE) + assert_equal 5, location.end_code_units_offset(Encoding::UTF_32LE) + + assert_equal 4, location.start_code_units_column(Encoding::UTF_8) + assert_equal 5, location.start_code_units_column(Encoding::UTF_16LE) + assert_equal 4, location.start_code_units_column(Encoding::UTF_32LE) + + assert_equal 5, location.end_code_units_column(Encoding::UTF_8) + assert_equal 7, location.end_code_units_column(Encoding::UTF_16LE) + assert_equal 5, location.end_code_units_column(Encoding::UTF_32LE) + + # first 😍 + location = program.statements.body.last.name_loc + + assert_equal 6, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 8, location.start_code_units_offset(Encoding::UTF_16LE) + assert_equal 6, location.start_code_units_offset(Encoding::UTF_32LE) + + assert_equal 7, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 10, location.end_code_units_offset(Encoding::UTF_16LE) + assert_equal 7, location.end_code_units_offset(Encoding::UTF_32LE) + + assert_equal 0, location.start_code_units_column(Encoding::UTF_8) + assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) + assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) + + assert_equal 1, location.end_code_units_column(Encoding::UTF_8) + assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) + assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) + + # second 😍 + location = program.statements.body.last.value.location + + assert_equal 12, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 15, location.start_code_units_offset(Encoding::UTF_16LE) + assert_equal 12, location.start_code_units_offset(Encoding::UTF_32LE) + + assert_equal 13, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 17, location.end_code_units_offset(Encoding::UTF_16LE) + assert_equal 13, location.end_code_units_offset(Encoding::UTF_32LE) + + assert_equal 6, location.start_code_units_column(Encoding::UTF_8) + assert_equal 7, location.start_code_units_column(Encoding::UTF_16LE) + assert_equal 6, location.start_code_units_column(Encoding::UTF_32LE) + + assert_equal 7, location.end_code_units_column(Encoding::UTF_8) + assert_equal 9, location.end_code_units_column(Encoding::UTF_16LE) + assert_equal 7, location.end_code_units_column(Encoding::UTF_32LE) + end + + def test_chop + location = Prism.parse("foo").value.location + + assert_equal "fo", location.chop.slice + assert_equal "", location.chop.chop.chop.slice + + # Check that we don't go negative. + 10.times { location = location.chop } + assert_equal "", location.slice + end + + def test_slice_lines + method = Prism.parse_statement("\nprivate def foo\nend\n").arguments.arguments.first + + assert_equal "private def foo\nend\n", method.slice_lines + end + + def test_adjoin + program = Prism.parse("foo.bar = 1").value + + location = program.statements.body.first.message_loc + adjoined = location.adjoin("=") + + assert_kind_of Location, adjoined + refute_equal location, adjoined + + assert_equal 4, adjoined.start_offset + assert_equal 9, adjoined.end_offset + end + end +end diff --git a/test/prism/parameters_signature_test.rb b/test/prism/ruby/parameters_signature_test.rb index 0eed8d993d..9256bcc070 100644 --- a/test/prism/parameters_signature_test.rb +++ b/test/prism/ruby/parameters_signature_test.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require_relative "test_helper" - return if RUBY_VERSION < "3.2" +require_relative "../test_helper" + module Prism class ParametersSignatureTest < TestCase def test_req @@ -56,7 +56,6 @@ module Prism def test_key_ordering omit("TruffleRuby returns keys in order they were declared") if RUBY_ENGINE == "truffleruby" - assert_parameters([[:keyreq, :a], [:keyreq, :b], [:key, :c], [:key, :d]], "a:, c: 1, b:, d: 2") end @@ -75,14 +74,13 @@ module Prism private def assert_parameters(expected, source) - eval("def self.m(#{source}); end") - - begin - assert_equal(expected, method(:m).parameters) - assert_equal(expected, signature(source)) - ensure - singleton_class.undef_method(:m) - end + # Compare against our expectation. + assert_equal(expected, signature(source)) + + # Compare against Ruby's expectation. + object = Object.new + eval("def object.m(#{source}); end") + assert_equal(expected, object.method(:m).parameters) end def signature(source) diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb new file mode 100644 index 0000000000..65535af0fd --- /dev/null +++ b/test/prism/ruby/parser_test.rb @@ -0,0 +1,291 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +begin + verbose, $VERBOSE = $VERBOSE, nil + require "parser/ruby33" + require "prism/translation/parser33" +rescue LoadError + # In CRuby's CI, we're not going to test against the parser gem because we + # don't want to have to install it. So in this case we'll just skip this test. + return +ensure + $VERBOSE = verbose +end + +# First, opt in to every AST feature. +Parser::Builders::Default.modernize + +# Modify the source map == check so that it doesn't check against the node +# itself so we don't get into a recursive loop. +Parser::Source::Map.prepend( + Module.new { + def ==(other) + self.class == other.class && + (instance_variables - %i[@node]).map do |ivar| + instance_variable_get(ivar) == other.instance_variable_get(ivar) + end.reduce(:&) + end + } +) + +# Next, ensure that we're comparing the nodes and also comparing the source +# ranges so that we're getting all of the necessary information. +Parser::AST::Node.prepend( + Module.new { + def ==(other) + super && (location == other.location) + end + } +) + +module Prism + class ParserTest < TestCase + # These files contain code that is being parsed incorrectly by the parser + # gem, and therefore we don't want to compare against our translation. + skip_incorrect = [ + # https://github.com/whitequark/parser/issues/1017 + "spanning_heredoc.txt", + "spanning_heredoc_newlines.txt", + + # https://github.com/whitequark/parser/issues/1021 + "seattlerb/heredoc_nested.txt", + + # https://github.com/whitequark/parser/issues/1016 + "whitequark/unary_num_pow_precedence.txt" + ] + + # These files are either failing to parse or failing to translate, so we'll + # skip them for now. + skip_all = skip_incorrect | [ + "regex.txt", + "regex_char_width.txt", + "unescaping.txt", + "seattlerb/bug190.txt", + "seattlerb/heredoc_with_extra_carriage_returns_windows.txt", + "seattlerb/heredoc_with_only_carriage_returns_windows.txt", + "seattlerb/heredoc_with_only_carriage_returns.txt", + "seattlerb/parse_line_heredoc_hardnewline.txt", + "seattlerb/pctW_lineno.txt", + "seattlerb/regexp_esc_C_slash.txt", + "unparser/corpus/literal/literal.txt", + "unparser/corpus/semantic/dstr.txt", + "whitequark/dedenting_interpolating_heredoc_fake_line_continuation.txt", + "whitequark/parser_slash_slash_n_escaping_in_literals.txt", + "whitequark/ruby_bug_11989.txt" + ] + + # Not sure why these files are failing on JRuby, but skipping them for now. + if RUBY_ENGINE == "jruby" + skip_all.push("emoji_method_calls.txt", "symbols.txt") + end + + # These files are failing to translate their lexer output into the lexer + # output expected by the parser gem, so we'll skip them for now. + skip_tokens = [ + "comments.txt", + "dash_heredocs.txt", + "dos_endings.txt", + "embdoc_no_newline_at_end.txt", + "heredoc_with_comment.txt", + "heredocs_with_ignored_newlines.txt", + "indented_file_end.txt", + "methods.txt", + "strings.txt", + "tilde_heredocs.txt", + "xstring_with_backslash.txt", + "seattlerb/backticks_interpolation_line.txt", + "seattlerb/bug169.txt", + "seattlerb/case_in.txt", + "seattlerb/class_comments.txt", + "seattlerb/difficult4__leading_dots2.txt", + "seattlerb/difficult6__7.txt", + "seattlerb/difficult6__8.txt", + "seattlerb/dsym_esc_to_sym.txt", + "seattlerb/heredoc__backslash_dos_format.txt", + "seattlerb/heredoc_backslash_nl.txt", + "seattlerb/heredoc_comma_arg.txt", + "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt", + "seattlerb/heredoc_squiggly_blank_lines.txt", + "seattlerb/heredoc_squiggly_interp.txt", + "seattlerb/heredoc_squiggly_tabs_extra.txt", + "seattlerb/heredoc_squiggly_tabs.txt", + "seattlerb/heredoc_squiggly_visually_blank_lines.txt", + "seattlerb/heredoc_squiggly.txt", + "seattlerb/heredoc_unicode.txt", + "seattlerb/heredoc_with_carriage_return_escapes_windows.txt", + "seattlerb/heredoc_with_carriage_return_escapes.txt", + "seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt", + "seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt", + "seattlerb/interpolated_symbol_array_line_breaks.txt", + "seattlerb/interpolated_word_array_line_breaks.txt", + "seattlerb/label_vs_string.txt", + "seattlerb/module_comments.txt", + "seattlerb/non_interpolated_symbol_array_line_breaks.txt", + "seattlerb/non_interpolated_word_array_line_breaks.txt", + "seattlerb/parse_line_block_inline_comment_leading_newlines.txt", + "seattlerb/parse_line_block_inline_comment.txt", + "seattlerb/parse_line_block_inline_multiline_comment.txt", + "seattlerb/parse_line_dstr_escaped_newline.txt", + "seattlerb/parse_line_heredoc.txt", + "seattlerb/parse_line_multiline_str_literal_n.txt", + "seattlerb/parse_line_str_with_newline_escape.txt", + "seattlerb/pct_Q_backslash_nl.txt", + "seattlerb/pct_w_heredoc_interp_nested.txt", + "seattlerb/qsymbols_empty_space.txt", + "seattlerb/qw_escape_term.txt", + "seattlerb/qWords_space.txt", + "seattlerb/read_escape_unicode_curlies.txt", + "seattlerb/read_escape_unicode_h4.txt", + "seattlerb/required_kwarg_no_value.txt", + "seattlerb/slashy_newlines_within_string.txt", + "seattlerb/str_double_escaped_newline.txt", + "seattlerb/str_double_newline.txt", + "seattlerb/str_evstr_escape.txt", + "seattlerb/str_newline_hash_line_number.txt", + "seattlerb/str_single_newline.txt", + "seattlerb/symbol_empty.txt", + "seattlerb/symbols_empty_space.txt", + "seattlerb/TestRubyParserShared.txt", + "unparser/corpus/literal/assignment.txt", + "unparser/corpus/literal/dstr.txt", + "unparser/corpus/semantic/opasgn.txt", + "whitequark/args.txt", + "whitequark/beginless_erange_after_newline.txt", + "whitequark/beginless_irange_after_newline.txt", + "whitequark/bug_ascii_8bit_in_literal.txt", + "whitequark/bug_def_no_paren_eql_begin.txt", + "whitequark/dedenting_heredoc.txt", + "whitequark/dedenting_non_interpolating_heredoc_line_continuation.txt", + "whitequark/forward_arg_with_open_args.txt", + "whitequark/interp_digit_var.txt", + "whitequark/lbrace_arg_after_command_args.txt", + "whitequark/multiple_pattern_matches.txt", + "whitequark/newline_in_hash_argument.txt", + "whitequark/parser_bug_640.txt", + "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt", + "whitequark/ruby_bug_11990.txt", + "whitequark/ruby_bug_14690.txt", + "whitequark/ruby_bug_9669.txt", + "whitequark/slash_newline_in_heredocs.txt", + "whitequark/space_args_arg_block.txt", + "whitequark/space_args_block.txt" + ] + + Fixture.each do |fixture| + define_method(fixture.test_name) do + assert_equal_parses( + fixture, + compare_asts: !skip_all.include?(fixture.path), + compare_tokens: !skip_tokens.include?(fixture.path), + compare_comments: fixture.path != "embdoc_no_newline_at_end.txt" + ) + end + end + + private + + def assert_equal_parses(fixture, compare_asts: true, compare_tokens: true, compare_comments: true) + buffer = Parser::Source::Buffer.new(fixture.path, 1) + buffer.source = fixture.read + + parser = Parser::Ruby33.new + parser.diagnostics.consumer = ->(*) {} + parser.diagnostics.all_errors_are_fatal = true + + expected_ast, expected_comments, expected_tokens = + begin + ignore_warnings { parser.tokenize(buffer) } + rescue ArgumentError, Parser::SyntaxError + return + end + + actual_ast, actual_comments, actual_tokens = + ignore_warnings { Prism::Translation::Parser33.new.tokenize(buffer) } + + if expected_ast == actual_ast + if !compare_asts + puts "#{fixture.path} is now passing" + end + + assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) } + assert_equal_tokens(expected_tokens, actual_tokens) if compare_tokens + assert_equal_comments(expected_comments, actual_comments) if compare_comments + elsif compare_asts + assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) } + end + end + + def assert_equal_asts_message(expected_ast, actual_ast) + queue = [[expected_ast, actual_ast]] + + while (left, right = queue.shift) + if left.type != right.type + return "expected: #{left.type}\nactual: #{right.type}" + end + + if left.location != right.location + return "expected:\n#{left.inspect}\n#{left.location.inspect}\nactual:\n#{right.inspect}\n#{right.location.inspect}" + end + + if left.type == :str && left.children[0] != right.children[0] + return "expected: #{left.inspect}\nactual: #{right.inspect}" + end + + left.children.zip(right.children).each do |left_child, right_child| + queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node) + end + end + + "expected: #{expected_ast.inspect}\nactual: #{actual_ast.inspect}" + end + + def assert_equal_tokens(expected_tokens, actual_tokens) + if expected_tokens != actual_tokens + expected_index = 0 + actual_index = 0 + + while expected_index < expected_tokens.length + expected_token = expected_tokens[expected_index] + actual_token = actual_tokens[actual_index] + + expected_index += 1 + actual_index += 1 + + # The parser gem always has a space before a string end in list + # literals, but we don't. So we'll skip over the space. + if expected_token[0] == :tSPACE && actual_token[0] == :tSTRING_END + expected_index += 1 + next + end + + # There are a lot of tokens that have very specific meaning according + # to the context of the parser. We don't expose that information in + # prism, so we need to normalize these tokens a bit. + case actual_token[0] + when :kDO + actual_token[0] = expected_token[0] if %i[kDO_BLOCK kDO_LAMBDA].include?(expected_token[0]) + when :tLPAREN + actual_token[0] = expected_token[0] if expected_token[0] == :tLPAREN2 + when :tPOW + actual_token[0] = expected_token[0] if expected_token[0] == :tDSTAR + end + + # Now we can assert that the tokens are actually equal. + assert_equal expected_token, actual_token, -> { + "expected: #{expected_token.inspect}\n" \ + "actual: #{actual_token.inspect}" + } + end + end + end + + def assert_equal_comments(expected_comments, actual_comments) + assert_equal expected_comments, actual_comments, -> { + "expected: #{expected_comments.inspect}\n" \ + "actual: #{actual_comments.inspect}" + } + end + end +end diff --git a/test/prism/pattern_test.rb b/test/prism/ruby/pattern_test.rb index e0aa079cb9..23f512fc1c 100644 --- a/test/prism/pattern_test.rb +++ b/test/prism/ruby/pattern_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class PatternTest < TestCase diff --git a/test/prism/reflection_test.rb b/test/prism/ruby/reflection_test.rb index 869b68b1f8..3ac462e1ac 100644 --- a/test/prism/reflection_test.rb +++ b/test/prism/ruby/reflection_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "test_helper" +require_relative "../test_helper" module Prism class ReflectionTest < TestCase diff --git a/test/prism/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 07238fc3d5..8db47da3d3 100644 --- a/test/prism/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -2,13 +2,11 @@ return if RUBY_VERSION < "3.3" -require_relative "test_helper" +require_relative "../test_helper" module Prism class RipperTest < TestCase - base = File.join(__dir__, "fixtures") - relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] - + # Skip these tests that Ripper is reporting the wrong results for. incorrect = [ # Ripper incorrectly attributes the block to the keyword. "seattlerb/block_break.txt", @@ -31,6 +29,7 @@ module Prism "spanning_heredoc.txt" ] + # Skip these tests that we haven't implemented yet. omitted = [ "dos_endings.txt", "heredocs_with_ignored_newlines.txt", @@ -50,30 +49,8 @@ module Prism "whitequark/slash_newline_in_heredocs.txt" ] - relatives.each do |relative| - # Skip the tests that Ripper is reporting the wrong results for. - next if incorrect.include?(relative) - - # Skip the tests we haven't implemented yet. - next if omitted.include?(relative) - - filepath = File.join(__dir__, "fixtures", relative) - - define_method "test_ripper_#{relative}" do - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - - case relative - when /break|next|redo|if|unless|rescue|control|keywords|retry/ - source = "-> do\nrescue\n#{source}\nend" - end - - case source - when /^ *yield/ - source = "def __invalid_yield__\n#{source}\nend" - end - - assert_ripper(source) - end + Fixture.each(except: incorrect | omitted) do |fixture| + define_method(fixture.test_name) { assert_ripper(fixture.read) } end private diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb new file mode 100644 index 0000000000..a13daeeb84 --- /dev/null +++ b/test/prism/ruby/ruby_parser_test.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +return if RUBY_ENGINE == "jruby" + +require_relative "../test_helper" + +begin + require "ruby_parser" +rescue LoadError + # In CRuby's CI, we're not going to test against the ruby_parser gem because + # we don't want to have to install it. So in this case we'll just skip this + # test. + return +end + +# We want to also compare lines and files to make sure we're setting them +# correctly. +Sexp.prepend( + Module.new do + def ==(other) + super && line == other.line && file == other.file # && line_max == other.line_max + end + end +) + +module Prism + class RubyParserTest < TestCase + todos = [ + "newline_terminated.txt", + "regex_char_width.txt", + "seattlerb/bug169.txt", + "seattlerb/masgn_colon3.txt", + "seattlerb/messy_op_asgn_lineno.txt", + "seattlerb/op_asgn_primary_colon_const_command_call.txt", + "seattlerb/regexp_esc_C_slash.txt", + "seattlerb/str_lit_concat_bad_encodings.txt", + "unescaping.txt", + "unparser/corpus/literal/kwbegin.txt", + "unparser/corpus/literal/send.txt", + "whitequark/masgn_const.txt", + "whitequark/ruby_bug_12402.txt", + "whitequark/ruby_bug_14690.txt", + "whitequark/space_args_block.txt" + ] + + # https://github.com/seattlerb/ruby_parser/issues/344 + failures = [ + "alias.txt", + "dos_endings.txt", + "heredocs_with_ignored_newlines.txt", + "method_calls.txt", + "methods.txt", + "multi_write.txt", + "not.txt", + "patterns.txt", + "regex.txt", + "seattlerb/and_multi.txt", + "seattlerb/heredoc__backslash_dos_format.txt", + "seattlerb/heredoc_bad_hex_escape.txt", + "seattlerb/heredoc_bad_oct_escape.txt", + "seattlerb/heredoc_with_extra_carriage_horrible_mix.txt", + "seattlerb/heredoc_with_extra_carriage_returns_windows.txt", + "seattlerb/heredoc_with_only_carriage_returns_windows.txt", + "seattlerb/heredoc_with_only_carriage_returns.txt", + "spanning_heredoc_newlines.txt", + "spanning_heredoc.txt", + "tilde_heredocs.txt", + "unparser/corpus/literal/literal.txt", + "while.txt", + "whitequark/cond_eflipflop.txt", + "whitequark/cond_iflipflop.txt", + "whitequark/cond_match_current_line.txt", + "whitequark/dedenting_heredoc.txt", + "whitequark/lvar_injecting_match.txt", + "whitequark/not.txt", + "whitequark/numparam_ruby_bug_19025.txt", + "whitequark/op_asgn_cmd.txt", + "whitequark/parser_bug_640.txt", + "whitequark/parser_slash_slash_n_escaping_in_literals.txt", + "whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt", + "whitequark/pattern_matching_single_line.txt", + "whitequark/ruby_bug_11989.txt", + "whitequark/slash_newline_in_heredocs.txt" + ] + + Fixture.each(except: failures) do |fixture| + define_method(fixture.test_name) do + assert_ruby_parser(fixture, todos.include?(fixture.path)) + end + end + + private + + def assert_ruby_parser(fixture, allowed_failure) + source = fixture.read + expected = ignore_warnings { ::RubyParser.new.parse(source, fixture.path) } + actual = Prism::Translation::RubyParser.new.parse(source, fixture.path) + + if !allowed_failure + assert_equal(expected, actual, -> { message(expected, actual) }) + elsif expected == actual + puts "#{name} now passes" + end + end + + def message(expected, actual) + if expected == actual + nil + elsif expected.is_a?(Sexp) && actual.is_a?(Sexp) + if expected.line != actual.line + "expected: (#{expected.inspect} line=#{expected.line}), actual: (#{actual.inspect} line=#{actual.line})" + elsif expected.file != actual.file + "expected: (#{expected.inspect} file=#{expected.file}), actual: (#{actual.inspect} file=#{actual.file})" + elsif expected.length != actual.length + "expected: (#{expected.inspect} length=#{expected.length}), actual: (#{actual.inspect} length=#{actual.length})" + else + expected.zip(actual).find do |expected_field, actual_field| + result = message(expected_field, actual_field) + break result if result + end + end + else + "expected: #{expected.inspect}, actual: #{actual.inspect}" + end + end + end +end diff --git a/test/prism/ruby/tunnel_test.rb b/test/prism/ruby/tunnel_test.rb new file mode 100644 index 0000000000..0214681604 --- /dev/null +++ b/test/prism/ruby/tunnel_test.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class TunnelTest < TestCase + def test_tunnel + program = Prism.parse("foo(1) +\n bar(2, 3) +\n baz(3, 4, 5)").value + + tunnel = program.tunnel(1, 4).last + assert_kind_of IntegerNode, tunnel + assert_equal 1, tunnel.value + + tunnel = program.tunnel(2, 6).last + assert_kind_of IntegerNode, tunnel + assert_equal 2, tunnel.value + + tunnel = program.tunnel(3, 9).last + assert_kind_of IntegerNode, tunnel + assert_equal 4, tunnel.value + + tunnel = program.tunnel(3, 8) + assert_equal [ProgramNode, StatementsNode, CallNode, ArgumentsNode, CallNode, ArgumentsNode], tunnel.map(&:class) + end + end +end diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb deleted file mode 100644 index a1e2592d3d..0000000000 --- a/test/prism/ruby_api_test.rb +++ /dev/null @@ -1,307 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -module Prism - class RubyAPITest < TestCase - if !ENV["PRISM_BUILD_MINIMAL"] - def test_ruby_api - filepath = __FILE__ - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - - assert_equal Prism.lex(source, filepath: filepath).value, Prism.lex_file(filepath).value - assert_equal Prism.dump(source, filepath: filepath), Prism.dump_file(filepath) - - serialized = Prism.dump(source, filepath: filepath) - ast1 = Prism.load(source, serialized).value - ast2 = Prism.parse(source, filepath: filepath).value - ast3 = Prism.parse_file(filepath).value - - assert_equal_nodes ast1, ast2 - assert_equal_nodes ast2, ast3 - end - end - - def test_parse_success? - assert Prism.parse_success?("1") - refute Prism.parse_success?("<>") - end - - def test_parse_file_success? - assert Prism.parse_file_success?(__FILE__) - end - - def test_options - assert_equal "", Prism.parse("__FILE__").value.statements.body[0].filepath - assert_equal "foo.rb", Prism.parse("__FILE__", filepath: "foo.rb").value.statements.body[0].filepath - - assert_equal 1, Prism.parse("foo").value.statements.body[0].location.start_line - assert_equal 10, Prism.parse("foo", line: 10).value.statements.body[0].location.start_line - - refute Prism.parse("\"foo\"").value.statements.body[0].frozen? - assert Prism.parse("\"foo\"", frozen_string_literal: true).value.statements.body[0].frozen? - refute Prism.parse("\"foo\"", frozen_string_literal: false).value.statements.body[0].frozen? - - assert_kind_of Prism::CallNode, Prism.parse("foo").value.statements.body[0] - assert_kind_of Prism::LocalVariableReadNode, Prism.parse("foo", scopes: [[:foo]]).value.statements.body[0] - assert_equal 1, Prism.parse("foo", scopes: [[:foo], []]).value.statements.body[0].depth - - assert_equal [:foo], Prism.parse("foo", scopes: [[:foo]]).value.locals - end - - def test_literal_value_method - assert_equal 123, parse_expression("123").value - assert_equal 3.14, parse_expression("3.14").value - assert_equal 42i, parse_expression("42i").value - assert_equal 42.1ri, parse_expression("42.1ri").value - assert_equal 3.14i, parse_expression("3.14i").value - assert_equal 42r, parse_expression("42r").value - assert_equal 0.5r, parse_expression("0.5r").value - assert_equal 42ri, parse_expression("42ri").value - assert_equal 0.5ri, parse_expression("0.5ri").value - assert_equal 0xFFr, parse_expression("0xFFr").value - assert_equal 0xFFri, parse_expression("0xFFri").value - end - - def test_location_join - recv, args_node, _ = parse_expression("1234 + 567").child_nodes - arg = args_node.arguments[0] - - joined = recv.location.join(arg.location) - assert_equal 0, joined.start_offset - assert_equal 10, joined.length - - assert_raise RuntimeError, "Incompatible locations" do - arg.location.join(recv.location) - end - - other_arg = parse_expression("1234 + 567").arguments.arguments[0] - - assert_raise RuntimeError, "Incompatible sources" do - other_arg.location.join(recv.location) - end - - assert_raise RuntimeError, "Incompatible sources" do - recv.location.join(other_arg.location) - end - end - - def test_location_character_offsets - program = Prism.parse("😀 + 😀\n😍 ||= 😍").value - - # first 😀 - location = program.statements.body.first.receiver.location - assert_equal 0, location.start_character_offset - assert_equal 1, location.end_character_offset - assert_equal 0, location.start_character_column - assert_equal 1, location.end_character_column - - # second 😀 - location = program.statements.body.first.arguments.arguments.first.location - assert_equal 4, location.start_character_offset - assert_equal 5, location.end_character_offset - assert_equal 4, location.start_character_column - assert_equal 5, location.end_character_column - - # first 😍 - location = program.statements.body.last.name_loc - assert_equal 6, location.start_character_offset - assert_equal 7, location.end_character_offset - assert_equal 0, location.start_character_column - assert_equal 1, location.end_character_column - - # second 😍 - location = program.statements.body.last.value.location - assert_equal 12, location.start_character_offset - assert_equal 13, location.end_character_offset - assert_equal 6, location.start_character_column - assert_equal 7, location.end_character_column - end - - def test_location_code_units - program = Prism.parse("😀 + 😀\n😍 ||= 😍").value - - # first 😀 - location = program.statements.body.first.receiver.location - - assert_equal 0, location.start_code_units_offset(Encoding::UTF_8) - assert_equal 0, location.start_code_units_offset(Encoding::UTF_16LE) - assert_equal 0, location.start_code_units_offset(Encoding::UTF_32LE) - - assert_equal 1, location.end_code_units_offset(Encoding::UTF_8) - assert_equal 2, location.end_code_units_offset(Encoding::UTF_16LE) - assert_equal 1, location.end_code_units_offset(Encoding::UTF_32LE) - - assert_equal 0, location.start_code_units_column(Encoding::UTF_8) - assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) - assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) - - assert_equal 1, location.end_code_units_column(Encoding::UTF_8) - assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) - assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) - - # second 😀 - location = program.statements.body.first.arguments.arguments.first.location - - assert_equal 4, location.start_code_units_offset(Encoding::UTF_8) - assert_equal 5, location.start_code_units_offset(Encoding::UTF_16LE) - assert_equal 4, location.start_code_units_offset(Encoding::UTF_32LE) - - assert_equal 5, location.end_code_units_offset(Encoding::UTF_8) - assert_equal 7, location.end_code_units_offset(Encoding::UTF_16LE) - assert_equal 5, location.end_code_units_offset(Encoding::UTF_32LE) - - assert_equal 4, location.start_code_units_column(Encoding::UTF_8) - assert_equal 5, location.start_code_units_column(Encoding::UTF_16LE) - assert_equal 4, location.start_code_units_column(Encoding::UTF_32LE) - - assert_equal 5, location.end_code_units_column(Encoding::UTF_8) - assert_equal 7, location.end_code_units_column(Encoding::UTF_16LE) - assert_equal 5, location.end_code_units_column(Encoding::UTF_32LE) - - # first 😍 - location = program.statements.body.last.name_loc - - assert_equal 6, location.start_code_units_offset(Encoding::UTF_8) - assert_equal 8, location.start_code_units_offset(Encoding::UTF_16LE) - assert_equal 6, location.start_code_units_offset(Encoding::UTF_32LE) - - assert_equal 7, location.end_code_units_offset(Encoding::UTF_8) - assert_equal 10, location.end_code_units_offset(Encoding::UTF_16LE) - assert_equal 7, location.end_code_units_offset(Encoding::UTF_32LE) - - assert_equal 0, location.start_code_units_column(Encoding::UTF_8) - assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) - assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) - - assert_equal 1, location.end_code_units_column(Encoding::UTF_8) - assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) - assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) - - # second 😍 - location = program.statements.body.last.value.location - - assert_equal 12, location.start_code_units_offset(Encoding::UTF_8) - assert_equal 15, location.start_code_units_offset(Encoding::UTF_16LE) - assert_equal 12, location.start_code_units_offset(Encoding::UTF_32LE) - - assert_equal 13, location.end_code_units_offset(Encoding::UTF_8) - assert_equal 17, location.end_code_units_offset(Encoding::UTF_16LE) - assert_equal 13, location.end_code_units_offset(Encoding::UTF_32LE) - - assert_equal 6, location.start_code_units_column(Encoding::UTF_8) - assert_equal 7, location.start_code_units_column(Encoding::UTF_16LE) - assert_equal 6, location.start_code_units_column(Encoding::UTF_32LE) - - assert_equal 7, location.end_code_units_column(Encoding::UTF_8) - assert_equal 9, location.end_code_units_column(Encoding::UTF_16LE) - assert_equal 7, location.end_code_units_column(Encoding::UTF_32LE) - end - - def test_location_chop - location = Prism.parse("foo").value.location - - assert_equal "fo", location.chop.slice - assert_equal "", location.chop.chop.chop.slice - - # Check that we don't go negative. - 10.times { location = location.chop } - assert_equal "", location.slice - end - - def test_location_slice_lines - result = Prism.parse("\nprivate def foo\nend\n") - method = result.value.statements.body.first.arguments.arguments.first - - assert_equal "private def foo\nend\n", method.slice_lines - end - - def test_heredoc? - refute parse_expression("\"foo\"").heredoc? - refute parse_expression("\"foo \#{1}\"").heredoc? - refute parse_expression("`foo`").heredoc? - refute parse_expression("`foo \#{1}`").heredoc? - - assert parse_expression("<<~HERE\nfoo\nHERE\n").heredoc? - assert parse_expression("<<~HERE\nfoo \#{1}\nHERE\n").heredoc? - assert parse_expression("<<~`HERE`\nfoo\nHERE\n").heredoc? - assert parse_expression("<<~`HERE`\nfoo \#{1}\nHERE\n").heredoc? - end - - # Through some bit hackery, we want to allow consumers to use the integer - # base flags as the base itself. It has a nice property that the current - # alignment provides them in the correct order. So here we test that our - # assumption holds so that it doesn't change out from under us. - # - # In C, this would look something like: - # - # ((flags & ~DECIMAL) << 1) || 10 - # - # We have to do some other work in Ruby because 0 is truthy and ~ on an - # integer doesn't have a fixed width. - def test_integer_base_flags - base = -> (node) do - value = (node.send(:flags) & (0b1111 - IntegerBaseFlags::DECIMAL)) << 1 - value == 0 ? 10 : value - end - - assert_equal 2, base[parse_expression("0b1")] - assert_equal 8, base[parse_expression("0o1")] - assert_equal 10, base[parse_expression("0d1")] - assert_equal 16, base[parse_expression("0x1")] - end - - def test_node_equality - assert_operator parse_expression("1"), :===, parse_expression("1") - assert_operator Prism.parse("1").value, :===, Prism.parse("1").value - - complex_source = "class Something; @var = something.else { _1 }; end" - assert_operator parse_expression(complex_source), :===, parse_expression(complex_source) - - refute_operator parse_expression("1"), :===, parse_expression("2") - refute_operator parse_expression("1"), :===, parse_expression("0x1") - - complex_source_1 = "class Something; @var = something.else { _1 }; end" - complex_source_2 = "class Something; @var = something.else { _2 }; end" - refute_operator parse_expression(complex_source_1), :===, parse_expression(complex_source_2) - end - - def test_node_tunnel - program = Prism.parse("foo(1) +\n bar(2, 3) +\n baz(3, 4, 5)").value - - tunnel = program.tunnel(1, 4).last - assert_kind_of IntegerNode, tunnel - assert_equal 1, tunnel.value - - tunnel = program.tunnel(2, 6).last - assert_kind_of IntegerNode, tunnel - assert_equal 2, tunnel.value - - tunnel = program.tunnel(3, 9).last - assert_kind_of IntegerNode, tunnel - assert_equal 4, tunnel.value - - tunnel = program.tunnel(3, 8) - assert_equal [ProgramNode, StatementsNode, CallNode, ArgumentsNode, CallNode, ArgumentsNode], tunnel.map(&:class) - end - - def test_location_adjoin - program = Prism.parse("foo.bar = 1").value - - location = program.statements.body.first.message_loc - adjoined = location.adjoin("=") - - assert_kind_of Location, adjoined - refute_equal location, adjoined - - assert_equal 4, adjoined.start_offset - assert_equal 9, adjoined.end_offset - end - - private - - def parse_expression(source) - Prism.parse(source).value.statements.body.first - end - end -end diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb deleted file mode 100644 index 8edeac4b4f..0000000000 --- a/test/prism/ruby_parser_test.rb +++ /dev/null @@ -1,144 +0,0 @@ -# frozen_string_literal: true - -return if RUBY_ENGINE == "jruby" - -require_relative "test_helper" - -begin - require "ruby_parser" -rescue LoadError - # In CRuby's CI, we're not going to test against the ruby_parser gem because - # we don't want to have to install it. So in this case we'll just skip this - # test. - return -end - -# We want to also compare lines and files to make sure we're setting them -# correctly. -Sexp.prepend( - Module.new do - def ==(other) - super && line == other.line && line_max == other.line_max && file == other.file - end - end -) - -module Prism - class RubyParserTest < TestCase - base = File.join(__dir__, "fixtures") - - todos = %w[ - heredocs_nested.txt - newline_terminated.txt - regex_char_width.txt - seattlerb/bug169.txt - seattlerb/dstr_evstr.txt - seattlerb/heredoc_squiggly_interp.txt - seattlerb/masgn_colon3.txt - seattlerb/messy_op_asgn_lineno.txt - seattlerb/op_asgn_primary_colon_const_command_call.txt - seattlerb/parse_line_evstr_after_break.txt - seattlerb/regexp_esc_C_slash.txt - seattlerb/str_lit_concat_bad_encodings.txt - seattlerb/str_pct_nested_nested.txt - unescaping.txt - unparser/corpus/literal/kwbegin.txt - unparser/corpus/literal/send.txt - unparser/corpus/semantic/dstr.txt - whitequark/masgn_const.txt - whitequark/ruby_bug_12402.txt - whitequark/ruby_bug_14690.txt - whitequark/space_args_block.txt - whitequark/string_concat.txt - ] - - # https://github.com/seattlerb/ruby_parser/issues/344 - failures = %w[ - alias.txt - dos_endings.txt - heredocs_with_ignored_newlines.txt - method_calls.txt - methods.txt - multi_write.txt - not.txt - patterns.txt - regex.txt - seattlerb/and_multi.txt - seattlerb/heredoc__backslash_dos_format.txt - seattlerb/heredoc_bad_hex_escape.txt - seattlerb/heredoc_bad_oct_escape.txt - seattlerb/heredoc_with_extra_carriage_horrible_mix.txt - seattlerb/heredoc_with_extra_carriage_returns_windows.txt - seattlerb/heredoc_with_only_carriage_returns_windows.txt - seattlerb/heredoc_with_only_carriage_returns.txt - spanning_heredoc_newlines.txt - spanning_heredoc.txt - tilde_heredocs.txt - unparser/corpus/literal/literal.txt - while.txt - whitequark/class_definition_in_while_cond.txt - whitequark/cond_eflipflop.txt - whitequark/cond_iflipflop.txt - whitequark/cond_match_current_line.txt - whitequark/dedenting_heredoc.txt - whitequark/if_while_after_class__since_32.txt - whitequark/lvar_injecting_match.txt - whitequark/not.txt - whitequark/numparam_ruby_bug_19025.txt - whitequark/op_asgn_cmd.txt - whitequark/parser_bug_640.txt - whitequark/parser_slash_slash_n_escaping_in_literals.txt - whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt - whitequark/pattern_matching_single_line.txt - whitequark/ruby_bug_11989.txt - whitequark/slash_newline_in_heredocs.txt - ] - - Dir["**/*.txt", base: base].each do |name| - next if failures.include?(name) - - define_method("test_#{name}") do - begin - # Parsing with ruby parser tends to be noisy with warnings, so we're - # turning those off. - previous_verbose, $VERBOSE = $VERBOSE, nil - assert_parse_file(base, name, todos.include?(name)) - ensure - $VERBOSE = previous_verbose - end - end - end - - private - - def assert_parse_file(base, name, allowed_failure) - filepath = File.join(base, name) - expected = ::RubyParser.new.parse(File.read(filepath), filepath) - actual = Prism::Translation::RubyParser.parse_file(filepath) - - if !allowed_failure - assert_equal_nodes expected, actual - elsif expected == actual - puts "#{name} now passes" - end - end - - def assert_equal_nodes(left, right) - return if left == right - - if left.is_a?(Sexp) && right.is_a?(Sexp) - if left.line != right.line - assert_equal "(#{left.inspect} line=#{left.line})", "(#{right.inspect} line=#{right.line})" - elsif left.file != right.file - assert_equal "(#{left.inspect} file=#{left.file})", "(#{right.inspect} file=#{right.file})" - elsif left.length != right.length - assert_equal "(#{left.inspect} length=#{left.length})", "(#{right.inspect} length=#{right.length})" - else - left.zip(right).each { |l, r| assert_equal_nodes(l, r) } - end - else - assert_equal left, right - end - end - end -end diff --git a/test/prism/snapshots/arrays.txt b/test/prism/snapshots/arrays.txt index e8e53aacb9..90a4d8f3bb 100644 --- a/test/prism/snapshots/arrays.txt +++ b/test/prism/snapshots/arrays.txt @@ -960,8 +960,8 @@ │ ├── arguments: ∅ │ ├── closing_loc: (84,4)-(84,5) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (84,6)-(84,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (84,6)-(84,8) = "+=" │ └── value: │ @ IntegerNode (location: (84,9)-(84,10)) │ ├── flags: decimal @@ -1040,8 +1040,8 @@ │ ├── arguments: ∅ │ ├── closing_loc: (90,8)-(90,9) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (90,10)-(90,12) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (90,10)-(90,12) = "+=" │ └── value: │ @ IntegerNode (location: (90,13)-(90,14)) │ ├── flags: decimal @@ -1143,8 +1143,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (96,7)-(96,8) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (96,9)-(96,11) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (96,9)-(96,11) = "+=" │ └── value: │ @ IntegerNode (location: (96,12)-(96,13)) │ ├── flags: decimal @@ -1262,8 +1262,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (102,11)-(102,12) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (102,13)-(102,15) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (102,13)-(102,15) = "+=" │ └── value: │ @ IntegerNode (location: (102,16)-(102,17)) │ ├── flags: decimal @@ -1633,8 +1633,8 @@ │ │ │ └── expression: ∅ │ │ ├── closing_loc: (116,13)-(116,14) = "]" │ │ ├── block: ∅ - │ │ ├── operator: :+ - │ │ ├── operator_loc: (116,15)-(116,17) = "+=" + │ │ ├── binary_operator: :+ + │ │ ├── binary_operator_loc: (116,15)-(116,17) = "+=" │ │ └── value: │ │ @ IntegerNode (location: (116,18)-(116,19)) │ │ ├── flags: decimal diff --git a/test/prism/snapshots/blocks.txt b/test/prism/snapshots/blocks.txt index 0b1ec52e38..1c996ebd09 100644 --- a/test/prism/snapshots/blocks.txt +++ b/test/prism/snapshots/blocks.txt @@ -158,13 +158,13 @@ │ │ └── body: (length: 1) │ │ └── @ LocalVariableOperatorWriteNode (location: (7,24)-(7,33)) │ │ ├── name_loc: (7,24)-(7,28) = "memo" - │ │ ├── operator_loc: (7,29)-(7,31) = "+=" + │ │ ├── binary_operator_loc: (7,29)-(7,31) = "+=" │ │ ├── value: │ │ │ @ LocalVariableReadNode (location: (7,32)-(7,33)) │ │ │ ├── name: :x │ │ │ └── depth: 0 │ │ ├── name: :memo - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ ├── opening_loc: (7,12)-(7,13) = "{" │ └── closing_loc: (7,34)-(7,35) = "}" diff --git a/test/prism/snapshots/boolean_operators.txt b/test/prism/snapshots/boolean_operators.txt index ace8047e18..3bf33430c9 100644 --- a/test/prism/snapshots/boolean_operators.txt +++ b/test/prism/snapshots/boolean_operators.txt @@ -21,7 +21,7 @@ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,6)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ CallNode (location: (3,5)-(3,6)) │ │ ├── flags: variable_call, ignore_visibility @@ -34,7 +34,7 @@ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ LocalVariableOrWriteNode (location: (5,0)-(5,7)) ├── name_loc: (5,0)-(5,1) = "a" diff --git a/test/prism/snapshots/break.txt b/test/prism/snapshots/break.txt index c15a9e4675..7d5bf5e69a 100644 --- a/test/prism/snapshots/break.txt +++ b/test/prism/snapshots/break.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(25,23)) +@ ProgramNode (location: (1,0)-(29,21)) ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(25,23)) - └── body: (length: 11) + @ StatementsNode (location: (1,0)-(29,21)) + └── body: (length: 13) ├── @ CallNode (location: (1,0)-(1,13)) │ ├── flags: ignore_visibility │ ├── receiver: ∅ @@ -346,56 +346,102 @@ │ │ └── value: 42 │ ├── closing_loc: ∅ │ └── block: ∅ - └── @ CallNode (location: (25,0)-(25,23)) + ├── @ CallNode (location: (25,0)-(25,23)) + │ ├── flags: ∅ + │ ├── receiver: + │ │ @ CallNode (location: (25,0)-(25,17)) + │ │ ├── flags: ignore_visibility + │ │ ├── receiver: ∅ + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :foo + │ │ ├── message_loc: (25,0)-(25,3) = "foo" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: ∅ + │ │ ├── closing_loc: ∅ + │ │ └── block: + │ │ @ BlockNode (location: (25,4)-(25,17)) + │ │ ├── locals: [:a] + │ │ ├── parameters: + │ │ │ @ BlockParametersNode (location: (25,6)-(25,9)) + │ │ │ ├── parameters: + │ │ │ │ @ ParametersNode (location: (25,7)-(25,8)) + │ │ │ │ ├── requireds: (length: 1) + │ │ │ │ │ └── @ RequiredParameterNode (location: (25,7)-(25,8)) + │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ └── name: :a + │ │ │ │ ├── optionals: (length: 0) + │ │ │ │ ├── rest: ∅ + │ │ │ │ ├── posts: (length: 0) + │ │ │ │ ├── keywords: (length: 0) + │ │ │ │ ├── keyword_rest: ∅ + │ │ │ │ └── block: ∅ + │ │ │ ├── locals: (length: 0) + │ │ │ ├── opening_loc: (25,6)-(25,7) = "|" + │ │ │ └── closing_loc: (25,8)-(25,9) = "|" + │ │ ├── body: + │ │ │ @ StatementsNode (location: (25,10)-(25,15)) + │ │ │ └── body: (length: 1) + │ │ │ └── @ BreakNode (location: (25,10)-(25,15)) + │ │ │ ├── arguments: ∅ + │ │ │ └── keyword_loc: (25,10)-(25,15) = "break" + │ │ ├── opening_loc: (25,4)-(25,5) = "{" + │ │ └── closing_loc: (25,16)-(25,17) = "}" + │ ├── call_operator_loc: ∅ + │ ├── name: :== + │ ├── message_loc: (25,18)-(25,20) = "==" + │ ├── opening_loc: ∅ + │ ├── arguments: + │ │ @ ArgumentsNode (location: (25,21)-(25,23)) + │ │ ├── flags: ∅ + │ │ └── arguments: (length: 1) + │ │ └── @ IntegerNode (location: (25,21)-(25,23)) + │ │ ├── flags: decimal + │ │ └── value: 42 + │ ├── closing_loc: ∅ + │ └── block: ∅ + ├── @ WhileNode (location: (27,0)-(27,21)) + │ ├── flags: ∅ + │ ├── keyword_loc: (27,0)-(27,5) = "while" + │ ├── closing_loc: (27,18)-(27,21) = "end" + │ ├── predicate: + │ │ @ AndNode (location: (27,6)-(27,16)) + │ │ ├── left: + │ │ │ @ CallNode (location: (27,6)-(27,7)) + │ │ │ ├── flags: variable_call, ignore_visibility + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :_ + │ │ │ ├── message_loc: (27,6)-(27,7) = "_" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ └── block: ∅ + │ │ ├── right: + │ │ │ @ BreakNode (location: (27,11)-(27,16)) + │ │ │ ├── arguments: ∅ + │ │ │ └── keyword_loc: (27,11)-(27,16) = "break" + │ │ └── operator_loc: (27,8)-(27,10) = "&&" + │ └── statements: ∅ + └── @ UntilNode (location: (29,0)-(29,21)) ├── flags: ∅ - ├── receiver: - │ @ CallNode (location: (25,0)-(25,17)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :foo - │ ├── message_loc: (25,0)-(25,3) = "foo" - │ ├── opening_loc: ∅ - │ ├── arguments: ∅ - │ ├── closing_loc: ∅ - │ └── block: - │ @ BlockNode (location: (25,4)-(25,17)) - │ ├── locals: [:a] - │ ├── parameters: - │ │ @ BlockParametersNode (location: (25,6)-(25,9)) - │ │ ├── parameters: - │ │ │ @ ParametersNode (location: (25,7)-(25,8)) - │ │ │ ├── requireds: (length: 1) - │ │ │ │ └── @ RequiredParameterNode (location: (25,7)-(25,8)) - │ │ │ │ ├── flags: ∅ - │ │ │ │ └── name: :a - │ │ │ ├── optionals: (length: 0) - │ │ │ ├── rest: ∅ - │ │ │ ├── posts: (length: 0) - │ │ │ ├── keywords: (length: 0) - │ │ │ ├── keyword_rest: ∅ - │ │ │ └── block: ∅ - │ │ ├── locals: (length: 0) - │ │ ├── opening_loc: (25,6)-(25,7) = "|" - │ │ └── closing_loc: (25,8)-(25,9) = "|" - │ ├── body: - │ │ @ StatementsNode (location: (25,10)-(25,15)) - │ │ └── body: (length: 1) - │ │ └── @ BreakNode (location: (25,10)-(25,15)) - │ │ ├── arguments: ∅ - │ │ └── keyword_loc: (25,10)-(25,15) = "break" - │ ├── opening_loc: (25,4)-(25,5) = "{" - │ └── closing_loc: (25,16)-(25,17) = "}" - ├── call_operator_loc: ∅ - ├── name: :== - ├── message_loc: (25,18)-(25,20) = "==" - ├── opening_loc: ∅ - ├── arguments: - │ @ ArgumentsNode (location: (25,21)-(25,23)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ IntegerNode (location: (25,21)-(25,23)) - │ ├── flags: decimal - │ └── value: 42 - ├── closing_loc: ∅ - └── block: ∅ + ├── keyword_loc: (29,0)-(29,5) = "until" + ├── closing_loc: (29,18)-(29,21) = "end" + ├── predicate: + │ @ AndNode (location: (29,6)-(29,16)) + │ ├── left: + │ │ @ CallNode (location: (29,6)-(29,7)) + │ │ ├── flags: variable_call, ignore_visibility + │ │ ├── receiver: ∅ + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :_ + │ │ ├── message_loc: (29,6)-(29,7) = "_" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: ∅ + │ │ ├── closing_loc: ∅ + │ │ └── block: ∅ + │ ├── right: + │ │ @ BreakNode (location: (29,11)-(29,16)) + │ │ ├── arguments: ∅ + │ │ └── keyword_loc: (29,11)-(29,16) = "break" + │ └── operator_loc: (29,8)-(29,10) = "&&" + └── statements: ∅ diff --git a/test/prism/snapshots/defined.txt b/test/prism/snapshots/defined.txt index 53a5081811..c60173ff37 100644 --- a/test/prism/snapshots/defined.txt +++ b/test/prism/snapshots/defined.txt @@ -28,13 +28,13 @@ │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (3,9)-(3,15)) │ │ ├── name_loc: (3,9)-(3,10) = "x" - │ │ ├── operator_loc: (3,11)-(3,13) = "%=" + │ │ ├── binary_operator_loc: (3,11)-(3,13) = "%=" │ │ ├── value: │ │ │ @ IntegerNode (location: (3,14)-(3,15)) │ │ │ ├── flags: decimal │ │ │ └── value: 2 │ │ ├── name: :x - │ │ ├── operator: :% + │ │ ├── binary_operator: :% │ │ └── depth: 0 │ ├── rparen_loc: (3,15)-(3,16) = ")" │ └── keyword_loc: (3,0)-(3,8) = "defined?" diff --git a/test/prism/snapshots/numbers.txt b/test/prism/snapshots/numbers.txt index 740f3f5a2a..58aea454fa 100644 --- a/test/prism/snapshots/numbers.txt +++ b/test/prism/snapshots/numbers.txt @@ -65,52 +65,48 @@ │ ├── flags: decimal │ └── value: 1 ├── @ RationalNode (location: (41,0)-(41,2)) - │ └── numeric: - │ @ IntegerNode (location: (41,0)-(41,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ IntegerNode (location: (43,0)-(43,2)) │ ├── flags: decimal │ └── value: -1 ├── @ ImaginaryNode (location: (45,0)-(45,3)) │ └── numeric: │ @ RationalNode (location: (45,0)-(45,2)) - │ └── numeric: - │ @ IntegerNode (location: (45,0)-(45,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (47,0)-(47,4)) - │ └── numeric: - │ @ FloatNode (location: (47,0)-(47,3)) - │ └── value: 1.2 + │ ├── flags: decimal + │ ├── numerator: 6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (49,0)-(49,5)) │ └── numeric: │ @ RationalNode (location: (49,0)-(49,4)) - │ └── numeric: - │ @ FloatNode (location: (49,0)-(49,3)) - │ └── value: 1.2 + │ ├── flags: decimal + │ ├── numerator: 6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (51,0)-(51,4)) │ └── numeric: │ @ RationalNode (location: (51,0)-(51,3)) - │ └── numeric: - │ @ IntegerNode (location: (51,0)-(51,2)) - │ ├── flags: decimal - │ └── value: -1 + │ ├── flags: decimal + │ ├── numerator: -1 + │ └── denominator: 1 ├── @ RationalNode (location: (53,0)-(53,5)) - │ └── numeric: - │ @ FloatNode (location: (53,0)-(53,4)) - │ └── value: -1.2 + │ ├── flags: decimal + │ ├── numerator: -6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (55,0)-(55,6)) │ └── numeric: │ @ RationalNode (location: (55,0)-(55,5)) - │ └── numeric: - │ @ FloatNode (location: (55,0)-(55,4)) - │ └── value: -1.2 + │ ├── flags: decimal + │ ├── numerator: -6 + │ └── denominator: 5 ├── @ RationalNode (location: (57,0)-(57,4)) - │ └── numeric: - │ @ IntegerNode (location: (57,0)-(57,3)) - │ ├── flags: octal - │ └── value: 1 + │ ├── flags: octal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ ImaginaryNode (location: (59,0)-(59,4)) │ └── numeric: │ @ IntegerNode (location: (59,0)-(59,3)) @@ -119,15 +115,13 @@ ├── @ ImaginaryNode (location: (61,0)-(61,5)) │ └── numeric: │ @ RationalNode (location: (61,0)-(61,4)) - │ └── numeric: - │ @ IntegerNode (location: (61,0)-(61,3)) - │ ├── flags: octal - │ └── value: 1 + │ ├── flags: octal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (63,0)-(63,4)) - │ └── numeric: - │ @ IntegerNode (location: (63,0)-(63,3)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ ImaginaryNode (location: (65,0)-(65,4)) │ └── numeric: │ @ IntegerNode (location: (65,0)-(65,3)) @@ -136,7 +130,6 @@ └── @ ImaginaryNode (location: (67,0)-(67,5)) └── numeric: @ RationalNode (location: (67,0)-(67,4)) - └── numeric: - @ IntegerNode (location: (67,0)-(67,3)) - ├── flags: binary - └── value: 1 + ├── flags: binary + ├── numerator: 1 + └── denominator: 1 diff --git a/test/prism/snapshots/patterns.txt b/test/prism/snapshots/patterns.txt index 16298e7984..17aa23b4b9 100644 --- a/test/prism/snapshots/patterns.txt +++ b/test/prism/snapshots/patterns.txt @@ -86,10 +86,9 @@ │ │ └── block: ∅ │ ├── pattern: │ │ @ RationalNode (location: (5,7)-(5,9)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (5,7)-(5,8)) - │ │ ├── flags: decimal - │ │ └── value: 1 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ └── operator_loc: (5,4)-(5,6) = "=>" ├── @ MatchRequiredNode (location: (6,0)-(6,11)) │ ├── value: @@ -598,16 +597,14 @@ │ │ ├── flags: ∅ │ │ ├── left: │ │ │ @ RationalNode (location: (31,7)-(31,9)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (31,7)-(31,8)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ ├── right: │ │ │ @ RationalNode (location: (31,13)-(31,15)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (31,13)-(31,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ └── operator_loc: (31,10)-(31,12) = ".." │ └── operator_loc: (31,4)-(31,6) = "=>" ├── @ MatchRequiredNode (location: (32,0)-(32,19)) @@ -2461,10 +2458,9 @@ │ │ └── block: ∅ │ ├── pattern: │ │ @ RationalNode (location: (108,7)-(108,9)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (108,7)-(108,8)) - │ │ ├── flags: decimal - │ │ └── value: 1 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ └── operator_loc: (108,4)-(108,6) = "in" ├── @ MatchPredicateNode (location: (109,0)-(109,11)) │ ├── value: @@ -3017,10 +3013,9 @@ │ │ └── @ InNode (location: (139,10)-(139,20)) │ │ ├── pattern: │ │ │ @ RationalNode (location: (139,13)-(139,15)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (139,13)-(139,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ ├── statements: ∅ │ │ ├── in_loc: (139,10)-(139,12) = "in" │ │ └── then_loc: (139,16)-(139,20) = "then" @@ -3758,10 +3753,9 @@ │ │ │ │ @ StatementsNode (location: (166,13)-(166,15)) │ │ │ │ └── body: (length: 1) │ │ │ │ └── @ RationalNode (location: (166,13)-(166,15)) - │ │ │ │ └── numeric: - │ │ │ │ @ IntegerNode (location: (166,13)-(166,14)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 + │ │ │ │ ├── flags: decimal + │ │ │ │ ├── numerator: 1 + │ │ │ │ └── denominator: 1 │ │ │ ├── consequent: ∅ │ │ │ └── end_keyword_loc: ∅ │ │ ├── statements: ∅ diff --git a/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt b/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt index b5acfa1d8b..f9792aebb3 100644 --- a/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt +++ b/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt @@ -10,9 +10,9 @@ │ ├── name: :X │ ├── delimiter_loc: (1,0)-(1,2) = "::" │ └── name_loc: (1,2)-(1,3) = "X" - ├── operator_loc: (1,4)-(1,6) = "&=" + ├── binary_operator_loc: (1,4)-(1,6) = "&=" ├── value: │ @ IntegerNode (location: (1,7)-(1,8)) │ ├── flags: decimal │ └── value: 1 - └── operator: :& + └── binary_operator: :& diff --git a/test/prism/snapshots/seattlerb/dasgn_icky2.txt b/test/prism/snapshots/seattlerb/dasgn_icky2.txt deleted file mode 100644 index e118362f87..0000000000 --- a/test/prism/snapshots/seattlerb/dasgn_icky2.txt +++ /dev/null @@ -1,61 +0,0 @@ -@ ProgramNode (location: (1,0)-(8,3)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(8,3)) - └── body: (length: 1) - └── @ CallNode (location: (1,0)-(8,3)) - ├── flags: ignore_visibility - ├── receiver: ∅ - ├── call_operator_loc: ∅ - ├── name: :a - ├── message_loc: (1,0)-(1,1) = "a" - ├── opening_loc: ∅ - ├── arguments: ∅ - ├── closing_loc: ∅ - └── block: - @ BlockNode (location: (1,2)-(8,3)) - ├── locals: [:v] - ├── parameters: ∅ - ├── body: - │ @ StatementsNode (location: (2,2)-(7,5)) - │ └── body: (length: 2) - │ ├── @ LocalVariableWriteNode (location: (2,2)-(2,9)) - │ │ ├── name: :v - │ │ ├── depth: 0 - │ │ ├── name_loc: (2,2)-(2,3) = "v" - │ │ ├── value: - │ │ │ @ NilNode (location: (2,6)-(2,9)) - │ │ └── operator_loc: (2,4)-(2,5) = "=" - │ └── @ BeginNode (location: (3,2)-(7,5)) - │ ├── begin_keyword_loc: (3,2)-(3,7) = "begin" - │ ├── statements: - │ │ @ StatementsNode (location: (4,4)-(4,9)) - │ │ └── body: (length: 1) - │ │ └── @ YieldNode (location: (4,4)-(4,9)) - │ │ ├── keyword_loc: (4,4)-(4,9) = "yield" - │ │ ├── lparen_loc: ∅ - │ │ ├── arguments: ∅ - │ │ └── rparen_loc: ∅ - │ ├── rescue_clause: - │ │ @ RescueNode (location: (5,2)-(6,9)) - │ │ ├── keyword_loc: (5,2)-(5,8) = "rescue" - │ │ ├── exceptions: (length: 1) - │ │ │ └── @ ConstantReadNode (location: (5,9)-(5,18)) - │ │ │ └── name: :Exception - │ │ ├── operator_loc: (5,19)-(5,21) = "=>" - │ │ ├── reference: - │ │ │ @ LocalVariableTargetNode (location: (5,22)-(5,23)) - │ │ │ ├── name: :v - │ │ │ └── depth: 0 - │ │ ├── statements: - │ │ │ @ StatementsNode (location: (6,4)-(6,9)) - │ │ │ └── body: (length: 1) - │ │ │ └── @ BreakNode (location: (6,4)-(6,9)) - │ │ │ ├── arguments: ∅ - │ │ │ └── keyword_loc: (6,4)-(6,9) = "break" - │ │ └── consequent: ∅ - │ ├── else_clause: ∅ - │ ├── ensure_clause: ∅ - │ └── end_keyword_loc: (7,2)-(7,5) = "end" - ├── opening_loc: (1,2)-(1,4) = "do" - └── closing_loc: (8,0)-(8,3) = "end" diff --git a/test/prism/snapshots/seattlerb/index_0_opasgn.txt b/test/prism/snapshots/seattlerb/index_0_opasgn.txt index 239a549253..322eae9907 100644 --- a/test/prism/snapshots/seattlerb/index_0_opasgn.txt +++ b/test/prism/snapshots/seattlerb/index_0_opasgn.txt @@ -21,8 +21,8 @@ ├── arguments: ∅ ├── closing_loc: (1,2)-(1,3) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,4)-(1,6) = "+=" └── value: @ CallNode (location: (1,7)-(1,8)) ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt b/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt index 92373aeeaf..edef23044a 100644 --- a/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt +++ b/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt @@ -27,7 +27,7 @@ │ │ │ ├── name: :C │ │ │ ├── delimiter_loc: (1,4)-(1,6) = "::" │ │ │ └── name_loc: (1,6)-(1,7) = "C" - │ │ ├── operator_loc: (1,8)-(1,10) = "*=" + │ │ ├── binary_operator_loc: (1,8)-(1,10) = "*=" │ │ ├── value: │ │ │ @ CallNode (location: (1,11)-(1,14)) │ │ │ ├── flags: ignore_visibility @@ -52,7 +52,7 @@ │ │ │ │ └── block: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ - │ │ └── operator: :* + │ │ └── binary_operator: :* │ ├── opening_loc: (1,2)-(1,3) = "(" │ └── closing_loc: (1,14)-(1,15) = ")" ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt index e0e8c5b3e3..523ccde455 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt @@ -12,7 +12,7 @@ │ ├── name: :B │ ├── delimiter_loc: (1,1)-(1,3) = "::" │ └── name_loc: (1,3)-(1,4) = "B" - ├── operator_loc: (1,5)-(1,7) = "*=" + ├── binary_operator_loc: (1,5)-(1,7) = "*=" ├── value: │ @ CallNode (location: (1,8)-(1,11)) │ ├── flags: ignore_visibility @@ -37,4 +37,4 @@ │ │ └── block: ∅ │ ├── closing_loc: ∅ │ └── block: ∅ - └── operator: :* + └── binary_operator: :* diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt index 0daadcf6ff..b9d00edc30 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt @@ -12,8 +12,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :+ - ├── operator_loc: (1,5)-(1,7) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,5)-(1,7) = "+=" └── value: @ IntegerNode (location: (1,8)-(1,9)) ├── flags: decimal diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt index ea8603165b..c12ea3983c 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt @@ -12,8 +12,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :* - ├── operator_loc: (1,5)-(1,7) = "*=" + ├── binary_operator: :* + ├── binary_operator_loc: (1,5)-(1,7) = "*=" └── value: @ CallNode (location: (1,8)-(1,11)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt b/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt index 8199eb2449..84eef70b25 100644 --- a/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt +++ b/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt @@ -40,13 +40,13 @@ │ │ └── block: ∅ │ ├── @ LocalVariableOperatorWriteNode (location: (3,2)-(3,8)) │ │ ├── name_loc: (3,2)-(3,3) = "y" - │ │ ├── operator_loc: (3,4)-(3,6) = "*=" + │ │ ├── binary_operator_loc: (3,4)-(3,6) = "*=" │ │ ├── value: │ │ │ @ IntegerNode (location: (3,7)-(3,8)) │ │ │ ├── flags: decimal │ │ │ └── value: 2 │ │ ├── name: :y - │ │ ├── operator: :* + │ │ ├── binary_operator: :* │ │ └── depth: 0 │ └── @ ReturnNode (location: (4,2)-(4,10)) │ ├── flags: redundant diff --git a/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt b/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt index 5c2eb2da3c..d113f2af9d 100644 --- a/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt +++ b/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt @@ -5,7 +5,7 @@ └── body: (length: 2) ├── @ LocalVariableOperatorWriteNode (location: (1,6)-(2,11)) │ ├── name_loc: (1,6)-(1,9) = "foo" - │ ├── operator_loc: (1,10)-(1,12) = "+=" + │ ├── binary_operator_loc: (1,10)-(1,12) = "+=" │ ├── value: │ │ @ CallNode (location: (2,8)-(2,11)) │ │ ├── flags: variable_call, ignore_visibility @@ -18,7 +18,7 @@ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ CallNode (location: (3,6)-(3,9)) ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/seattlerb/pct_Q_backslash_nl.txt b/test/prism/snapshots/seattlerb/pct_Q_backslash_nl.txt index c8a990c672..31cc0941ee 100644 --- a/test/prism/snapshots/seattlerb/pct_Q_backslash_nl.txt +++ b/test/prism/snapshots/seattlerb/pct_Q_backslash_nl.txt @@ -5,7 +5,7 @@ └── body: (length: 1) └── @ StringNode (location: (1,0)-(2,1)) ├── flags: ∅ - ├── opening_loc: (1,0)-(1,3) = "%Q{" + ├── opening_loc: (1,0)-(1,3) = "%q{" ├── content_loc: (1,3)-(2,0) = " \\\n" ├── closing_loc: (2,0)-(2,1) = "}" - └── unescaped: " " + └── unescaped: " \\\n" diff --git a/test/prism/snapshots/seattlerb/ruby21_numbers.txt b/test/prism/snapshots/seattlerb/ruby21_numbers.txt index 34a3452d1b..e7eec943f8 100644 --- a/test/prism/snapshots/seattlerb/ruby21_numbers.txt +++ b/test/prism/snapshots/seattlerb/ruby21_numbers.txt @@ -12,16 +12,14 @@ │ │ ├── flags: decimal │ │ └── value: 1 │ ├── @ RationalNode (location: (1,5)-(1,7)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (1,5)-(1,6)) - │ │ ├── flags: decimal - │ │ └── value: 2 + │ │ ├── flags: decimal + │ │ ├── numerator: 2 + │ │ └── denominator: 1 │ └── @ ImaginaryNode (location: (1,9)-(1,12)) │ └── numeric: │ @ RationalNode (location: (1,9)-(1,11)) - │ └── numeric: - │ @ IntegerNode (location: (1,9)-(1,10)) - │ ├── flags: decimal - │ └── value: 3 + │ ├── flags: decimal + │ ├── numerator: 3 + │ └── denominator: 1 ├── opening_loc: (1,0)-(1,1) = "[" └── closing_loc: (1,12)-(1,13) = "]" diff --git a/test/prism/snapshots/seattlerb/safe_op_asgn.txt b/test/prism/snapshots/seattlerb/safe_op_asgn.txt index 7a9fd2b7f7..ebcedd6b5e 100644 --- a/test/prism/snapshots/seattlerb/safe_op_asgn.txt +++ b/test/prism/snapshots/seattlerb/safe_op_asgn.txt @@ -20,8 +20,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :+ - ├── operator_loc: (1,5)-(1,7) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,5)-(1,7) = "+=" └── value: @ CallNode (location: (1,8)-(1,11)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/seattlerb/yield_arg.txt b/test/prism/snapshots/seattlerb/yield_arg.txt deleted file mode 100644 index 22e0c14f83..0000000000 --- a/test/prism/snapshots/seattlerb/yield_arg.txt +++ /dev/null @@ -1,16 +0,0 @@ -@ ProgramNode (location: (1,0)-(1,8)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(1,8)) - └── body: (length: 1) - └── @ YieldNode (location: (1,0)-(1,8)) - ├── keyword_loc: (1,0)-(1,5) = "yield" - ├── lparen_loc: ∅ - ├── arguments: - │ @ ArgumentsNode (location: (1,6)-(1,8)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ IntegerNode (location: (1,6)-(1,8)) - │ ├── flags: decimal - │ └── value: 42 - └── rparen_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/yield_call_assocs.txt b/test/prism/snapshots/seattlerb/yield_call_assocs.txt deleted file mode 100644 index c04273f5aa..0000000000 --- a/test/prism/snapshots/seattlerb/yield_call_assocs.txt +++ /dev/null @@ -1,224 +0,0 @@ -@ ProgramNode (location: (1,0)-(11,13)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(11,13)) - └── body: (length: 6) - ├── @ YieldNode (location: (1,0)-(1,16)) - │ ├── keyword_loc: (1,0)-(1,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (1,6)-(1,16)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 2) - │ │ ├── @ IntegerNode (location: (1,6)-(1,7)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── @ KeywordHashNode (location: (1,9)-(1,16)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (1,9)-(1,16)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (1,9)-(1,11)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (1,9)-(1,10) = ":" - │ │ │ ├── value_loc: (1,10)-(1,11) = "z" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "z" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (1,15)-(1,16)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (1,12)-(1,14) = "=>" - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (3,0)-(3,25)) - │ ├── keyword_loc: (3,0)-(3,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (3,6)-(3,25)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 2) - │ │ ├── @ IntegerNode (location: (3,6)-(3,7)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── @ KeywordHashNode (location: (3,9)-(3,25)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 2) - │ │ ├── @ AssocNode (location: (3,9)-(3,16)) - │ │ │ ├── key: - │ │ │ │ @ SymbolNode (location: (3,9)-(3,11)) - │ │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ │ ├── opening_loc: (3,9)-(3,10) = ":" - │ │ │ │ ├── value_loc: (3,10)-(3,11) = "z" - │ │ │ │ ├── closing_loc: ∅ - │ │ │ │ └── unescaped: "z" - │ │ │ ├── value: - │ │ │ │ @ IntegerNode (location: (3,15)-(3,16)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 - │ │ │ └── operator_loc: (3,12)-(3,14) = "=>" - │ │ └── @ AssocNode (location: (3,18)-(3,25)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (3,18)-(3,20)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (3,18)-(3,19) = ":" - │ │ │ ├── value_loc: (3,19)-(3,20) = "w" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "w" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (3,24)-(3,25)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 2 - │ │ └── operator_loc: (3,21)-(3,23) = "=>" - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (5,0)-(5,13)) - │ ├── keyword_loc: (5,0)-(5,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (5,6)-(5,13)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ CallNode (location: (5,6)-(5,13)) - │ │ ├── flags: ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :y - │ │ ├── message_loc: (5,6)-(5,7) = "y" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: - │ │ │ @ ArgumentsNode (location: (5,8)-(5,13)) - │ │ │ ├── flags: ∅ - │ │ │ └── arguments: (length: 1) - │ │ │ └── @ KeywordHashNode (location: (5,8)-(5,13)) - │ │ │ ├── flags: symbol_keys - │ │ │ └── elements: (length: 1) - │ │ │ └── @ AssocNode (location: (5,8)-(5,13)) - │ │ │ ├── key: - │ │ │ │ @ SymbolNode (location: (5,8)-(5,10)) - │ │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ │ ├── opening_loc: (5,8)-(5,9) = ":" - │ │ │ │ ├── value_loc: (5,9)-(5,10) = "z" - │ │ │ │ ├── closing_loc: ∅ - │ │ │ │ └── unescaped: "z" - │ │ │ ├── value: - │ │ │ │ @ IntegerNode (location: (5,12)-(5,13)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 - │ │ │ └── operator_loc: (5,10)-(5,12) = "=>" - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (7,0)-(7,11)) - │ ├── keyword_loc: (7,0)-(7,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (7,6)-(7,11)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ CallNode (location: (7,6)-(7,11)) - │ │ ├── flags: ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :y - │ │ ├── message_loc: (7,6)-(7,7) = "y" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: - │ │ │ @ ArgumentsNode (location: (7,8)-(7,11)) - │ │ │ ├── flags: ∅ - │ │ │ └── arguments: (length: 1) - │ │ │ └── @ KeywordHashNode (location: (7,8)-(7,11)) - │ │ │ ├── flags: symbol_keys - │ │ │ └── elements: (length: 1) - │ │ │ └── @ AssocNode (location: (7,8)-(7,11)) - │ │ │ ├── key: - │ │ │ │ @ SymbolNode (location: (7,8)-(7,10)) - │ │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ │ ├── opening_loc: ∅ - │ │ │ │ ├── value_loc: (7,8)-(7,9) = "z" - │ │ │ │ ├── closing_loc: (7,9)-(7,10) = ":" - │ │ │ │ └── unescaped: "z" - │ │ │ ├── value: - │ │ │ │ @ IntegerNode (location: (7,10)-(7,11)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 - │ │ │ └── operator_loc: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (9,0)-(9,12)) - │ ├── keyword_loc: (9,0)-(9,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (9,6)-(9,12)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ CallNode (location: (9,6)-(9,12)) - │ │ ├── flags: ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :y - │ │ ├── message_loc: (9,6)-(9,7) = "y" - │ │ ├── opening_loc: (9,7)-(9,8) = "(" - │ │ ├── arguments: - │ │ │ @ ArgumentsNode (location: (9,8)-(9,11)) - │ │ │ ├── flags: ∅ - │ │ │ └── arguments: (length: 1) - │ │ │ └── @ KeywordHashNode (location: (9,8)-(9,11)) - │ │ │ ├── flags: symbol_keys - │ │ │ └── elements: (length: 1) - │ │ │ └── @ AssocNode (location: (9,8)-(9,11)) - │ │ │ ├── key: - │ │ │ │ @ SymbolNode (location: (9,8)-(9,10)) - │ │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ │ ├── opening_loc: ∅ - │ │ │ │ ├── value_loc: (9,8)-(9,9) = "z" - │ │ │ │ ├── closing_loc: (9,9)-(9,10) = ":" - │ │ │ │ └── unescaped: "z" - │ │ │ ├── value: - │ │ │ │ @ IntegerNode (location: (9,10)-(9,11)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 - │ │ │ └── operator_loc: ∅ - │ │ ├── closing_loc: (9,11)-(9,12) = ")" - │ │ └── block: ∅ - │ └── rparen_loc: ∅ - └── @ YieldNode (location: (11,0)-(11,13)) - ├── keyword_loc: (11,0)-(11,5) = "yield" - ├── lparen_loc: ∅ - ├── arguments: - │ @ ArgumentsNode (location: (11,6)-(11,13)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ CallNode (location: (11,6)-(11,13)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :y - │ ├── message_loc: (11,6)-(11,7) = "y" - │ ├── opening_loc: (11,7)-(11,8) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (11,8)-(11,12)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (11,8)-(11,12)) - │ │ ├── flags: ∅ - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (11,8)-(11,12)) - │ │ ├── key: - │ │ │ @ CallNode (location: (11,8)-(11,9)) - │ │ │ ├── flags: variable_call, ignore_visibility - │ │ │ ├── receiver: ∅ - │ │ │ ├── call_operator_loc: ∅ - │ │ │ ├── name: :z - │ │ │ ├── message_loc: (11,8)-(11,9) = "z" - │ │ │ ├── opening_loc: ∅ - │ │ │ ├── arguments: ∅ - │ │ │ ├── closing_loc: ∅ - │ │ │ └── block: ∅ - │ │ ├── value: - │ │ │ @ IntegerNode (location: (11,11)-(11,12)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (11,9)-(11,11) = "=>" - │ ├── closing_loc: (11,12)-(11,13) = ")" - │ └── block: ∅ - └── rparen_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/yield_empty_parens.txt b/test/prism/snapshots/seattlerb/yield_empty_parens.txt deleted file mode 100644 index 5ecd89823f..0000000000 --- a/test/prism/snapshots/seattlerb/yield_empty_parens.txt +++ /dev/null @@ -1,10 +0,0 @@ -@ ProgramNode (location: (1,0)-(1,7)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(1,7)) - └── body: (length: 1) - └── @ YieldNode (location: (1,0)-(1,7)) - ├── keyword_loc: (1,0)-(1,5) = "yield" - ├── lparen_loc: (1,5)-(1,6) = "(" - ├── arguments: ∅ - └── rparen_loc: (1,6)-(1,7) = ")" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index dbd3a4d030..48ff0d634f 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -146,10 +146,9 @@ │ │ ├── @ FloatNode (location: (29,4)-(29,7)) │ │ │ └── value: 1.0 │ │ ├── @ RationalNode (location: (29,9)-(29,11)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (29,9)-(29,10)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ └── @ ImaginaryNode (location: (29,13)-(29,15)) │ │ └── numeric: │ │ @ IntegerNode (location: (29,13)-(29,14)) diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index 98b88e11ce..ddb10456bf 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -316,18 +316,17 @@ │ ├── flags: decimal │ └── value: 1 ├── @ RationalNode (location: (19,0)-(19,2)) - │ └── numeric: - │ @ IntegerNode (location: (19,0)-(19,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (20,0)-(20,4)) - │ └── numeric: - │ @ FloatNode (location: (20,0)-(20,3)) - │ └── value: 1.5 + │ ├── flags: decimal + │ ├── numerator: 3 + │ └── denominator: 2 ├── @ RationalNode (location: (21,0)-(21,4)) - │ └── numeric: - │ @ FloatNode (location: (21,0)-(21,3)) - │ └── value: 1.3 + │ ├── flags: decimal + │ ├── numerator: 13 + │ └── denominator: 10 ├── @ ImaginaryNode (location: (22,0)-(22,2)) │ └── numeric: │ @ IntegerNode (location: (22,0)-(22,1)) @@ -354,10 +353,9 @@ ├── @ ImaginaryNode (location: (27,0)-(27,3)) │ └── numeric: │ @ RationalNode (location: (27,0)-(27,2)) - │ └── numeric: - │ @ IntegerNode (location: (27,0)-(27,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ InterpolatedStringNode (location: (28,0)-(28,11)) │ ├── flags: ∅ │ ├── opening_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/opasgn.txt b/test/prism/snapshots/unparser/corpus/literal/opasgn.txt index 8dc0849638..0761b47348 100644 --- a/test/prism/snapshots/unparser/corpus/literal/opasgn.txt +++ b/test/prism/snapshots/unparser/corpus/literal/opasgn.txt @@ -5,53 +5,53 @@ └── body: (length: 24) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,6)) │ ├── name_loc: (1,0)-(1,1) = "a" - │ ├── operator_loc: (1,2)-(1,4) = "+=" + │ ├── binary_operator_loc: (1,2)-(1,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (1,5)-(1,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (2,0)-(2,6)) │ ├── name_loc: (2,0)-(2,1) = "a" - │ ├── operator_loc: (2,2)-(2,4) = "-=" + │ ├── binary_operator_loc: (2,2)-(2,4) = "-=" │ ├── value: │ │ @ IntegerNode (location: (2,5)-(2,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :- + │ ├── binary_operator: :- │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,7)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,5) = "**=" + │ ├── binary_operator_loc: (3,2)-(3,5) = "**=" │ ├── value: │ │ @ IntegerNode (location: (3,6)-(3,7)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :** + │ ├── binary_operator: :** │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (4,0)-(4,6)) │ ├── name_loc: (4,0)-(4,1) = "a" - │ ├── operator_loc: (4,2)-(4,4) = "*=" + │ ├── binary_operator_loc: (4,2)-(4,4) = "*=" │ ├── value: │ │ @ IntegerNode (location: (4,5)-(4,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :* + │ ├── binary_operator: :* │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (5,0)-(5,6)) │ ├── name_loc: (5,0)-(5,1) = "a" - │ ├── operator_loc: (5,2)-(5,4) = "/=" + │ ├── binary_operator_loc: (5,2)-(5,4) = "/=" │ ├── value: │ │ @ IntegerNode (location: (5,5)-(5,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :/ + │ ├── binary_operator: :/ │ └── depth: 0 ├── @ LocalVariableAndWriteNode (location: (6,0)-(6,7)) │ ├── name_loc: (6,0)-(6,1) = "a" @@ -162,8 +162,8 @@ │ ├── message_loc: (10,2)-(10,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :+ - │ ├── operator_loc: (10,4)-(10,6) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (10,4)-(10,6) = "+=" │ └── value: │ @ IntegerNode (location: (10,7)-(10,8)) │ ├── flags: decimal @@ -178,8 +178,8 @@ │ ├── message_loc: (11,2)-(11,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :- - │ ├── operator_loc: (11,4)-(11,6) = "-=" + │ ├── binary_operator: :- + │ ├── binary_operator_loc: (11,4)-(11,6) = "-=" │ └── value: │ @ IntegerNode (location: (11,7)-(11,8)) │ ├── flags: decimal @@ -194,8 +194,8 @@ │ ├── message_loc: (12,2)-(12,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :** - │ ├── operator_loc: (12,4)-(12,7) = "**=" + │ ├── binary_operator: :** + │ ├── binary_operator_loc: (12,4)-(12,7) = "**=" │ └── value: │ @ IntegerNode (location: (12,8)-(12,9)) │ ├── flags: decimal @@ -210,8 +210,8 @@ │ ├── message_loc: (13,2)-(13,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :* - │ ├── operator_loc: (13,4)-(13,6) = "*=" + │ ├── binary_operator: :* + │ ├── binary_operator_loc: (13,4)-(13,6) = "*=" │ └── value: │ @ IntegerNode (location: (13,7)-(13,8)) │ ├── flags: decimal @@ -226,8 +226,8 @@ │ ├── message_loc: (14,2)-(14,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :/ - │ ├── operator_loc: (14,4)-(14,6) = "/=" + │ ├── binary_operator: :/ + │ ├── binary_operator_loc: (14,4)-(14,6) = "/=" │ └── value: │ @ IntegerNode (location: (14,7)-(14,8)) │ ├── flags: decimal @@ -293,8 +293,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (17,3)-(17,4) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (17,5)-(17,7) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (17,5)-(17,7) = "+=" │ └── value: │ @ IntegerNode (location: (17,8)-(17,9)) │ ├── flags: decimal @@ -323,8 +323,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (18,3)-(18,4) = "]" │ ├── block: ∅ - │ ├── operator: :- - │ ├── operator_loc: (18,5)-(18,7) = "-=" + │ ├── binary_operator: :- + │ ├── binary_operator_loc: (18,5)-(18,7) = "-=" │ └── value: │ @ IntegerNode (location: (18,8)-(18,9)) │ ├── flags: decimal @@ -353,8 +353,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (19,3)-(19,4) = "]" │ ├── block: ∅ - │ ├── operator: :** - │ ├── operator_loc: (19,5)-(19,8) = "**=" + │ ├── binary_operator: :** + │ ├── binary_operator_loc: (19,5)-(19,8) = "**=" │ └── value: │ @ IntegerNode (location: (19,9)-(19,10)) │ ├── flags: decimal @@ -383,8 +383,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (20,3)-(20,4) = "]" │ ├── block: ∅ - │ ├── operator: :* - │ ├── operator_loc: (20,5)-(20,7) = "*=" + │ ├── binary_operator: :* + │ ├── binary_operator_loc: (20,5)-(20,7) = "*=" │ └── value: │ @ IntegerNode (location: (20,8)-(20,9)) │ ├── flags: decimal @@ -413,8 +413,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (21,3)-(21,4) = "]" │ ├── block: ∅ - │ ├── operator: :/ - │ ├── operator_loc: (21,5)-(21,7) = "/=" + │ ├── binary_operator: :/ + │ ├── binary_operator_loc: (21,5)-(21,7) = "/=" │ └── value: │ @ IntegerNode (location: (21,8)-(21,9)) │ ├── flags: decimal @@ -501,8 +501,8 @@ ├── message_loc: (24,4)-(24,5) = "A" ├── read_name: :A ├── write_name: :A= - ├── operator: :+ - ├── operator_loc: (24,6)-(24,8) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (24,6)-(24,8) = "+=" └── value: @ IntegerNode (location: (24,9)-(24,10)) ├── flags: decimal diff --git a/test/prism/snapshots/unparser/corpus/literal/yield.txt b/test/prism/snapshots/unparser/corpus/literal/yield.txt deleted file mode 100644 index 4d2b272e35..0000000000 --- a/test/prism/snapshots/unparser/corpus/literal/yield.txt +++ /dev/null @@ -1,56 +0,0 @@ -@ ProgramNode (location: (1,0)-(3,11)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(3,11)) - └── body: (length: 3) - ├── @ YieldNode (location: (1,0)-(1,5)) - │ ├── keyword_loc: (1,0)-(1,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: ∅ - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (2,0)-(2,8)) - │ ├── keyword_loc: (2,0)-(2,5) = "yield" - │ ├── lparen_loc: (2,5)-(2,6) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (2,6)-(2,7)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ CallNode (location: (2,6)-(2,7)) - │ │ ├── flags: variable_call, ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :a - │ │ ├── message_loc: (2,6)-(2,7) = "a" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── rparen_loc: (2,7)-(2,8) = ")" - └── @ YieldNode (location: (3,0)-(3,11)) - ├── keyword_loc: (3,0)-(3,5) = "yield" - ├── lparen_loc: (3,5)-(3,6) = "(" - ├── arguments: - │ @ ArgumentsNode (location: (3,6)-(3,10)) - │ ├── flags: ∅ - │ └── arguments: (length: 2) - │ ├── @ CallNode (location: (3,6)-(3,7)) - │ │ ├── flags: variable_call, ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :a - │ │ ├── message_loc: (3,6)-(3,7) = "a" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── @ CallNode (location: (3,9)-(3,10)) - │ ├── flags: variable_call, ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :b - │ ├── message_loc: (3,9)-(3,10) = "b" - │ ├── opening_loc: ∅ - │ ├── arguments: ∅ - │ ├── closing_loc: ∅ - │ └── block: ∅ - └── rparen_loc: (3,10)-(3,11) = ")" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index ef666890be..448207f5d3 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -4,14 +4,13 @@ @ StatementsNode (location: (1,0)-(14,10)) └── body: (length: 14) ├── @ RationalNode (location: (1,0)-(1,4)) - │ └── numeric: - │ @ FloatNode (location: (1,0)-(1,3)) - │ └── value: 1.0 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (2,0)-(2,3)) - │ └── numeric: - │ @ IntegerNode (location: (2,0)-(2,2)) - │ ├── flags: decimal - │ └── value: 0 + │ ├── flags: decimal + │ ├── numerator: 0 + │ └── denominator: 1 ├── @ IntegerNode (location: (3,0)-(3,3)) │ ├── flags: hexadecimal │ └── value: 1 diff --git a/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt b/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt index e100dd8ecb..7dd26a38dc 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt @@ -44,8 +44,8 @@ │ └── closing_loc: (1,10)-(1,11) = "\"" ├── closing_loc: (1,11)-(1,12) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,13)-(1,15) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,13)-(1,15) = "+=" └── value: @ InterpolatedStringNode (location: (1,16)-(1,25)) ├── flags: ∅ diff --git a/test/prism/snapshots/whitequark/args_assocs.txt b/test/prism/snapshots/whitequark/args_assocs.txt deleted file mode 100644 index 47cb68d899..0000000000 --- a/test/prism/snapshots/whitequark/args_assocs.txt +++ /dev/null @@ -1,195 +0,0 @@ -@ ProgramNode (location: (1,0)-(11,17)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(11,17)) - └── body: (length: 6) - ├── @ CallNode (location: (1,0)-(1,14)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :fun - │ ├── message_loc: (1,0)-(1,3) = "fun" - │ ├── opening_loc: (1,3)-(1,4) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (1,4)-(1,13)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (1,4)-(1,13)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (1,4)-(1,13)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (1,4)-(1,8)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (1,4)-(1,5) = ":" - │ │ │ ├── value_loc: (1,5)-(1,8) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (1,12)-(1,13)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (1,9)-(1,11) = "=>" - │ ├── closing_loc: (1,13)-(1,14) = ")" - │ └── block: ∅ - ├── @ CallNode (location: (3,0)-(3,19)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :fun - │ ├── message_loc: (3,0)-(3,3) = "fun" - │ ├── opening_loc: (3,3)-(3,4) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (3,4)-(3,13)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (3,4)-(3,13)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (3,4)-(3,13)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (3,4)-(3,8)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (3,4)-(3,5) = ":" - │ │ │ ├── value_loc: (3,5)-(3,8) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (3,12)-(3,13)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (3,9)-(3,11) = "=>" - │ ├── closing_loc: (3,19)-(3,20) = ")" - │ └── block: - │ @ BlockArgumentNode (location: (3,15)-(3,19)) - │ ├── expression: - │ │ @ CallNode (location: (3,16)-(3,19)) - │ │ ├── flags: variable_call, ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :baz - │ │ ├── message_loc: (3,16)-(3,19) = "baz" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── operator_loc: (3,15)-(3,16) = "&" - ├── @ CallNode (location: (5,0)-(5,21)) - │ ├── flags: ignore_visibility - │ ├── receiver: - │ │ @ SelfNode (location: (5,0)-(5,4)) - │ ├── call_operator_loc: (5,4)-(5,5) = "." - │ ├── name: :[]= - │ ├── message_loc: (5,5)-(5,8) = "[]=" - │ ├── opening_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (5,9)-(5,21)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 2) - │ │ ├── @ CallNode (location: (5,9)-(5,12)) - │ │ │ ├── flags: variable_call, ignore_visibility - │ │ │ ├── receiver: ∅ - │ │ │ ├── call_operator_loc: ∅ - │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (5,9)-(5,12) = "foo" - │ │ │ ├── opening_loc: ∅ - │ │ │ ├── arguments: ∅ - │ │ │ ├── closing_loc: ∅ - │ │ │ └── block: ∅ - │ │ └── @ KeywordHashNode (location: (5,14)-(5,21)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (5,14)-(5,21)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (5,14)-(5,16)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (5,14)-(5,15) = ":" - │ │ │ ├── value_loc: (5,15)-(5,16) = "a" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "a" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (5,20)-(5,21)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (5,17)-(5,19) = "=>" - │ ├── closing_loc: ∅ - │ └── block: ∅ - ├── @ CallNode (location: (7,0)-(7,15)) - │ ├── flags: ignore_visibility - │ ├── receiver: - │ │ @ SelfNode (location: (7,0)-(7,4)) - │ ├── call_operator_loc: ∅ - │ ├── name: :[] - │ ├── message_loc: (7,4)-(7,15) = "[:bar => 1]" - │ ├── opening_loc: (7,4)-(7,5) = "[" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (7,5)-(7,14)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (7,5)-(7,14)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (7,5)-(7,14)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (7,5)-(7,9)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (7,5)-(7,6) = ":" - │ │ │ ├── value_loc: (7,6)-(7,9) = "bar" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "bar" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (7,13)-(7,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (7,10)-(7,12) = "=>" - │ ├── closing_loc: (7,14)-(7,15) = "]" - │ └── block: ∅ - ├── @ SuperNode (location: (9,0)-(9,17)) - │ ├── keyword_loc: (9,0)-(9,5) = "super" - │ ├── lparen_loc: (9,5)-(9,6) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (9,6)-(9,16)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (9,6)-(9,16)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (9,6)-(9,16)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (9,6)-(9,10)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (9,6)-(9,7) = ":" - │ │ │ ├── value_loc: (9,7)-(9,10) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (9,14)-(9,16)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 42 - │ │ └── operator_loc: (9,11)-(9,13) = "=>" - │ ├── rparen_loc: (9,16)-(9,17) = ")" - │ └── block: ∅ - └── @ YieldNode (location: (11,0)-(11,17)) - ├── keyword_loc: (11,0)-(11,5) = "yield" - ├── lparen_loc: (11,5)-(11,6) = "(" - ├── arguments: - │ @ ArgumentsNode (location: (11,6)-(11,16)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ KeywordHashNode (location: (11,6)-(11,16)) - │ ├── flags: symbol_keys - │ └── elements: (length: 1) - │ └── @ AssocNode (location: (11,6)-(11,16)) - │ ├── key: - │ │ @ SymbolNode (location: (11,6)-(11,10)) - │ │ ├── flags: forced_us_ascii_encoding - │ │ ├── opening_loc: (11,6)-(11,7) = ":" - │ │ ├── value_loc: (11,7)-(11,10) = "foo" - │ │ ├── closing_loc: ∅ - │ │ └── unescaped: "foo" - │ ├── value: - │ │ @ IntegerNode (location: (11,14)-(11,16)) - │ │ ├── flags: decimal - │ │ └── value: 42 - │ └── operator_loc: (11,11)-(11,13) = "=>" - └── rparen_loc: (11,16)-(11,17) = ")" diff --git a/test/prism/snapshots/whitequark/args_assocs_legacy.txt b/test/prism/snapshots/whitequark/args_assocs_legacy.txt deleted file mode 100644 index 47cb68d899..0000000000 --- a/test/prism/snapshots/whitequark/args_assocs_legacy.txt +++ /dev/null @@ -1,195 +0,0 @@ -@ ProgramNode (location: (1,0)-(11,17)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(11,17)) - └── body: (length: 6) - ├── @ CallNode (location: (1,0)-(1,14)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :fun - │ ├── message_loc: (1,0)-(1,3) = "fun" - │ ├── opening_loc: (1,3)-(1,4) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (1,4)-(1,13)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (1,4)-(1,13)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (1,4)-(1,13)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (1,4)-(1,8)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (1,4)-(1,5) = ":" - │ │ │ ├── value_loc: (1,5)-(1,8) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (1,12)-(1,13)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (1,9)-(1,11) = "=>" - │ ├── closing_loc: (1,13)-(1,14) = ")" - │ └── block: ∅ - ├── @ CallNode (location: (3,0)-(3,19)) - │ ├── flags: ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :fun - │ ├── message_loc: (3,0)-(3,3) = "fun" - │ ├── opening_loc: (3,3)-(3,4) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (3,4)-(3,13)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (3,4)-(3,13)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (3,4)-(3,13)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (3,4)-(3,8)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (3,4)-(3,5) = ":" - │ │ │ ├── value_loc: (3,5)-(3,8) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (3,12)-(3,13)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (3,9)-(3,11) = "=>" - │ ├── closing_loc: (3,19)-(3,20) = ")" - │ └── block: - │ @ BlockArgumentNode (location: (3,15)-(3,19)) - │ ├── expression: - │ │ @ CallNode (location: (3,16)-(3,19)) - │ │ ├── flags: variable_call, ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :baz - │ │ ├── message_loc: (3,16)-(3,19) = "baz" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── operator_loc: (3,15)-(3,16) = "&" - ├── @ CallNode (location: (5,0)-(5,21)) - │ ├── flags: ignore_visibility - │ ├── receiver: - │ │ @ SelfNode (location: (5,0)-(5,4)) - │ ├── call_operator_loc: (5,4)-(5,5) = "." - │ ├── name: :[]= - │ ├── message_loc: (5,5)-(5,8) = "[]=" - │ ├── opening_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (5,9)-(5,21)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 2) - │ │ ├── @ CallNode (location: (5,9)-(5,12)) - │ │ │ ├── flags: variable_call, ignore_visibility - │ │ │ ├── receiver: ∅ - │ │ │ ├── call_operator_loc: ∅ - │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (5,9)-(5,12) = "foo" - │ │ │ ├── opening_loc: ∅ - │ │ │ ├── arguments: ∅ - │ │ │ ├── closing_loc: ∅ - │ │ │ └── block: ∅ - │ │ └── @ KeywordHashNode (location: (5,14)-(5,21)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (5,14)-(5,21)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (5,14)-(5,16)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (5,14)-(5,15) = ":" - │ │ │ ├── value_loc: (5,15)-(5,16) = "a" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "a" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (5,20)-(5,21)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (5,17)-(5,19) = "=>" - │ ├── closing_loc: ∅ - │ └── block: ∅ - ├── @ CallNode (location: (7,0)-(7,15)) - │ ├── flags: ignore_visibility - │ ├── receiver: - │ │ @ SelfNode (location: (7,0)-(7,4)) - │ ├── call_operator_loc: ∅ - │ ├── name: :[] - │ ├── message_loc: (7,4)-(7,15) = "[:bar => 1]" - │ ├── opening_loc: (7,4)-(7,5) = "[" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (7,5)-(7,14)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (7,5)-(7,14)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (7,5)-(7,14)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (7,5)-(7,9)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (7,5)-(7,6) = ":" - │ │ │ ├── value_loc: (7,6)-(7,9) = "bar" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "bar" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (7,13)-(7,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 - │ │ └── operator_loc: (7,10)-(7,12) = "=>" - │ ├── closing_loc: (7,14)-(7,15) = "]" - │ └── block: ∅ - ├── @ SuperNode (location: (9,0)-(9,17)) - │ ├── keyword_loc: (9,0)-(9,5) = "super" - │ ├── lparen_loc: (9,5)-(9,6) = "(" - │ ├── arguments: - │ │ @ ArgumentsNode (location: (9,6)-(9,16)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ KeywordHashNode (location: (9,6)-(9,16)) - │ │ ├── flags: symbol_keys - │ │ └── elements: (length: 1) - │ │ └── @ AssocNode (location: (9,6)-(9,16)) - │ │ ├── key: - │ │ │ @ SymbolNode (location: (9,6)-(9,10)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (9,6)-(9,7) = ":" - │ │ │ ├── value_loc: (9,7)-(9,10) = "foo" - │ │ │ ├── closing_loc: ∅ - │ │ │ └── unescaped: "foo" - │ │ ├── value: - │ │ │ @ IntegerNode (location: (9,14)-(9,16)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 42 - │ │ └── operator_loc: (9,11)-(9,13) = "=>" - │ ├── rparen_loc: (9,16)-(9,17) = ")" - │ └── block: ∅ - └── @ YieldNode (location: (11,0)-(11,17)) - ├── keyword_loc: (11,0)-(11,5) = "yield" - ├── lparen_loc: (11,5)-(11,6) = "(" - ├── arguments: - │ @ ArgumentsNode (location: (11,6)-(11,16)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ KeywordHashNode (location: (11,6)-(11,16)) - │ ├── flags: symbol_keys - │ └── elements: (length: 1) - │ └── @ AssocNode (location: (11,6)-(11,16)) - │ ├── key: - │ │ @ SymbolNode (location: (11,6)-(11,10)) - │ │ ├── flags: forced_us_ascii_encoding - │ │ ├── opening_loc: (11,6)-(11,7) = ":" - │ │ ├── value_loc: (11,7)-(11,10) = "foo" - │ │ ├── closing_loc: ∅ - │ │ └── unescaped: "foo" - │ ├── value: - │ │ @ IntegerNode (location: (11,14)-(11,16)) - │ │ ├── flags: decimal - │ │ └── value: 42 - │ └── operator_loc: (11,11)-(11,13) = "=>" - └── rparen_loc: (11,16)-(11,17) = ")" diff --git a/test/prism/snapshots/whitequark/complex.txt b/test/prism/snapshots/whitequark/complex.txt index e688585a5f..bc748db09b 100644 --- a/test/prism/snapshots/whitequark/complex.txt +++ b/test/prism/snapshots/whitequark/complex.txt @@ -10,9 +10,9 @@ ├── @ ImaginaryNode (location: (3,0)-(3,6)) │ └── numeric: │ @ RationalNode (location: (3,0)-(3,5)) - │ └── numeric: - │ @ FloatNode (location: (3,0)-(3,4)) - │ └── value: 42.1 + │ ├── flags: decimal + │ ├── numerator: 421 + │ └── denominator: 10 ├── @ ImaginaryNode (location: (5,0)-(5,3)) │ └── numeric: │ @ IntegerNode (location: (5,0)-(5,2)) @@ -21,7 +21,6 @@ └── @ ImaginaryNode (location: (7,0)-(7,4)) └── numeric: @ RationalNode (location: (7,0)-(7,3)) - └── numeric: - @ IntegerNode (location: (7,0)-(7,2)) - ├── flags: decimal - └── value: 42 + ├── flags: decimal + ├── numerator: 42 + └── denominator: 1 diff --git a/test/prism/snapshots/whitequark/const_op_asgn.txt b/test/prism/snapshots/whitequark/const_op_asgn.txt index 505ce6ef64..71df6208d2 100644 --- a/test/prism/snapshots/whitequark/const_op_asgn.txt +++ b/test/prism/snapshots/whitequark/const_op_asgn.txt @@ -10,21 +10,21 @@ │ │ ├── name: :A │ │ ├── delimiter_loc: (1,0)-(1,2) = "::" │ │ └── name_loc: (1,2)-(1,3) = "A" - │ ├── operator_loc: (1,4)-(1,6) = "+=" + │ ├── binary_operator_loc: (1,4)-(1,6) = "+=" │ ├── value: │ │ @ IntegerNode (location: (1,7)-(1,8)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ ConstantOperatorWriteNode (location: (3,0)-(3,6)) │ ├── name: :A │ ├── name_loc: (3,0)-(3,1) = "A" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (3,5)-(3,6)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ ConstantPathOperatorWriteNode (location: (5,0)-(5,9)) │ ├── target: │ │ @ ConstantPathNode (location: (5,0)-(5,4)) @@ -34,12 +34,12 @@ │ │ ├── name: :A │ │ ├── delimiter_loc: (5,1)-(5,3) = "::" │ │ └── name_loc: (5,3)-(5,4) = "A" - │ ├── operator_loc: (5,5)-(5,7) = "+=" + │ ├── binary_operator_loc: (5,5)-(5,7) = "+=" │ ├── value: │ │ @ IntegerNode (location: (5,8)-(5,9)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ DefNode (location: (7,0)-(7,21)) │ ├── name: :x │ ├── name_loc: (7,4)-(7,5) = "x" diff --git a/test/prism/snapshots/whitequark/method_definition_in_while_cond.txt b/test/prism/snapshots/whitequark/method_definition_in_while_cond.txt new file mode 100644 index 0000000000..963686b73e --- /dev/null +++ b/test/prism/snapshots/whitequark/method_definition_in_while_cond.txt @@ -0,0 +1,199 @@ +@ ProgramNode (location: (1,0)-(7,47)) +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(7,47)) + └── body: (length: 4) + ├── @ WhileNode (location: (1,0)-(1,45)) + │ ├── flags: ∅ + │ ├── keyword_loc: (1,0)-(1,5) = "while" + │ ├── closing_loc: (1,42)-(1,45) = "end" + │ ├── predicate: + │ │ @ DefNode (location: (1,6)-(1,33)) + │ │ ├── name: :foo + │ │ ├── name_loc: (1,10)-(1,13) = "foo" + │ │ ├── receiver: ∅ + │ │ ├── parameters: + │ │ │ @ ParametersNode (location: (1,14)-(1,28)) + │ │ │ ├── requireds: (length: 0) + │ │ │ ├── optionals: (length: 1) + │ │ │ │ └── @ OptionalParameterNode (location: (1,14)-(1,28)) + │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── name: :a + │ │ │ │ ├── name_loc: (1,14)-(1,15) = "a" + │ │ │ │ ├── operator_loc: (1,16)-(1,17) = "=" + │ │ │ │ └── value: + │ │ │ │ @ CallNode (location: (1,18)-(1,28)) + │ │ │ │ ├── flags: ignore_visibility + │ │ │ │ ├── receiver: ∅ + │ │ │ │ ├── call_operator_loc: ∅ + │ │ │ │ ├── name: :tap + │ │ │ │ ├── message_loc: (1,18)-(1,21) = "tap" + │ │ │ │ ├── opening_loc: ∅ + │ │ │ │ ├── arguments: ∅ + │ │ │ │ ├── closing_loc: ∅ + │ │ │ │ └── block: + │ │ │ │ @ BlockNode (location: (1,22)-(1,28)) + │ │ │ │ ├── locals: [] + │ │ │ │ ├── parameters: ∅ + │ │ │ │ ├── body: ∅ + │ │ │ │ ├── opening_loc: (1,22)-(1,24) = "do" + │ │ │ │ └── closing_loc: (1,25)-(1,28) = "end" + │ │ │ ├── rest: ∅ + │ │ │ ├── posts: (length: 0) + │ │ │ ├── keywords: (length: 0) + │ │ │ ├── keyword_rest: ∅ + │ │ │ └── block: ∅ + │ │ ├── body: ∅ + │ │ ├── locals: [:a] + │ │ ├── def_keyword_loc: (1,6)-(1,9) = "def" + │ │ ├── operator_loc: ∅ + │ │ ├── lparen_loc: ∅ + │ │ ├── rparen_loc: ∅ + │ │ ├── equal_loc: ∅ + │ │ └── end_keyword_loc: (1,30)-(1,33) = "end" + │ └── statements: + │ @ StatementsNode (location: (1,35)-(1,40)) + │ └── body: (length: 1) + │ └── @ BreakNode (location: (1,35)-(1,40)) + │ ├── arguments: ∅ + │ └── keyword_loc: (1,35)-(1,40) = "break" + ├── @ WhileNode (location: (3,0)-(3,42)) + │ ├── flags: ∅ + │ ├── keyword_loc: (3,0)-(3,5) = "while" + │ ├── closing_loc: (3,39)-(3,42) = "end" + │ ├── predicate: + │ │ @ DefNode (location: (3,6)-(3,30)) + │ │ ├── name: :foo + │ │ ├── name_loc: (3,10)-(3,13) = "foo" + │ │ ├── receiver: ∅ + │ │ ├── parameters: ∅ + │ │ ├── body: + │ │ │ @ StatementsNode (location: (3,15)-(3,25)) + │ │ │ └── body: (length: 1) + │ │ │ └── @ CallNode (location: (3,15)-(3,25)) + │ │ │ ├── flags: ignore_visibility + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :tap + │ │ │ ├── message_loc: (3,15)-(3,18) = "tap" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ └── block: + │ │ │ @ BlockNode (location: (3,19)-(3,25)) + │ │ │ ├── locals: [] + │ │ │ ├── parameters: ∅ + │ │ │ ├── body: ∅ + │ │ │ ├── opening_loc: (3,19)-(3,21) = "do" + │ │ │ └── closing_loc: (3,22)-(3,25) = "end" + │ │ ├── locals: [] + │ │ ├── def_keyword_loc: (3,6)-(3,9) = "def" + │ │ ├── operator_loc: ∅ + │ │ ├── lparen_loc: ∅ + │ │ ├── rparen_loc: ∅ + │ │ ├── equal_loc: ∅ + │ │ └── end_keyword_loc: (3,27)-(3,30) = "end" + │ └── statements: + │ @ StatementsNode (location: (3,32)-(3,37)) + │ └── body: (length: 1) + │ └── @ BreakNode (location: (3,32)-(3,37)) + │ ├── arguments: ∅ + │ └── keyword_loc: (3,32)-(3,37) = "break" + ├── @ WhileNode (location: (5,0)-(5,50)) + │ ├── flags: ∅ + │ ├── keyword_loc: (5,0)-(5,5) = "while" + │ ├── closing_loc: (5,47)-(5,50) = "end" + │ ├── predicate: + │ │ @ DefNode (location: (5,6)-(5,38)) + │ │ ├── name: :foo + │ │ ├── name_loc: (5,15)-(5,18) = "foo" + │ │ ├── receiver: + │ │ │ @ SelfNode (location: (5,10)-(5,14)) + │ │ ├── parameters: + │ │ │ @ ParametersNode (location: (5,19)-(5,33)) + │ │ │ ├── requireds: (length: 0) + │ │ │ ├── optionals: (length: 1) + │ │ │ │ └── @ OptionalParameterNode (location: (5,19)-(5,33)) + │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── name: :a + │ │ │ │ ├── name_loc: (5,19)-(5,20) = "a" + │ │ │ │ ├── operator_loc: (5,21)-(5,22) = "=" + │ │ │ │ └── value: + │ │ │ │ @ CallNode (location: (5,23)-(5,33)) + │ │ │ │ ├── flags: ignore_visibility + │ │ │ │ ├── receiver: ∅ + │ │ │ │ ├── call_operator_loc: ∅ + │ │ │ │ ├── name: :tap + │ │ │ │ ├── message_loc: (5,23)-(5,26) = "tap" + │ │ │ │ ├── opening_loc: ∅ + │ │ │ │ ├── arguments: ∅ + │ │ │ │ ├── closing_loc: ∅ + │ │ │ │ └── block: + │ │ │ │ @ BlockNode (location: (5,27)-(5,33)) + │ │ │ │ ├── locals: [] + │ │ │ │ ├── parameters: ∅ + │ │ │ │ ├── body: ∅ + │ │ │ │ ├── opening_loc: (5,27)-(5,29) = "do" + │ │ │ │ └── closing_loc: (5,30)-(5,33) = "end" + │ │ │ ├── rest: ∅ + │ │ │ ├── posts: (length: 0) + │ │ │ ├── keywords: (length: 0) + │ │ │ ├── keyword_rest: ∅ + │ │ │ └── block: ∅ + │ │ ├── body: ∅ + │ │ ├── locals: [:a] + │ │ ├── def_keyword_loc: (5,6)-(5,9) = "def" + │ │ ├── operator_loc: (5,14)-(5,15) = "." + │ │ ├── lparen_loc: ∅ + │ │ ├── rparen_loc: ∅ + │ │ ├── equal_loc: ∅ + │ │ └── end_keyword_loc: (5,35)-(5,38) = "end" + │ └── statements: + │ @ StatementsNode (location: (5,40)-(5,45)) + │ └── body: (length: 1) + │ └── @ BreakNode (location: (5,40)-(5,45)) + │ ├── arguments: ∅ + │ └── keyword_loc: (5,40)-(5,45) = "break" + └── @ WhileNode (location: (7,0)-(7,47)) + ├── flags: ∅ + ├── keyword_loc: (7,0)-(7,5) = "while" + ├── closing_loc: (7,44)-(7,47) = "end" + ├── predicate: + │ @ DefNode (location: (7,6)-(7,35)) + │ ├── name: :foo + │ ├── name_loc: (7,15)-(7,18) = "foo" + │ ├── receiver: + │ │ @ SelfNode (location: (7,10)-(7,14)) + │ ├── parameters: ∅ + │ ├── body: + │ │ @ StatementsNode (location: (7,20)-(7,30)) + │ │ └── body: (length: 1) + │ │ └── @ CallNode (location: (7,20)-(7,30)) + │ │ ├── flags: ignore_visibility + │ │ ├── receiver: ∅ + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :tap + │ │ ├── message_loc: (7,20)-(7,23) = "tap" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: ∅ + │ │ ├── closing_loc: ∅ + │ │ └── block: + │ │ @ BlockNode (location: (7,24)-(7,30)) + │ │ ├── locals: [] + │ │ ├── parameters: ∅ + │ │ ├── body: ∅ + │ │ ├── opening_loc: (7,24)-(7,26) = "do" + │ │ └── closing_loc: (7,27)-(7,30) = "end" + │ ├── locals: [] + │ ├── def_keyword_loc: (7,6)-(7,9) = "def" + │ ├── operator_loc: (7,14)-(7,15) = "." + │ ├── lparen_loc: ∅ + │ ├── rparen_loc: ∅ + │ ├── equal_loc: ∅ + │ └── end_keyword_loc: (7,32)-(7,35) = "end" + └── statements: + @ StatementsNode (location: (7,37)-(7,42)) + └── body: (length: 1) + └── @ BreakNode (location: (7,37)-(7,42)) + ├── arguments: ∅ + └── keyword_loc: (7,37)-(7,42) = "break" diff --git a/test/prism/snapshots/whitequark/op_asgn.txt b/test/prism/snapshots/whitequark/op_asgn.txt index 8f24a35ad0..f726617904 100644 --- a/test/prism/snapshots/whitequark/op_asgn.txt +++ b/test/prism/snapshots/whitequark/op_asgn.txt @@ -20,8 +20,8 @@ │ ├── message_loc: (1,4)-(1,5) = "A" │ ├── read_name: :A │ ├── write_name: :A= - │ ├── operator: :+ - │ ├── operator_loc: (1,6)-(1,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (1,6)-(1,8) = "+=" │ └── value: │ @ IntegerNode (location: (1,9)-(1,10)) │ ├── flags: decimal @@ -43,8 +43,8 @@ │ ├── message_loc: (3,4)-(3,5) = "a" │ ├── read_name: :a │ ├── write_name: :a= - │ ├── operator: :+ - │ ├── operator_loc: (3,6)-(3,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (3,6)-(3,8) = "+=" │ └── value: │ @ IntegerNode (location: (3,9)-(3,10)) │ ├── flags: decimal @@ -66,8 +66,8 @@ ├── message_loc: (5,5)-(5,6) = "a" ├── read_name: :a ├── write_name: :a= - ├── operator: :+ - ├── operator_loc: (5,7)-(5,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (5,7)-(5,9) = "+=" └── value: @ IntegerNode (location: (5,10)-(5,11)) ├── flags: decimal diff --git a/test/prism/snapshots/whitequark/op_asgn_cmd.txt b/test/prism/snapshots/whitequark/op_asgn_cmd.txt index 7d5ad21d55..d2d86b1bf9 100644 --- a/test/prism/snapshots/whitequark/op_asgn_cmd.txt +++ b/test/prism/snapshots/whitequark/op_asgn_cmd.txt @@ -20,8 +20,8 @@ │ ├── message_loc: (1,4)-(1,5) = "A" │ ├── read_name: :A │ ├── write_name: :A= - │ ├── operator: :+ - │ ├── operator_loc: (1,6)-(1,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (1,6)-(1,8) = "+=" │ └── value: │ @ CallNode (location: (1,9)-(1,14)) │ ├── flags: ignore_visibility @@ -63,8 +63,8 @@ │ ├── message_loc: (3,4)-(3,5) = "a" │ ├── read_name: :a │ ├── write_name: :a= - │ ├── operator: :+ - │ ├── operator_loc: (3,6)-(3,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (3,6)-(3,8) = "+=" │ └── value: │ @ CallNode (location: (3,9)-(3,14)) │ ├── flags: ignore_visibility @@ -106,7 +106,7 @@ │ │ ├── name: :A │ │ ├── delimiter_loc: (5,3)-(5,5) = "::" │ │ └── name_loc: (5,5)-(5,6) = "A" - │ ├── operator_loc: (5,7)-(5,9) = "+=" + │ ├── binary_operator_loc: (5,7)-(5,9) = "+=" │ ├── value: │ │ @ CallNode (location: (5,10)-(5,15)) │ │ ├── flags: ignore_visibility @@ -131,7 +131,7 @@ │ │ │ └── block: ∅ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ - │ └── operator: :+ + │ └── binary_operator: :+ └── @ CallOperatorWriteNode (location: (7,0)-(7,15)) ├── flags: ∅ ├── receiver: @@ -149,8 +149,8 @@ ├── message_loc: (7,5)-(7,6) = "a" ├── read_name: :a ├── write_name: :a= - ├── operator: :+ - ├── operator_loc: (7,7)-(7,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (7,7)-(7,9) = "+=" └── value: @ CallNode (location: (7,10)-(7,15)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/whitequark/op_asgn_index.txt b/test/prism/snapshots/whitequark/op_asgn_index.txt index fe077fae13..b302abdafe 100644 --- a/test/prism/snapshots/whitequark/op_asgn_index.txt +++ b/test/prism/snapshots/whitequark/op_asgn_index.txt @@ -30,8 +30,8 @@ │ └── value: 1 ├── closing_loc: (1,8)-(1,9) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,10)-(1,12) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,10)-(1,12) = "+=" └── value: @ IntegerNode (location: (1,13)-(1,14)) ├── flags: decimal diff --git a/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt b/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt index 87082aad94..319ed1a51a 100644 --- a/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt +++ b/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt @@ -30,8 +30,8 @@ │ └── value: 1 ├── closing_loc: (1,8)-(1,9) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,10)-(1,12) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,10)-(1,12) = "+=" └── value: @ CallNode (location: (1,13)-(1,18)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/whitequark/rational.txt b/test/prism/snapshots/whitequark/rational.txt index 90bbd17929..e8c8eed508 100644 --- a/test/prism/snapshots/whitequark/rational.txt +++ b/test/prism/snapshots/whitequark/rational.txt @@ -4,11 +4,10 @@ @ StatementsNode (location: (1,0)-(3,3)) └── body: (length: 2) ├── @ RationalNode (location: (1,0)-(1,5)) - │ └── numeric: - │ @ FloatNode (location: (1,0)-(1,4)) - │ └── value: 42.1 + │ ├── flags: decimal + │ ├── numerator: 421 + │ └── denominator: 10 └── @ RationalNode (location: (3,0)-(3,3)) - └── numeric: - @ IntegerNode (location: (3,0)-(3,2)) - ├── flags: decimal - └── value: 42 + ├── flags: decimal + ├── numerator: 42 + └── denominator: 1 diff --git a/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt b/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt index b269104f30..840e5a4fc0 100644 --- a/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt +++ b/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt @@ -5,7 +5,7 @@ └── body: (length: 1) └── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,22)) ├── name_loc: (1,0)-(1,3) = "foo" - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator_loc: (1,4)-(1,6) = "+=" ├── value: │ @ RescueModifierNode (location: (1,7)-(1,22)) │ ├── expression: @@ -32,5 +32,5 @@ │ ├── closing_loc: ∅ │ └── block: ∅ ├── name: :foo - ├── operator: :+ + ├── binary_operator: :+ └── depth: 0 diff --git a/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt b/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt index 93418e6448..831d57e30d 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt @@ -225,9 +225,9 @@ │ │ │ ├── closing_loc: (7,7)-(7,8) = ")" │ │ │ └── block: ∅ │ │ └── @ RationalNode (location: (7,10)-(7,14)) - │ │ └── numeric: - │ │ @ FloatNode (location: (7,10)-(7,13)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (7,15)-(7,21)) @@ -519,9 +519,9 @@ │ │ │ ├── closing_loc: (17,8)-(17,9) = ")" │ │ │ └── block: ∅ │ │ └── @ RationalNode (location: (17,11)-(17,15)) - │ │ └── numeric: - │ │ @ FloatNode (location: (17,11)-(17,14)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (17,16)-(17,22)) @@ -833,9 +833,9 @@ │ │ │ ├── opening_loc: (27,3)-(27,4) = "{" │ │ │ └── closing_loc: (27,7)-(27,8) = "}" │ │ └── @ RationalNode (location: (27,10)-(27,14)) - │ │ └── numeric: - │ │ @ FloatNode (location: (27,10)-(27,13)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (27,15)-(27,21)) @@ -1152,9 +1152,9 @@ │ │ │ ├── opening_loc: (37,3)-(37,4) = "{" │ │ │ └── closing_loc: (37,8)-(37,9) = "}" │ │ └── @ RationalNode (location: (37,11)-(37,15)) - │ │ └── numeric: - │ │ @ FloatNode (location: (37,11)-(37,14)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (37,16)-(37,22)) diff --git a/test/prism/snapshots/whitequark/ruby_bug_12402.txt b/test/prism/snapshots/whitequark/ruby_bug_12402.txt index 20bcfeafbe..4935007f8a 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_12402.txt @@ -5,7 +5,7 @@ └── body: (length: 14) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,27)) │ ├── name_loc: (1,0)-(1,3) = "foo" - │ ├── operator_loc: (1,4)-(1,6) = "+=" + │ ├── binary_operator_loc: (1,4)-(1,6) = "+=" │ ├── value: │ │ @ RescueModifierNode (location: (1,7)-(1,27)) │ │ ├── expression: @@ -36,11 +36,11 @@ │ │ └── rescue_expression: │ │ @ NilNode (location: (1,24)-(1,27)) │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,28)) │ ├── name_loc: (3,0)-(3,3) = "foo" - │ ├── operator_loc: (3,4)-(3,6) = "+=" + │ ├── binary_operator_loc: (3,4)-(3,6) = "+=" │ ├── value: │ │ @ RescueModifierNode (location: (3,7)-(3,28)) │ │ ├── expression: @@ -71,7 +71,7 @@ │ │ └── rescue_expression: │ │ @ NilNode (location: (3,25)-(3,28)) │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableWriteNode (location: (5,0)-(5,26)) │ ├── name: :foo @@ -151,8 +151,8 @@ │ ├── message_loc: (9,4)-(9,5) = "C" │ ├── read_name: :C │ ├── write_name: :C= - │ ├── operator: :+ - │ ├── operator_loc: (9,6)-(9,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (9,6)-(9,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (9,9)-(9,29)) │ ├── expression: @@ -192,8 +192,8 @@ │ ├── message_loc: (11,4)-(11,5) = "C" │ ├── read_name: :C │ ├── write_name: :C= - │ ├── operator: :+ - │ ├── operator_loc: (11,6)-(11,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (11,6)-(11,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (11,9)-(11,30)) │ ├── expression: @@ -233,8 +233,8 @@ │ ├── message_loc: (13,4)-(13,5) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (13,6)-(13,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (13,6)-(13,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (13,9)-(13,29)) │ ├── expression: @@ -274,8 +274,8 @@ │ ├── message_loc: (15,4)-(15,5) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (15,6)-(15,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (15,6)-(15,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (15,9)-(15,30)) │ ├── expression: @@ -395,8 +395,8 @@ │ ├── message_loc: (21,5)-(21,6) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (21,7)-(21,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (21,7)-(21,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (21,10)-(21,30)) │ ├── expression: @@ -436,8 +436,8 @@ │ ├── message_loc: (23,5)-(23,6) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (23,7)-(23,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (23,7)-(23,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (23,10)-(23,31)) │ ├── expression: @@ -484,8 +484,8 @@ │ │ └── value: 0 │ ├── closing_loc: (25,5)-(25,6) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (25,7)-(25,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (25,7)-(25,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (25,10)-(25,30)) │ ├── expression: @@ -532,8 +532,8 @@ │ └── value: 0 ├── closing_loc: (27,5)-(27,6) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (27,7)-(27,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (27,7)-(27,9) = "+=" └── value: @ RescueModifierNode (location: (27,10)-(27,31)) ├── expression: diff --git a/test/prism/snapshots/whitequark/ruby_bug_12669.txt b/test/prism/snapshots/whitequark/ruby_bug_12669.txt index 86b021351b..6151143ba8 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_12669.txt @@ -5,11 +5,11 @@ └── body: (length: 4) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,18)) │ ├── name_loc: (1,0)-(1,1) = "a" - │ ├── operator_loc: (1,2)-(1,4) = "+=" + │ ├── binary_operator_loc: (1,2)-(1,4) = "+=" │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (1,5)-(1,18)) │ │ ├── name_loc: (1,5)-(1,6) = "b" - │ │ ├── operator_loc: (1,7)-(1,9) = "+=" + │ │ ├── binary_operator_loc: (1,7)-(1,9) = "+=" │ │ ├── value: │ │ │ @ CallNode (location: (1,10)-(1,18)) │ │ │ ├── flags: ignore_visibility @@ -31,14 +31,14 @@ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ │ │ ├── name: :b - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,17)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ LocalVariableWriteNode (location: (3,5)-(3,17)) │ │ ├── name: :b @@ -66,7 +66,7 @@ │ │ │ └── block: ∅ │ │ └── operator_loc: (3,7)-(3,8) = "=" │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableWriteNode (location: (5,0)-(5,17)) │ ├── name: :a @@ -75,7 +75,7 @@ │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (5,4)-(5,17)) │ │ ├── name_loc: (5,4)-(5,5) = "b" - │ │ ├── operator_loc: (5,6)-(5,8) = "+=" + │ │ ├── binary_operator_loc: (5,6)-(5,8) = "+=" │ │ ├── value: │ │ │ @ CallNode (location: (5,9)-(5,17)) │ │ │ ├── flags: ignore_visibility @@ -97,7 +97,7 @@ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ │ │ ├── name: :b - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ └── operator_loc: (5,2)-(5,3) = "=" └── @ LocalVariableWriteNode (location: (7,0)-(7,16)) diff --git a/test/prism/snapshots/whitequark/var_op_asgn.txt b/test/prism/snapshots/whitequark/var_op_asgn.txt index f423a62dee..f20f612fa2 100644 --- a/test/prism/snapshots/whitequark/var_op_asgn.txt +++ b/test/prism/snapshots/whitequark/var_op_asgn.txt @@ -6,30 +6,30 @@ ├── @ ClassVariableOperatorWriteNode (location: (1,0)-(1,11)) │ ├── name: :@@var │ ├── name_loc: (1,0)-(1,5) = "@@var" - │ ├── operator_loc: (1,6)-(1,8) = "|=" + │ ├── binary_operator_loc: (1,6)-(1,8) = "|=" │ ├── value: │ │ @ IntegerNode (location: (1,9)-(1,11)) │ │ ├── flags: decimal │ │ └── value: 10 - │ └── operator: :| + │ └── binary_operator: :| ├── @ InstanceVariableOperatorWriteNode (location: (3,0)-(3,7)) │ ├── name: :@a │ ├── name_loc: (3,0)-(3,2) = "@a" - │ ├── operator_loc: (3,3)-(3,5) = "|=" + │ ├── binary_operator_loc: (3,3)-(3,5) = "|=" │ ├── value: │ │ @ IntegerNode (location: (3,6)-(3,7)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :| + │ └── binary_operator: :| ├── @ LocalVariableOperatorWriteNode (location: (5,0)-(5,6)) │ ├── name_loc: (5,0)-(5,1) = "a" - │ ├── operator_loc: (5,2)-(5,4) = "+=" + │ ├── binary_operator_loc: (5,2)-(5,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (5,5)-(5,6)) │ │ ├── flags: decimal │ │ └── value: 1 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ DefNode (location: (7,0)-(7,23)) ├── name: :a @@ -42,12 +42,12 @@ │ └── @ ClassVariableOperatorWriteNode (location: (7,7)-(7,18)) │ ├── name: :@@var │ ├── name_loc: (7,7)-(7,12) = "@@var" - │ ├── operator_loc: (7,13)-(7,15) = "|=" + │ ├── binary_operator_loc: (7,13)-(7,15) = "|=" │ ├── value: │ │ @ IntegerNode (location: (7,16)-(7,18)) │ │ ├── flags: decimal │ │ └── value: 10 - │ └── operator: :| + │ └── binary_operator: :| ├── locals: [] ├── def_keyword_loc: (7,0)-(7,3) = "def" ├── operator_loc: ∅ diff --git a/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt b/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt index d56c099c7e..0bfa06d5b7 100644 --- a/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt +++ b/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt @@ -5,7 +5,7 @@ └── body: (length: 1) └── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,12)) ├── name_loc: (1,0)-(1,3) = "foo" - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator_loc: (1,4)-(1,6) = "+=" ├── value: │ @ CallNode (location: (1,7)-(1,12)) │ ├── flags: ignore_visibility @@ -24,5 +24,5 @@ │ ├── closing_loc: ∅ │ └── block: ∅ ├── name: :foo - ├── operator: :+ + ├── binary_operator: :+ └── depth: 0 diff --git a/test/prism/snapshots/whitequark/yield.txt b/test/prism/snapshots/whitequark/yield.txt deleted file mode 100644 index 2b37dd479f..0000000000 --- a/test/prism/snapshots/whitequark/yield.txt +++ /dev/null @@ -1,51 +0,0 @@ -@ ProgramNode (location: (1,0)-(7,10)) -├── locals: [] -└── statements: - @ StatementsNode (location: (1,0)-(7,10)) - └── body: (length: 4) - ├── @ YieldNode (location: (1,0)-(1,5)) - │ ├── keyword_loc: (1,0)-(1,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: ∅ - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (3,0)-(3,9)) - │ ├── keyword_loc: (3,0)-(3,5) = "yield" - │ ├── lparen_loc: ∅ - │ ├── arguments: - │ │ @ ArgumentsNode (location: (3,6)-(3,9)) - │ │ ├── flags: ∅ - │ │ └── arguments: (length: 1) - │ │ └── @ CallNode (location: (3,6)-(3,9)) - │ │ ├── flags: variable_call, ignore_visibility - │ │ ├── receiver: ∅ - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :foo - │ │ ├── message_loc: (3,6)-(3,9) = "foo" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── rparen_loc: ∅ - ├── @ YieldNode (location: (5,0)-(5,7)) - │ ├── keyword_loc: (5,0)-(5,5) = "yield" - │ ├── lparen_loc: (5,5)-(5,6) = "(" - │ ├── arguments: ∅ - │ └── rparen_loc: (5,6)-(5,7) = ")" - └── @ YieldNode (location: (7,0)-(7,10)) - ├── keyword_loc: (7,0)-(7,5) = "yield" - ├── lparen_loc: (7,5)-(7,6) = "(" - ├── arguments: - │ @ ArgumentsNode (location: (7,6)-(7,9)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ CallNode (location: (7,6)-(7,9)) - │ ├── flags: variable_call, ignore_visibility - │ ├── receiver: ∅ - │ ├── call_operator_loc: ∅ - │ ├── name: :foo - │ ├── message_loc: (7,6)-(7,9) = "foo" - │ ├── opening_loc: ∅ - │ ├── arguments: ∅ - │ ├── closing_loc: ∅ - │ └── block: ∅ - └── rparen_loc: (7,9)-(7,10) = ")" diff --git a/test/prism/snapshots_test.rb b/test/prism/snapshots_test.rb new file mode 100644 index 0000000000..0744eafad3 --- /dev/null +++ b/test/prism/snapshots_test.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module Prism + class SnapshotsTest < TestCase + # When we pretty-print the trees to compare against the snapshots, we want + # to be certain that we print with the same external encoding. This is + # because methods like Symbol#inspect take into account external encoding + # and it could change how the snapshot is generated. On machines with + # certain settings (like LANG=C or -Eascii-8bit) this could have been + # changed. So here we're going to force it to be UTF-8 to keep the snapshots + # consistent. + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end + + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end + + except = [] + + # These fail on TruffleRuby due to a difference in Symbol#inspect: + # :测试 vs :"测试" + if RUBY_ENGINE == "truffleruby" + except.push( + "emoji_method_calls.txt", + "seattlerb/bug202.txt", + "seattlerb/magic_encoding_comment.txt" + ) + end + + Fixture.each(except: except) do |fixture| + define_method(fixture.test_name) { assert_snapshot(fixture) } + end + + private + + def assert_snapshot(fixture) + source = fixture.read + + result = Prism.parse(source, filepath: fixture.path) + assert result.success? + + printed = PP.pp(result.value, +"", 79) + snapshot = fixture.snapshot_path + + if File.exist?(snapshot) + saved = File.read(snapshot) + + # If the snapshot file exists, but the printed value does not match the + # snapshot, then update the snapshot file. + if printed != saved + File.write(snapshot, printed) + warn("Updated snapshot at #{snapshot}.") + end + + # If the snapshot file exists, then assert that the printed value + # matches the snapshot. + assert_equal(saved, printed) + else + # If the snapshot file does not yet exist, then write it out now. + directory = File.dirname(snapshot) + FileUtils.mkdir_p(directory) unless File.directory?(directory) + + File.write(snapshot, printed) + warn("Created snapshot at #{snapshot}.") + end + end + end +end diff --git a/test/prism/snippets_test.rb b/test/prism/snippets_test.rb new file mode 100644 index 0000000000..26847da184 --- /dev/null +++ b/test/prism/snippets_test.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module Prism + class SnippetsTest < TestCase + except = [ + "newline_terminated.txt", + "seattlerb/begin_rescue_else_ensure_no_bodies.txt", + "seattlerb/case_in.txt", + "seattlerb/parse_line_defn_no_parens.txt", + "seattlerb/pct_nl.txt", + "seattlerb/str_heredoc_interp.txt", + "spanning_heredoc_newlines.txt", + "unparser/corpus/semantic/dstr.txt", + "whitequark/dedenting_heredoc.txt", + "whitequark/multiple_pattern_matches.txt" + ] + + Fixture.each(except: except) do |fixture| + define_method(fixture.test_name) { assert_snippets(fixture) } + end + + private + + # We test every snippet (separated by \n\n) in isolation to ensure the + # parser does not try to read bytes further than the end of each snippet. + def assert_snippets(fixture) + fixture.read.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| + snippet = snippet.rstrip + + result = Prism.parse(snippet, filepath: fixture.path) + assert result.success? + + if !ENV["PRISM_BUILD_MINIMAL"] + dumped = Prism.dump(snippet, filepath: fixture.path) + assert_equal_nodes(result.value, Prism.load(snippet, dumped).value) + end + end + end + end +end diff --git a/test/prism/test_helper.rb b/test/prism/test_helper.rb index 77af7e7b45..d6d0abf548 100644 --- a/test/prism/test_helper.rb +++ b/test/prism/test_helper.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true require "prism" -require "ripper" require "pp" +require "ripper" +require "stringio" require "test/unit" require "tempfile" @@ -16,19 +17,202 @@ if defined?(Test::Unit::Assertions::AssertionMessage) end module Prism + # A convenience method for retrieving the first statement in the source string + # parsed by Prism. + def self.parse_statement(source, **options) + parse(source, **options).value.statements.body.first + end + + class ParseResult < Result + # Returns the first statement in the body of the parsed source. + def statement + value.statements.body.first + end + end + class TestCase < ::Test::Unit::TestCase + # We have a set of fixtures that we use to test various aspects of the + # parser. They are all represented as .txt files under the + # test/prism/fixtures directory. Typically in test files you will find calls + # to Fixture.each which yields Fixture objects to the given block. These + # are used to define test methods that assert against each fixture in some + # way. + class Fixture + BASE = File.join(__dir__, "fixtures") + + attr_reader :path + + def initialize(path) + @path = path + end + + def read + File.read(full_path, binmode: true, external_encoding: Encoding::UTF_8) + end + + def full_path + File.join(BASE, path) + end + + def snapshot_path + File.join(__dir__, "snapshots", path) + end + + def test_name + :"test_#{path}" + end + + def self.each(except: [], &block) + paths = Dir[ENV.fetch("FOCUS") { File.join("**", "*.txt") }, base: BASE] - except + paths.each { |path| yield Fixture.new(path) } + end + end + + # Yield each encoding that we want to test, along with a range of the + # codepoints that should be tested. + def self.each_encoding + codepoints_1byte = 0...0x100 + + yield Encoding::ASCII_8BIT, codepoints_1byte + yield Encoding::US_ASCII, codepoints_1byte + + if !ENV["PRISM_BUILD_MINIMAL"] + yield Encoding::Windows_1253, codepoints_1byte + end + + # By default we don't test every codepoint in these encodings because it + # takes a very long time. + return unless ENV["PRISM_TEST_ALL_ENCODINGS"] + + yield Encoding::CP850, codepoints_1byte + yield Encoding::CP852, codepoints_1byte + yield Encoding::CP855, codepoints_1byte + yield Encoding::GB1988, codepoints_1byte + yield Encoding::IBM437, codepoints_1byte + yield Encoding::IBM720, codepoints_1byte + yield Encoding::IBM737, codepoints_1byte + yield Encoding::IBM775, codepoints_1byte + yield Encoding::IBM852, codepoints_1byte + yield Encoding::IBM855, codepoints_1byte + yield Encoding::IBM857, codepoints_1byte + yield Encoding::IBM860, codepoints_1byte + yield Encoding::IBM861, codepoints_1byte + yield Encoding::IBM862, codepoints_1byte + yield Encoding::IBM863, codepoints_1byte + yield Encoding::IBM864, codepoints_1byte + yield Encoding::IBM865, codepoints_1byte + yield Encoding::IBM866, codepoints_1byte + yield Encoding::IBM869, codepoints_1byte + yield Encoding::ISO_8859_1, codepoints_1byte + yield Encoding::ISO_8859_2, codepoints_1byte + yield Encoding::ISO_8859_3, codepoints_1byte + yield Encoding::ISO_8859_4, codepoints_1byte + yield Encoding::ISO_8859_5, codepoints_1byte + yield Encoding::ISO_8859_6, codepoints_1byte + yield Encoding::ISO_8859_7, codepoints_1byte + yield Encoding::ISO_8859_8, codepoints_1byte + yield Encoding::ISO_8859_9, codepoints_1byte + yield Encoding::ISO_8859_10, codepoints_1byte + yield Encoding::ISO_8859_11, codepoints_1byte + yield Encoding::ISO_8859_13, codepoints_1byte + yield Encoding::ISO_8859_14, codepoints_1byte + yield Encoding::ISO_8859_15, codepoints_1byte + yield Encoding::ISO_8859_16, codepoints_1byte + yield Encoding::KOI8_R, codepoints_1byte + yield Encoding::KOI8_U, codepoints_1byte + yield Encoding::MACCENTEURO, codepoints_1byte + yield Encoding::MACCROATIAN, codepoints_1byte + yield Encoding::MACCYRILLIC, codepoints_1byte + yield Encoding::MACGREEK, codepoints_1byte + yield Encoding::MACICELAND, codepoints_1byte + yield Encoding::MACROMAN, codepoints_1byte + yield Encoding::MACROMANIA, codepoints_1byte + yield Encoding::MACTHAI, codepoints_1byte + yield Encoding::MACTURKISH, codepoints_1byte + yield Encoding::MACUKRAINE, codepoints_1byte + yield Encoding::TIS_620, codepoints_1byte + yield Encoding::Windows_1250, codepoints_1byte + yield Encoding::Windows_1251, codepoints_1byte + yield Encoding::Windows_1252, codepoints_1byte + yield Encoding::Windows_1254, codepoints_1byte + yield Encoding::Windows_1255, codepoints_1byte + yield Encoding::Windows_1256, codepoints_1byte + yield Encoding::Windows_1257, codepoints_1byte + yield Encoding::Windows_1258, codepoints_1byte + yield Encoding::Windows_874, codepoints_1byte + + codepoints_2bytes = 0...0x10000 + + yield Encoding::Big5, codepoints_2bytes + yield Encoding::Big5_HKSCS, codepoints_2bytes + yield Encoding::Big5_UAO, codepoints_2bytes + yield Encoding::CP949, codepoints_2bytes + yield Encoding::CP950, codepoints_2bytes + yield Encoding::CP951, codepoints_2bytes + yield Encoding::EUC_KR, codepoints_2bytes + yield Encoding::GBK, codepoints_2bytes + yield Encoding::GB12345, codepoints_2bytes + yield Encoding::GB2312, codepoints_2bytes + yield Encoding::MACJAPANESE, codepoints_2bytes + yield Encoding::Shift_JIS, codepoints_2bytes + yield Encoding::SJIS_DoCoMo, codepoints_2bytes + yield Encoding::SJIS_KDDI, codepoints_2bytes + yield Encoding::SJIS_SoftBank, codepoints_2bytes + yield Encoding::Windows_31J, codepoints_2bytes + + codepoints_unicode = (0...0x110000) + + yield Encoding::UTF_8, codepoints_unicode + yield Encoding::UTF8_MAC, codepoints_unicode + yield Encoding::UTF8_DoCoMo, codepoints_unicode + yield Encoding::UTF8_KDDI, codepoints_unicode + yield Encoding::UTF8_SoftBank, codepoints_unicode + yield Encoding::CESU_8, codepoints_unicode + + codepoints_eucjp = [ + *(0...0x10000), + *(0...0x10000).map { |bytes| bytes | 0x8F0000 } + ] + + yield Encoding::CP51932, codepoints_eucjp + yield Encoding::EUC_JP, codepoints_eucjp + yield Encoding::EUCJP_MS, codepoints_eucjp + yield Encoding::EUC_JIS_2004, codepoints_eucjp + + codepoints_emacs_mule = [ + *(0...0x80), + *((0x81...0x90).flat_map { |byte1| (0x90...0x100).map { |byte2| byte1 << 8 | byte2 } }), + *((0x90...0x9C).flat_map { |byte1| (0xA0...0x100).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| byte1 << 16 | byte2 << 8 | byte3 } } }), + *((0xF0...0xF5).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| (0xA0...0x100).flat_map { |byte4| 0x9C << 24 | byte3 << 16 | byte3 << 8 | byte4 } } }), + ] + + yield Encoding::EMACS_MULE, codepoints_emacs_mule + yield Encoding::STATELESS_ISO_2022_JP, codepoints_emacs_mule + yield Encoding::STATELESS_ISO_2022_JP_KDDI, codepoints_emacs_mule + + codepoints_gb18030 = [ + *(0...0x80), + *((0x81..0xFE).flat_map { |byte1| (0x40...0x100).map { |byte2| byte1 << 8 | byte2 } }), + *((0x81..0xFE).flat_map { |byte1| (0x30...0x40).flat_map { |byte2| (0x81..0xFE).flat_map { |byte3| (0x2F...0x41).map { |byte4| byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } }), + ] + + yield Encoding::GB18030, codepoints_gb18030 + + codepoints_euc_tw = [ + *(0..0x7F), + *(0xA1..0xFF).flat_map { |byte1| (0xA1..0xFF).map { |byte2| (byte1 << 8) | byte2 } }, + *(0xA1..0xB0).flat_map { |byte2| (0xA1..0xFF).flat_map { |byte3| (0xA1..0xFF).flat_map { |byte4| 0x8E << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } + ] + + yield Encoding::EUC_TW, codepoints_euc_tw + end + private if RUBY_ENGINE == "ruby" # Check that the given source is valid syntax by compiling it with RubyVM. def check_syntax(source) - $VERBOSE, previous = nil, $VERBOSE - - begin - RubyVM::InstructionSequence.compile(source) - ensure - $VERBOSE = previous - end + ignore_warnings { RubyVM::InstructionSequence.compile(source) } end # Assert that the given source is valid Ruby syntax by attempting to @@ -51,6 +235,8 @@ module Prism end end + # CRuby has this same method, so define it so that we don't accidentally + # break CRuby CI. def assert_raises(*args, &block) raise "Use assert_raise instead" end @@ -122,5 +308,16 @@ module Prism assert_equal expected, actual end end + + def ignore_warnings + previous = $VERBOSE + $VERBOSE = nil + + begin + yield + ensure + $VERBOSE = previous + end + end end end diff --git a/test/prism/unescape_test.rb b/test/prism/unescape_test.rb index 3f78a59b11..24a9e3f6bc 100644 --- a/test/prism/unescape_test.rb +++ b/test/prism/unescape_test.rb @@ -41,7 +41,7 @@ module Prism result = Prism.parse(code(escape), encoding: "binary") if result.success? - yield result.value.statements.body.first + yield result.statement else :error end diff --git a/test/reline/helper.rb b/test/reline/helper.rb index 26fe834482..9a346a17a1 100644 --- a/test/reline/helper.rb +++ b/test/reline/helper.rb @@ -22,29 +22,36 @@ module Reline class <<self def test_mode(ansi: false) @original_iogate = IOGate - remove_const('IOGate') - const_set('IOGate', ansi ? Reline::ANSI : Reline::GeneralIO) + if ENV['RELINE_TEST_ENCODING'] encoding = Encoding.find(ENV['RELINE_TEST_ENCODING']) else encoding = Encoding::UTF_8 end - @original_get_screen_size = IOGate.method(:get_screen_size) - IOGate.singleton_class.remove_method(:get_screen_size) - def IOGate.get_screen_size - [24, 80] + + if ansi + new_io_gate = ANSI.new + # Setting ANSI gate's screen size through set_screen_size will also change the tester's stdin's screen size + # Let's avoid that side-effect by stubbing the get_screen_size method + new_io_gate.define_singleton_method(:get_screen_size) do + [24, 80] + end + new_io_gate.define_singleton_method(:encoding) do + encoding + end + else + new_io_gate = Dumb.new(encoding: encoding) end - Reline::GeneralIO.reset(encoding: encoding) unless ansi + + remove_const('IOGate') + const_set('IOGate', new_io_gate) core.config.instance_variable_set(:@test_mode, true) core.config.reset end def test_reset - IOGate.singleton_class.remove_method(:get_screen_size) - IOGate.define_singleton_method(:get_screen_size, @original_get_screen_size) remove_const('IOGate') const_set('IOGate', @original_iogate) - Reline::GeneralIO.reset Reline.instance_variable_set(:@core, nil) end @@ -146,7 +153,7 @@ class Reline::TestCase < Test::Unit::TestCase expected.bytesize, byte_pointer, <<~EOM) <#{expected.inspect} (#{expected.encoding.inspect})> expected but was - <#{chunk.inspect} (#{chunk.encoding.inspect})> in <Terminal #{Reline::GeneralIO.encoding.inspect}> + <#{chunk.inspect} (#{chunk.encoding.inspect})> in <Terminal #{Reline::Dumb.new.encoding.inspect}> EOM end @@ -161,7 +168,7 @@ class Reline::TestCase < Test::Unit::TestCase def assert_key_binding(input, method_symbol, editing_modes = [:emacs, :vi_insert, :vi_command]) editing_modes.each do |editing_mode| @config.editing_mode = editing_mode - assert_equal(method_symbol, @config.editing_mode.default_key_bindings[input.bytes]) + assert_equal(method_symbol, @config.editing_mode.get(input.bytes)) end end end diff --git a/test/reline/test_ansi_with_terminfo.rb b/test/reline/test_ansi_with_terminfo.rb index e1c56b9ee1..3adda10716 100644 --- a/test/reline/test_ansi_with_terminfo.rb +++ b/test/reline/test_ansi_with_terminfo.rb @@ -1,7 +1,7 @@ require_relative 'helper' -require 'reline/ansi' +require 'reline' -class Reline::ANSI::TestWithTerminfo < Reline::TestCase +class Reline::ANSI::WithTerminfoTest < Reline::TestCase def setup Reline.send(:test_mode, ansi: true) @config = Reline::Config.new diff --git a/test/reline/test_ansi_without_terminfo.rb b/test/reline/test_ansi_without_terminfo.rb index 3d153514f3..2db14cf0a0 100644 --- a/test/reline/test_ansi_without_terminfo.rb +++ b/test/reline/test_ansi_without_terminfo.rb @@ -1,7 +1,7 @@ require_relative 'helper' -require 'reline/ansi' +require 'reline' -class Reline::ANSI::TestWithoutTerminfo < Reline::TestCase +class Reline::ANSI::WithoutTerminfoTest < Reline::TestCase def setup Reline.send(:test_mode, ansi: true) @config = Reline::Config.new diff --git a/test/reline/test_config.rb b/test/reline/test_config.rb index 6068292847..16727c9bc9 100644 --- a/test/reline/test_config.rb +++ b/test/reline/test_config.rb @@ -22,6 +22,15 @@ class Reline::Config::Test < Reline::TestCase @config.reset end + def additional_key_bindings(keymap_label) + @config.instance_variable_get(:@additional_key_bindings)[keymap_label].instance_variable_get(:@key_bindings) + end + + def registered_key_bindings(keys) + key_bindings = @config.key_bindings + keys.to_h { |key| [key, key_bindings.get(key)] } + end + def test_read_lines @config.read_lines(<<~LINES.lines) set bell-style on @@ -85,28 +94,29 @@ class Reline::Config::Test < Reline::TestCase def test_encoding_is_ascii @config.reset - Reline.core.io_gate.reset(encoding: Encoding::US_ASCII) + Reline.core.io_gate.instance_variable_set(:@encoding, Encoding::US_ASCII) @config = Reline::Config.new assert_equal true, @config.convert_meta end def test_encoding_is_not_ascii - @config.reset - Reline.core.io_gate.reset(encoding: Encoding::UTF_8) @config = Reline::Config.new assert_equal nil, @config.convert_meta end - def test_comment_line - @config.read_lines([" #a: error\n"]) - assert_not_include @config.key_bindings, nil - end - def test_invalid_keystroke - @config.read_lines(["a: error\n"]) - assert_not_include @config.key_bindings, nil + @config.read_lines(<<~LINES.lines) + #"a": comment + a: error + "b": no-error + LINES + key_bindings = additional_key_bindings(:emacs) + assert_not_include key_bindings, 'a'.bytes + assert_not_include key_bindings, nil + assert_not_include key_bindings, [] + assert_include key_bindings, 'b'.bytes end def test_bind_key @@ -244,8 +254,8 @@ class Reline::Config::Test < Reline::TestCase "\xC": "O" LINES keys = [0x1, 0x3, 0x4, 0x6, 0x7, 0xC] - key_bindings = keys.to_h { |k| [[k.ord], ['O'.ord]] } - assert_equal(key_bindings, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + key_bindings = keys.to_h { |k| [[k], ['O'.ord]] } + assert_equal(key_bindings, additional_key_bindings(:emacs)) end def test_unclosed_if @@ -284,9 +294,9 @@ class Reline::Config::Test < Reline::TestCase $endif LINES - assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + assert_equal({[5] => :history_search_backward}, additional_key_bindings(:emacs)) + assert_equal({}, additional_key_bindings(:vi_insert)) + assert_equal({}, additional_key_bindings(:vi_command)) end def test_else @@ -298,9 +308,9 @@ class Reline::Config::Test < Reline::TestCase $endif LINES - assert_equal({[6] => :history_search_forward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + assert_equal({[6] => :history_search_forward}, additional_key_bindings(:emacs)) + assert_equal({}, additional_key_bindings(:vi_insert)) + assert_equal({}, additional_key_bindings(:vi_command)) end def test_if_with_invalid_mode @@ -312,9 +322,9 @@ class Reline::Config::Test < Reline::TestCase $endif LINES - assert_equal({[6] => :history_search_forward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + assert_equal({[6] => :history_search_forward}, additional_key_bindings(:emacs)) + assert_equal({}, additional_key_bindings(:vi_insert)) + assert_equal({}, additional_key_bindings(:vi_command)) end def test_mode_label_differs_from_keymap_label @@ -329,9 +339,9 @@ class Reline::Config::Test < Reline::TestCase "\C-e": history-search-backward $endif LINES - assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + assert_equal({[5] => :history_search_backward}, additional_key_bindings(:emacs)) + assert_equal({}, additional_key_bindings(:vi_insert)) + assert_equal({}, additional_key_bindings(:vi_command)) end def test_if_without_else_condition @@ -342,9 +352,9 @@ class Reline::Config::Test < Reline::TestCase $endif LINES - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) - assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) - assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + assert_equal({}, additional_key_bindings(:emacs)) + assert_equal({[5] => :history_search_backward}, additional_key_bindings(:vi_insert)) + assert_equal({}, additional_key_bindings(:vi_command)) end def test_default_key_bindings @@ -355,7 +365,7 @@ class Reline::Config::Test < Reline::TestCase LINES expected = { 'abcd'.bytes => 'ABCD'.bytes, 'ijkl'.bytes => 'IJKL'.bytes } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_additional_key_bindings @@ -365,7 +375,7 @@ class Reline::Config::Test < Reline::TestCase LINES expected = { 'ef'.bytes => 'EF'.bytes, 'gh'.bytes => 'GH'.bytes } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_additional_key_bindings_with_nesting_and_comment_out @@ -377,7 +387,7 @@ class Reline::Config::Test < Reline::TestCase LINES expected = { 'ef'.bytes => 'EF'.bytes, 'gh'.bytes => 'GH'.bytes } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_additional_key_bindings_for_other_keymap @@ -392,7 +402,7 @@ class Reline::Config::Test < Reline::TestCase LINES expected = { 'cd'.bytes => 'CD'.bytes } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_additional_key_bindings_for_auxiliary_emacs_keymaps @@ -414,7 +424,7 @@ class Reline::Config::Test < Reline::TestCase "\C-xef".bytes => 'EF'.bytes, "\egh".bytes => 'GH'.bytes, } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_key_bindings_with_reset @@ -426,7 +436,7 @@ class Reline::Config::Test < Reline::TestCase LINES @config.reset expected = { 'default'.bytes => 'DEFAULT'.bytes, 'additional'.bytes => 'ADDITIONAL'.bytes } - assert_equal expected, @config.key_bindings + assert_equal expected, registered_key_bindings(expected.keys) end def test_history_size @@ -459,6 +469,17 @@ class Reline::Config::Test < Reline::TestCase ENV['INPUTRC'] = inputrc_backup end + def test_inputrc_raw_value + @config.read_lines(<<~'LINES'.lines) + set editing-mode vi ignored-string + set vi-ins-mode-string aaa aaa + set vi-cmd-mode-string bbb ccc # comment + LINES + assert_equal :vi_insert, @config.instance_variable_get(:@editing_mode_label) + assert_equal 'aaa aaa', @config.vi_ins_mode_string + assert_equal 'bbb ccc # comment', @config.vi_cmd_mode_string + end + def test_inputrc_with_utf8 # This file is encoded by UTF-8 so this heredoc string is also UTF-8. @config.read_lines(<<~'LINES'.lines) @@ -542,4 +563,3 @@ class Reline::Config::Test < Reline::TestCase ENV['HOME'] = home_backup end end - diff --git a/test/reline/test_key_actor_emacs.rb b/test/reline/test_key_actor_emacs.rb index 409a7334cb..4dddf9c890 100644 --- a/test/reline/test_key_actor_emacs.rb +++ b/test/reline/test_key_actor_emacs.rb @@ -1,6 +1,6 @@ require_relative 'helper' -class Reline::KeyActor::Emacs::Test < Reline::TestCase +class Reline::KeyActor::EmacsTest < Reline::TestCase def setup Reline.send(:test_mode) @prompt = '> ' @@ -1242,14 +1242,22 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase '12345' # new ]) # The ed_search_prev_history doesn't have default binding - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12345') - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12aaa') - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12356') - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12356') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12345', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12356', '') + input_key_by_symbol(:ed_search_next_history) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_prev_char) + input_key_by_symbol(:ed_next_char) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12aaa', '') + 3.times { input_key_by_symbol(:ed_prev_char) } + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12', '356') end def test_ed_search_prev_history_without_match @@ -1291,18 +1299,25 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase '12345' # new ]) # The ed_search_prev_history and ed_search_next_history doesn't have default binding - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12345') - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12aaa') - @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_line_around_cursor('', '12356') - @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_line_around_cursor('', '12aaa') - @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_line_around_cursor('', '12345') - @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_line_around_cursor('', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12345', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12356', '') + input_key_by_symbol(:ed_search_next_history) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_search_next_history) + assert_line_around_cursor('12345', '') + input_key_by_symbol(:ed_search_prev_history) + assert_line_around_cursor('12aaa', '') + input_key_by_symbol(:ed_prev_char) + input_key_by_symbol(:ed_next_char) + input_key_by_symbol(:ed_search_next_history) + assert_line_around_cursor('12aaa', '') + 3.times { input_key_by_symbol(:ed_prev_char) } + input_key_by_symbol(:ed_search_next_history) + assert_line_around_cursor('12', '345') end def test_incremental_search_history_cancel_by_symbol_key @@ -1437,4 +1452,176 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase @line_editor.__send__(:vi_editing_mode, nil) assert(@config.editing_mode_is?(:vi_insert)) end + + def test_undo + input_keys("\C-_", false) + assert_line_around_cursor('', '') + input_keys("aあb\C-h\C-h\C-h", false) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあb', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('', '') + end + + def test_undo_with_cursor_position + input_keys("abc\C-b\C-h", false) + assert_line_around_cursor('a', 'c') + input_keys("\C-_", false) + assert_line_around_cursor('ab', 'c') + input_keys("あいう\C-b\C-h", false) + assert_line_around_cursor('abあ', 'うc') + input_keys("\C-_", false) + assert_line_around_cursor('abあい', 'うc') + end + + def test_undo_with_multiline + @line_editor.multiline_on + @line_editor.confirm_multiline_termination_proc = proc {} + input_keys("1\n2\n3", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(2) + assert_line_around_cursor('3', '') + input_keys("\C-p\C-h\C-h", false) + assert_whole_lines(["1", "3"]) + assert_line_index(0) + assert_line_around_cursor('1', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "", "3"]) + assert_line_index(1) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2", ""]) + assert_line_index(2) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + end + + def test_undo_with_many_times + str = "a" + "b" * 99 + input_keys(str, false) + 100.times { input_keys("\C-_", false) } + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + end + + def test_redo + input_keys("aあb", false) + assert_line_around_cursor('aあb', '') + input_keys("\M-\C-_", false) + assert_line_around_cursor('aあb', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + input_keys("\M-\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\M-\C-_", false) + assert_line_around_cursor('aあb', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("c", false) + assert_line_around_cursor('aあc', '') + input_keys("\M-\C-_", false) + assert_line_around_cursor('aあc', '') + end + + def test_redo_with_cursor_position + input_keys("abc\C-b\C-h", false) + assert_line_around_cursor('a', 'c') + input_keys("\M-\C-_", false) + assert_line_around_cursor('a', 'c') + input_keys("\C-_", false) + assert_line_around_cursor('ab', 'c') + input_keys("\M-\C-_", false) + assert_line_around_cursor('a', 'c') + end + + def test_redo_with_multiline + @line_editor.multiline_on + @line_editor.confirm_multiline_termination_proc = proc {} + input_keys("1\n2\n3", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(2) + assert_line_around_cursor('3', '') + + input_keys("\C-_", false) + assert_whole_lines(["1", "2", ""]) + assert_line_index(2) + assert_line_around_cursor('', '') + + input_keys("\C-_", false) + assert_whole_lines(["1", "2"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + + input_keys("\M-\C-_", false) + assert_whole_lines(["1", "2", ""]) + assert_line_index(2) + assert_line_around_cursor('', '') + + input_keys("\M-\C-_", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(2) + assert_line_around_cursor('3', '') + + input_keys("\C-p\C-h\C-h", false) + assert_whole_lines(["1", "3"]) + assert_line_index(0) + assert_line_around_cursor('1', '') + + input_keys("\C-n", false) + assert_whole_lines(["1", "3"]) + assert_line_index(1) + assert_line_around_cursor('3', '') + + input_keys("\C-_", false) + assert_whole_lines(["1", "", "3"]) + assert_line_index(1) + assert_line_around_cursor('', '') + + input_keys("\C-_", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + + input_keys("\M-\C-_", false) + assert_whole_lines(["1", "", "3"]) + assert_line_index(1) + assert_line_around_cursor('', '') + + input_keys("\M-\C-_", false) + assert_whole_lines(["1", "3"]) + assert_line_index(1) + assert_line_around_cursor('3', '') + end + + def test_redo_with_many_times + str = "a" + "b" * 98 + "c" + input_keys(str, false) + 100.times { input_keys("\C-_", false) } + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + 100.times { input_keys("\M-\C-_", false) } + assert_line_around_cursor(str, '') + input_keys("\M-\C-_", false) + assert_line_around_cursor(str, '') + end end diff --git a/test/reline/test_key_actor_vi.rb b/test/reline/test_key_actor_vi.rb index 4deae2dd83..1288b9aaa5 100644 --- a/test/reline/test_key_actor_vi.rb +++ b/test/reline/test_key_actor_vi.rb @@ -1,6 +1,6 @@ require_relative 'helper' -class Reline::KeyActor::ViInsert::Test < Reline::TestCase +class Reline::ViInsertTest < Reline::TestCase def setup Reline.send(:test_mode) @prompt = '> ' @@ -13,69 +13,73 @@ class Reline::KeyActor::ViInsert::Test < Reline::TestCase @line_editor.reset(@prompt, encoding: @encoding) end + def editing_mode_label + @config.instance_variable_get(:@editing_mode_label) + end + def teardown Reline.test_reset end def test_vi_command_mode input_keys("\C-[") - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) end def test_vi_command_mode_with_input input_keys("abc\C-[") - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) assert_line_around_cursor('ab', 'c') end def test_vi_insert - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys('i') assert_line_around_cursor('i', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys("\C-[") assert_line_around_cursor('', 'i') - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) input_keys('i') assert_line_around_cursor('', 'i') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) end def test_vi_add - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys('a') assert_line_around_cursor('a', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys("\C-[") assert_line_around_cursor('', 'a') - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) input_keys('a') assert_line_around_cursor('a', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) end def test_vi_insert_at_bol input_keys('I') assert_line_around_cursor('I', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys("12345\C-[hh") assert_line_around_cursor('I12', '345') - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) input_keys('I') assert_line_around_cursor('', 'I12345') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) end def test_vi_add_at_eol input_keys('A') assert_line_around_cursor('A', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) input_keys("12345\C-[hh") assert_line_around_cursor('A12', '345') - assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) + assert_equal(:vi_command, editing_mode_label) input_keys('A') assert_line_around_cursor('A12345', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) end def test_ed_insert_one @@ -901,11 +905,11 @@ class Reline::KeyActor::ViInsert::Test < Reline::TestCase assert_line_around_cursor('abc', '') input_keys("\C-[0C") assert_line_around_cursor('', '') - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) end def test_vi_motion_operators - assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) + assert_equal(:vi_insert, editing_mode_label) assert_nothing_raised do input_keys("test = { foo: bar }\C-[BBBldt}b") diff --git a/test/reline/test_key_stroke.rb b/test/reline/test_key_stroke.rb index cd205c7d9e..ec70a05957 100644 --- a/test/reline/test_key_stroke.rb +++ b/test/reline/test_key_stroke.rb @@ -24,16 +24,14 @@ class Reline::KeyStroke::Test < Reline::TestCase config.add_default_key_binding(key.bytes, func.bytes) end stroke = Reline::KeyStroke.new(config) - assert_equal(:matching, stroke.match_status("a".bytes)) - assert_equal(:matching, stroke.match_status("ab".bytes)) - assert_equal(:matched, stroke.match_status("abc".bytes)) - assert_equal(:matched, stroke.match_status("abz".bytes)) - assert_equal(:matched, stroke.match_status("abx".bytes)) - assert_equal(:matched, stroke.match_status("ac".bytes)) - assert_equal(:matched, stroke.match_status("aa".bytes)) - assert_equal(:matched, stroke.match_status("x".bytes)) - assert_equal(:unmatched, stroke.match_status("m".bytes)) - assert_equal(:matched, stroke.match_status("abzwabk".bytes)) + assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("a".bytes)) + assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("ab".bytes)) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("abc".bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status("abz".bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status("abcx".bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status("aa".bytes)) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("x".bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status("xa".bytes)) end def test_match_unknown @@ -47,12 +45,15 @@ class Reline::KeyStroke::Test < Reline::TestCase "\e[1;1R", # Cursor position report "\e[15~", # F5 "\eOP", # F1 - "\e\e[A" # Option+Up + "\e\e[A", # Option+Up + "\eX", + "\e\eX" ] sequences.each do |seq| - assert_equal(:matched, stroke.match_status(seq.bytes)) - (1...seq.size).each do |i| - assert_equal(:matching, stroke.match_status(seq.bytes.take(i))) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status(seq.bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status(seq.bytes + [32])) + (2...seq.size).each do |i| + assert_equal(Reline::KeyStroke::MATCHING, stroke.match_status(seq.bytes.take(i))) end end end @@ -61,16 +62,18 @@ class Reline::KeyStroke::Test < Reline::TestCase config = Reline::Config.new { 'abc' => '123', + 'ab' => '456' }.each_pair do |key, func| config.add_default_key_binding(key.bytes, func.bytes) end stroke = Reline::KeyStroke.new(config) - assert_equal('123'.bytes, stroke.expand('abc'.bytes)) + assert_equal(['123'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abcde'.bytes)) + assert_equal(['456'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abde'.bytes)) # CSI sequence - assert_equal([:ed_unassigned] + 'bc'.bytes, stroke.expand("\e[1;2;3;4;5abc".bytes)) - assert_equal([:ed_unassigned] + 'BC'.bytes, stroke.expand("\e\e[ABC".bytes)) + assert_equal([[], 'bc'.bytes], stroke.expand("\e[1;2;3;4;5abc".bytes)) + assert_equal([[], 'BC'.bytes], stroke.expand("\e\e[ABC".bytes)) # SS3 sequence - assert_equal([:ed_unassigned] + 'QR'.bytes, stroke.expand("\eOPQR".bytes)) + assert_equal([[], 'QR'.bytes], stroke.expand("\eOPQR".bytes)) end def test_oneshot_key_bindings @@ -81,25 +84,22 @@ class Reline::KeyStroke::Test < Reline::TestCase config.add_default_key_binding(key.bytes, func.bytes) end stroke = Reline::KeyStroke.new(config) - assert_equal(:unmatched, stroke.match_status('zzz'.bytes)) - assert_equal(:matched, stroke.match_status('abc'.bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('zzz'.bytes)) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status('abc'.bytes)) end def test_with_reline_key config = Reline::Config.new { - [ - Reline::Key.new(100, 228, true), # Alt+d - Reline::Key.new(97, 97, false) # a - ] => 'abc', + "\eda".bytes => 'abc', # Alt+d a [195, 164] => 'def' }.each_pair do |key, func| config.add_oneshot_key_binding(key, func.bytes) end stroke = Reline::KeyStroke.new(config) - assert_equal(:unmatched, stroke.match_status('da'.bytes)) - assert_equal(:matched, stroke.match_status("\M-da".bytes)) - assert_equal(:unmatched, stroke.match_status([32, 195, 164])) - assert_equal(:matched, stroke.match_status([195, 164])) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('da'.bytes)) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("\eda".bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status([32, 195, 164])) + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status([195, 164])) end end diff --git a/test/reline/test_line_editor.rb b/test/reline/test_line_editor.rb index 7a38ecd596..1859da8199 100644 --- a/test/reline/test_line_editor.rb +++ b/test/reline/test_line_editor.rb @@ -4,14 +4,12 @@ require 'stringio' class Reline::LineEditor class RenderLineDifferentialTest < Reline::TestCase - module TestIO - RESET_COLOR = "\e[0m" - - def self.move_cursor_column(col) + class TestIO < Reline::IO + def move_cursor_column(col) @output << "[COL_#{col}]" end - def self.erase_after_cursor + def erase_after_cursor @output << '[ERASE]' end end @@ -24,7 +22,7 @@ class Reline::LineEditor @line_editor.instance_variable_set(:@screen_size, [24, 80]) @line_editor.instance_variable_set(:@output, @output) Reline.send(:remove_const, :IOGate) - Reline.const_set(:IOGate, TestIO) + Reline.const_set(:IOGate, TestIO.new) Reline::IOGate.instance_variable_set(:@output, @output) ensure $VERBOSE = verbose diff --git a/test/reline/test_reline.rb b/test/reline/test_reline.rb index 40c880c11f..f2feab684d 100644 --- a/test/reline/test_reline.rb +++ b/test/reline/test_reline.rb @@ -303,12 +303,12 @@ class Reline::Test < Reline::TestCase def test_vi_editing_mode Reline.vi_editing_mode - assert_equal(Reline::KeyActor::ViInsert, Reline.core.config.editing_mode.class) + assert_equal(:vi_insert, Reline.core.config.instance_variable_get(:@editing_mode_label)) end def test_emacs_editing_mode Reline.emacs_editing_mode - assert_equal(Reline::KeyActor::Emacs, Reline.core.config.editing_mode.class) + assert_equal(:emacs, Reline.core.config.instance_variable_get(:@editing_mode_label)) end def test_add_dialog_proc @@ -375,13 +375,67 @@ class Reline::Test < Reline::TestCase def test_dumb_terminal lib = File.expand_path("../../lib", __dir__) out = IO.popen([{"TERM"=>"dumb"}, Reline.test_rubybin, "-I#{lib}", "-rreline", "-e", "p Reline.core.io_gate"], &:read) - assert_equal("Reline::GeneralIO", out.chomp) + assert_match(/#<Reline::Dumb/, out.chomp) + end + + def test_print_prompt_before_everything_else + pend if win? + lib = File.expand_path("../../lib", __dir__) + code = "p Reline::IOGate.class; p Reline.readline 'prompt> '" + out = IO.popen([Reline.test_rubybin, "-I#{lib}", "-rreline", "-e", code], "r+") do |io| + io.write "abc\n" + io.close_write + io.read + end + assert_match(/\AReline::ANSI\nprompt> /, out) + end + + def test_read_eof_returns_input + pend if win? + lib = File.expand_path("../../lib", __dir__) + code = "p result: Reline.readline" + out = IO.popen([Reline.test_rubybin, "-I#{lib}", "-rreline", "-e", code], "r+") do |io| + io.write "a\C-a" + io.close_write + io.read + end + assert_include(out, '{:result=>"a"}') + end + + def test_read_eof_returns_nil_if_empty + pend if win? + lib = File.expand_path("../../lib", __dir__) + code = "p result: Reline.readline" + out = IO.popen([Reline.test_rubybin, "-I#{lib}", "-rreline", "-e", code], "r+") do |io| + io.write "a\C-h" + io.close_write + io.read + end + assert_include(out, '{:result=>nil}') + end + + def test_require_reline_should_not_trigger_winsize + pend if win? + lib = File.expand_path("../../lib", __dir__) + code = <<~RUBY + require "io/console" + def STDIN.tty?; true; end + def STDOUT.tty?; true; end + def STDIN.winsize; raise; end + require("reline") && p(Reline.core.io_gate) + RUBY + out = IO.popen([{}, Reline.test_rubybin, "-I#{lib}", "-e", code], &:read) + assert_include(out.chomp, "Reline::ANSI") + end + + def win? + /mswin|mingw/.match?(RUBY_PLATFORM) end def get_reline_encoding if encoding = Reline.core.encoding encoding - elsif RUBY_PLATFORM =~ /mswin|mingw/ + elsif win? Encoding::UTF_8 else Encoding::default_external diff --git a/test/reline/test_reline_key.rb b/test/reline/test_reline_key.rb index 7f9a11394a..1e6b9fcb6c 100644 --- a/test/reline/test_reline_key.rb +++ b/test/reline/test_reline_key.rb @@ -2,53 +2,10 @@ require_relative 'helper' require "reline" class Reline::TestKey < Reline::TestCase - def setup - Reline.test_mode - end - - def teardown - Reline.test_reset - end - - def test_match_key - assert(Reline::Key.new(1, 2, false).match?(Reline::Key.new(1, 2, false))) - assert(Reline::Key.new(1, 2, false).match?(Reline::Key.new(nil, 2, false))) - assert(Reline::Key.new(1, 2, false).match?(Reline::Key.new(1, 2, nil))) - - assert(Reline::Key.new(nil, 2, false).match?(Reline::Key.new(nil, 2, false))) - assert(Reline::Key.new(1, nil, false).match?(Reline::Key.new(1, nil, false))) - assert(Reline::Key.new(1, 2, nil).match?(Reline::Key.new(1, 2, nil))) - - assert(Reline::Key.new(nil, 2, false).match?(Reline::Key.new(nil, 2, false))) - assert(Reline::Key.new(1, nil, false).match?(Reline::Key.new(1, nil, false))) - assert(Reline::Key.new(1, 2, nil).match?(Reline::Key.new(1, 2, nil))) - - assert(!Reline::Key.new(1, 2, false).match?(Reline::Key.new(3, 1, false))) - assert(!Reline::Key.new(1, 2, false).match?(Reline::Key.new(1, 3, false))) - assert(!Reline::Key.new(1, 2, false).match?(Reline::Key.new(1, 3, true))) - end - - def test_match_integer - assert(Reline::Key.new(1, 2, false).match?(2)) - assert(Reline::Key.new(nil, 2, false).match?(2)) - assert(Reline::Key.new(1, nil, false).match?(1)) - - assert(!Reline::Key.new(1, 2, false).match?(1)) - assert(!Reline::Key.new(1, nil, false).match?(2)) - assert(!Reline::Key.new(nil, nil, false).match?(1)) - end - def test_match_symbol - assert(Reline::Key.new(:key1, :key2, false).match?(:key2)) - assert(Reline::Key.new(:key1, nil, false).match?(:key1)) - - assert(!Reline::Key.new(:key1, :key2, false).match?(:key1)) - assert(!Reline::Key.new(:key1, nil, false).match?(:key2)) - assert(!Reline::Key.new(nil, nil, false).match?(:key1)) - end - - def test_match_other - assert(!Reline::Key.new(:key1, 2, false).match?("key1")) - assert(!Reline::Key.new(nil, nil, false).match?(nil)) + assert(Reline::Key.new(:key1, :key1, false).match?(:key1)) + refute(Reline::Key.new(:key1, :key1, false).match?(:key2)) + refute(Reline::Key.new(:key1, :key1, false).match?(nil)) + refute(Reline::Key.new(1, 1, false).match?(:key1)) end end diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index 74798c338f..f119d686cd 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -543,15 +543,10 @@ begin EOC end - def test_enable_bracketed_paste + def test_bracketed_paste omit if Reline.core.io_gate.win? - write_inputrc <<~LINES - set enable-bracketed-paste on - LINES start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') - write("\e[200~,") - write("def hoge\n 3\nend") - write("\e[200~.") + write("\e[200~def hoge\r\t3\rend\e[201~") close assert_screen(<<~EOC) Multiline REPL. @@ -561,6 +556,35 @@ begin EOC end + def test_bracketed_paste_with_undo + omit if Reline.core.io_gate.win? + start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') + write("abc") + write("\e[200~def hoge\r\t3\rend\e[201~") + write("\C-_") + close + assert_screen(<<~EOC) + Multiline REPL. + prompt> abc + EOC + end + + def test_bracketed_paste_with_redo + omit if Reline.core.io_gate.win? + start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') + write("abc") + write("\e[200~def hoge\r\t3\rend\e[201~") + write("\C-_") + write("\M-\C-_") + close + assert_screen(<<~EOC) + Multiline REPL. + prompt> abcdef hoge + prompt> 3 + prompt> end + EOC + end + def test_backspace_until_returns_to_initial start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') write("ABC") @@ -945,6 +969,18 @@ begin EOC end + def test_nontty + omit if Reline.core.io_gate.win? + cmd = %Q{ruby -e 'puts(%Q{ello\C-ah\C-e})' | ruby -I#{@pwd}/lib -rreline -e 'p Reline.readline(%{> })' | ruby -e 'print STDIN.read'} + start_terminal(40, 50, ['bash', '-c', cmd]) + sleep 1 + close rescue nil + assert_screen(<<~'EOC') + > hello + "hello" + EOC + end + def test_eof_with_newline omit if Reline.core.io_gate.win? cmd = %Q{ruby -e 'print(%Q{abc def \\e\\r})' | ruby -I#{@pwd}/lib -rreline -e 'p Reline.readline(%{> })'} diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb index 7d62a7ee28..64b4336375 100644 --- a/test/ripper/test_lexer.rb +++ b/test/ripper/test_lexer.rb @@ -253,18 +253,31 @@ world" assert_equal(code, Ripper.tokenize(code).join(""), bug) end + InvalidHeredocInsideBlockParam = <<~CODE + a do |b + <<-C + C + | + end + CODE + def test_heredoc_inside_block_param bug = '[Bug #19399]' - code = <<~CODE - a do |b - <<-C - C - | - end - CODE + code = InvalidHeredocInsideBlockParam assert_equal(code, Ripper.tokenize(code).join(""), bug) end + def test_heredoc_no_memory_leak + assert_no_memory_leak([], "#{<<-"begin;"}", "#{<<-'end;'}", rss: true) + require "ripper" + source = "" #{InvalidHeredocInsideBlockParam.dump} + begin; + 400_000.times do + Ripper.new(source).parse + end + end; + end + def test_heredoc_unterminated_interpolation code = <<~'HEREDOC' <<A+1 @@ -302,9 +315,8 @@ world" [[6, 2], :on_tstring_content, "3\n", state(:EXPR_BEG)], [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], ] - assert_equal(code, Ripper.tokenize(code).join("")) - assert_equal(expected, result = Ripper.lex(code), - proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) + + assert_lexer(expected, code) code = <<~'HEREDOC' <<-H1 @@ -330,6 +342,55 @@ world" [[6, 0], :on_tstring_content, " 3\n", state(:EXPR_BEG)], [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], ] + + assert_lexer(expected, code) + end + + def test_invalid_escape_ctrl_mbchar + code = %["\\C-\u{3042}"] + expected = [ + [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)], + [[1, 1], :on_tstring_content, "\\C-\u{3042}", state(:EXPR_BEG)], + [[1, 7], :on_tstring_end, '"', state(:EXPR_END)], + ] + + assert_lexer(expected, code) + end + + def test_invalid_escape_meta_mbchar + code = %["\\M-\u{3042}"] + expected = [ + [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)], + [[1, 1], :on_tstring_content, "\\M-\u{3042}", state(:EXPR_BEG)], + [[1, 7], :on_tstring_end, '"', state(:EXPR_END)], + ] + + assert_lexer(expected, code) + end + + def test_invalid_escape_meta_ctrl_mbchar + code = %["\\M-\\C-\u{3042}"] + expected = [ + [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)], + [[1, 1], :on_tstring_content, "\\M-\\C-\u{3042}", state(:EXPR_BEG)], + [[1, 10], :on_tstring_end, '"', state(:EXPR_END)], + ] + + assert_lexer(expected, code) + end + + def test_invalid_escape_ctrl_meta_mbchar + code = %["\\C-\\M-\u{3042}"] + expected = [ + [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)], + [[1, 1], :on_tstring_content, "\\C-\\M-\u{3042}", state(:EXPR_BEG)], + [[1, 10], :on_tstring_end, '"', state(:EXPR_END)], + ] + + assert_lexer(expected, code) + end + + def assert_lexer(expected, code) assert_equal(code, Ripper.tokenize(code).join("")) assert_equal(expected, result = Ripper.lex(code), proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb index cbae6e7ed5..5434acb523 100644 --- a/test/ripper/test_parser_events.rb +++ b/test/ripper/test_parser_events.rb @@ -267,17 +267,29 @@ class TestRipper::ParserEvents < Test::Unit::TestCase end def test_assign_error_backref - thru_assign_error = false + errors = [] result = - parse('$` = 1', :on_assign_error) {thru_assign_error = true} - assert_equal true, thru_assign_error - assert_equal '[assign(assign_error(var_field($`)),1)]', result + parse('$& = 1', %i[on_assign_error compile_error]) {|e, *| errors << e} + assert_equal %i[on_assign_error], errors + assert_equal '[assign(assign_error(var_field($&)),1)]', result - thru_assign_error = false + errors = [] result = - parse('$`, _ = 1', :on_assign_error) {thru_assign_error = true} - assert_equal true, thru_assign_error - assert_equal '[massign([assign_error(var_field($`)),var_field(_)],1)]', result + parse('$&, _ = 1', %i[on_assign_error compile_error]) {|e, *| errors << e} + assert_equal %i[on_assign_error], errors + assert_equal '[massign([assign_error(var_field($&)),var_field(_)],1)]', result + + errors = [] + result = + parse('$& += 1', %i[on_assign_error compile_error]) {|e, *| errors << e} + assert_equal %i[on_assign_error], errors + assert_equal '[assign_error(opassign(var_field($&),+=,1))]', result + + errors = [] + result = + parse('$& += cmd 1, 2', %i[on_assign_error compile_error]) {|e, *| errors << e} + assert_equal %i[on_assign_error], errors + assert_equal '[assign_error(opassign(var_field($&),+=,command(cmd,[1,2])))]', result end def test_assign_error_const_qualified @@ -1682,8 +1694,8 @@ class TestRipper::ParserEvents < Test::Unit::TestCase else end STR - assert_match(/duplicated 'when' clause/, fmt) - assert_equal([3], args) + assert_match(/duplicates 'when' clause/, fmt) + assert_equal([4, 3], args) end def test_warn_duplicated_hash_keys diff --git a/test/ruby/test_allocation.rb b/test/ruby/test_allocation.rb index 48348c0fbd..fbe0548899 100644 --- a/test/ruby/test_allocation.rb +++ b/test/ruby/test_allocation.rb @@ -25,6 +25,10 @@ class TestAllocation < Test::Unit::TestCase empty_array = empty_array = [] empty_hash = empty_hash = {} array1 = array1 = [1] + r2k_array = r2k_array = [Hash.ruby2_keywords_hash(a: 3)] + r2k_array1 = r2k_array1 = [1, Hash.ruby2_keywords_hash(a: 3)] + r2k_empty_array = r2k_empty_array = [Hash.ruby2_keywords_hash({})] + r2k_empty_array1 = r2k_empty_array1 = [1, Hash.ruby2_keywords_hash({})] hash1 = hash1 = {a: 2} nill = nill = nil block = block = lambda{} @@ -95,6 +99,8 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 0, "none(*empty_array, *empty_array#{block})") check_allocations(0, 1, "none(**empty_hash, **empty_hash#{block})") check_allocations(1, 1, "none(*empty_array, *empty_array, **empty_hash, **empty_hash#{block})") + + check_allocations(0, 0, "none(*r2k_empty_array#{block})") RUBY end @@ -114,6 +120,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "required(**hash1, **empty_hash#{block})") check_allocations(1, 0, "required(*array1, *empty_array, **empty_hash#{block})") + check_allocations(0, 0, "required(*r2k_empty_array1#{block})") + check_allocations(0, 1, "required(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required(*empty_array, **hash1, **empty_hash#{block})") RUBY @@ -135,6 +144,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "optional(**hash1, **empty_hash#{block})") check_allocations(1, 0, "optional(*array1, *empty_array, **empty_hash#{block})") + check_allocations(0, 0, "optional(*r2k_empty_array#{block})") + check_allocations(0, 0, "optional(*r2k_empty_array1#{block})") + check_allocations(0, 1, "optional(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "optional(*empty_array, **hash1, **empty_hash#{block})") RUBY @@ -166,6 +179,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat(**hash1, **empty_hash#{block})") check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "splat(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*r2k_empty_array#{block})") + check_allocations(1, 0, "splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat(*r2k_array#{block})") + check_allocations(1, 1, "splat(*r2k_array1#{block})") RUBY end @@ -195,6 +213,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "req_splat(**hash1, **empty_hash#{block})") check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "req_splat(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "req_splat(*r2k_array#{block})") + check_allocations(1, 1, "req_splat(*r2k_array1#{block})") RUBY end @@ -224,6 +246,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_post(**hash1, **empty_hash#{block})") check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "splat_post(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_post(*r2k_array#{block})") + check_allocations(1, 1, "splat_post(*r2k_array1#{block})") RUBY end @@ -251,6 +277,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword(*empty_array, **hash1, **empty_hash#{block})") @@ -281,6 +310,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword_splat(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword_splat(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword_splat(*empty_array, **hash1, **empty_hash#{block})") @@ -311,6 +343,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, **hash1, **empty_hash#{block})") @@ -350,6 +385,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(*r2k_empty_array1#{block})") + check_allocations(1, 1, "required_and_keyword(*r2k_array1#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") @@ -397,6 +435,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_and_keyword(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "splat_and_keyword(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "splat_and_keyword(*array1, **nil#{block})") + + check_allocations(1, 0, "splat_and_keyword(*r2k_empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword(*r2k_array#{block})") + check_allocations(1, 0, "splat_and_keyword(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_and_keyword(*r2k_array1#{block})") RUBY end @@ -433,6 +476,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*r2k_array1#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") @@ -480,6 +526,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_array1#{block})") RUBY end @@ -521,6 +572,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array1#{block})") RUBY end @@ -562,6 +618,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array1#{block})") RUBY end @@ -603,6 +664,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array1#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array1#{block})") RUBY end @@ -644,6 +710,61 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array1#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array1#{block})") + RUBY + end + + def test_ruby2_keywords + check_allocations(<<~RUBY) + def self.r2k(*a#{block}); end + singleton_class.send(:ruby2_keywords, :r2k) + + check_allocations(1, 1, "r2k(1, a: 2#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "r2k(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "r2k(1, **nil#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **hash1#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "r2k(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "r2k(1, *empty_array#{block})") + check_allocations(1, 1, "r2k(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(*array1, a: 2#{block})") + + check_allocations(1, 0, "r2k(*array1, **nill#{block})") + check_allocations(1, 1, "r2k(*array1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, **hash1#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "r2k(*array1, *empty_array#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "r2k(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "r2k(*array1, **nil#{block})") + + check_allocations(1, 0, "r2k(*r2k_empty_array#{block})") + check_allocations(1, 1, "r2k(*r2k_array#{block})") + unless defined?(RubyVM::YJIT.enabled?) && RubyVM::YJIT.enabled? + # YJIT may or may not allocate depending on arch? + check_allocations(1, 0, "r2k(*r2k_empty_array1#{block})") + check_allocations(1, 1, "r2k(*r2k_array1#{block})") + end RUBY end diff --git a/test/ruby/test_bignum.rb b/test/ruby/test_bignum.rb index 1f882c6cb9..1858793952 100644 --- a/test/ruby/test_bignum.rb +++ b/test/ruby/test_bignum.rb @@ -821,5 +821,11 @@ class TestBignum < Test::Unit::TestCase assert_nil(T1024P.infinite?) assert_nil((-T1024P).infinite?) end + + def test_gmp_version + if RbConfig::CONFIG.fetch('configure_args').include?("'--with-gmp'") + assert_kind_of(String, Integer::GMP_VERSION) + end + end if ENV['GITHUB_WORKFLOW'] == 'Compilations' end end diff --git a/test/ruby/test_call.rb b/test/ruby/test_call.rb index a52b75c267..ced1eaf5e9 100644 --- a/test/ruby/test_call.rb +++ b/test/ruby/test_call.rb @@ -136,7 +136,7 @@ class TestCall < Test::Unit::TestCase # Prevent "assigned but unused variable" warnings _ = [h, a, kw, b] - message = /keyword arg given in index/ + message = /keyword arg given in index assignment/ # +=, without block, non-popped assert_syntax_error(%q{h[**kw] += 1}, message) @@ -270,7 +270,7 @@ class TestCall < Test::Unit::TestCase def o.[](...) 2 end def o.[]=(...) end - message = /keyword arg given in index/ + message = /keyword arg given in index assignment/ assert_syntax_error(%q{o[kw: 1] += 1}, message) assert_syntax_error(%q{o[**o] += 1}, message) @@ -292,7 +292,7 @@ class TestCall < Test::Unit::TestCase def []=(*a, **b) @set = [a, b] end end.new - message = /keyword arg given in index/ + message = /keyword arg given in index assignment/ a = [] kw = {} diff --git a/test/ruby/test_file.rb b/test/ruby/test_file.rb index aa10566bfa..9fa96f8a7c 100644 --- a/test/ruby/test_file.rb +++ b/test/ruby/test_file.rb @@ -460,6 +460,39 @@ class TestFile < Test::Unit::TestCase end end + def test_initialize + Dir.mktmpdir(__method__.to_s) do |tmpdir| + path = File.join(tmpdir, "foo") + + assert_raise(Errno::ENOENT) {File.new(path)} + f = File.new(path, "w") + f.write("FOO\n") + f.close + f = File.new(path) + data = f.read + f.close + assert_equal("FOO\n", data) + + f = File.new(path, File::WRONLY) + f.write("BAR\n") + f.close + f = File.new(path, File::RDONLY) + data = f.read + f.close + assert_equal("BAR\n", data) + + data = File.open(path) {|file| + File.new(file.fileno, mode: File::RDONLY, autoclose: false).read + } + assert_equal("BAR\n", data) + + data = File.open(path) {|file| + File.new(file.fileno, File::RDONLY, autoclose: false).read + } + assert_equal("BAR\n", data) + end + end + def test_file_open_newline_option Dir.mktmpdir(__method__.to_s) do |tmpdir| path = File.join(tmpdir, "foo") diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 39b001c3d0..491746fe83 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -648,7 +648,7 @@ class TestGc < Test::Unit::TestCase def test_thrashing_for_young_objects # This test prevents bugs like [Bug #18929] - assert_separately([], __FILE__, __LINE__, <<-'RUBY') + assert_separately([], __FILE__, __LINE__, <<-'RUBY', timeout: 60) # Grow the heap @ary = 100_000.times.map { Object.new } diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index f47c0b046b..c331968b3d 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -146,7 +146,7 @@ class TestGCCompact < Test::Unit::TestCase end def test_ast_compacts - assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; def walk_ast ast children = ast.children.grep(RubyVM::AbstractSyntaxTree::Node) @@ -185,7 +185,7 @@ class TestGCCompact < Test::Unit::TestCase end def test_updating_references_for_heap_allocated_shared_arrays - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ary = [] 50.times { |i| ary << i } @@ -209,7 +209,7 @@ class TestGCCompact < Test::Unit::TestCase def test_updating_references_for_embed_shared_arrays omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ary = Array.new(50) 50.times { |i| ary[i] = i } @@ -233,7 +233,7 @@ class TestGCCompact < Test::Unit::TestCase end def test_updating_references_for_heap_allocated_frozen_shared_arrays - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ary = [] 50.times { |i| ary << i } @@ -258,7 +258,7 @@ class TestGCCompact < Test::Unit::TestCase def test_updating_references_for_embed_frozen_shared_arrays omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ary = Array.new(50) 50.times { |i| ary[i] = i } @@ -286,7 +286,7 @@ class TestGCCompact < Test::Unit::TestCase def test_moving_arrays_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ARY_COUNT = 50000 @@ -308,7 +308,7 @@ class TestGCCompact < Test::Unit::TestCase def test_moving_arrays_up_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; ARY_COUNT = 50000 @@ -332,7 +332,7 @@ class TestGCCompact < Test::Unit::TestCase def test_moving_objects_between_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 60) begin; class Foo def add_ivars @@ -364,7 +364,7 @@ class TestGCCompact < Test::Unit::TestCase def test_moving_strings_up_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; STR_COUNT = 50000 @@ -385,7 +385,7 @@ class TestGCCompact < Test::Unit::TestCase def test_moving_strings_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; STR_COUNT = 50000 @@ -407,7 +407,7 @@ class TestGCCompact < Test::Unit::TestCase # AR and ST hashes are in the same size pool on 32 bit omit unless RbConfig::SIZEOF["uint64_t"] <= RbConfig::SIZEOF["void*"] - assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30, signal: :SEGV) + assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; HASH_COUNT = 50000 diff --git a/test/ruby/test_io.rb b/test/ruby/test_io.rb index 476d9f882f..ce81286c4d 100644 --- a/test/ruby/test_io.rb +++ b/test/ruby/test_io.rb @@ -2929,6 +2929,15 @@ class TestIO < Test::Unit::TestCase f.close assert_equal("FOO\n", File.read(t.path)) + + fd = IO.sysopen(t.path) + %w[w r+ w+ a+].each do |mode| + assert_raise(Errno::EINVAL, "#{mode} [ruby-dev:38571]") {IO.new(fd, mode)} + end + f = IO.new(fd, "r") + data = f.read + f.close + assert_equal("FOO\n", data) } end diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb index d2a39e673f..a47419253b 100644 --- a/test/ruby/test_iseq.rb +++ b/test/ruby/test_iseq.rb @@ -347,11 +347,17 @@ class TestISeq < Test::Unit::TestCase end end assert_equal([m1, e1.message], [m2, e2.message], feature11951) - message = e1.message.each_line - message.with_index(1) do |line, i| - next if /^ / =~ line - assert_send([line, :start_with?, __FILE__], - proc {message.map {|l, j| (i == j ? ">" : " ") + l}.join("")}) + + if e1.message.lines[0] == "#{__FILE__}:#{line}: syntax errors found\n" + # Prism lays out the error messages in line with the source, so the + # following assertions do not make sense in that context. + else + message = e1.message.each_line + message.with_index(1) do |line, i| + next if /^ / =~ line + assert_send([line, :start_with?, __FILE__], + proc {message.map {|l, j| (i == j ? ">" : " ") + l}.join("")}) + end end end @@ -463,7 +469,7 @@ class TestISeq < Test::Unit::TestCase ["<class:C>@1", ["bar@10", ["block in bar@11", ["block (2 levels) in bar@12"]]], - ["foo@2", ["ensure in foo@2"], + ["foo@2", ["ensure in foo@7"], ["rescue in foo@4"]]], ["<class:D>@17"]] @@ -496,7 +502,7 @@ class TestISeq < Test::Unit::TestCase [4, :line], [7, :line], [9, :return]]], - [["ensure in foo@2", [[7, :line]]]], + [["ensure in foo@7", [[7, :line]]]], [["rescue in foo@4", [[5, :line], [5, :rescue]]]]]], [["<class:D>@17", [[17, :class], diff --git a/test/ruby/test_literal.rb b/test/ruby/test_literal.rb index c6154af1f6..8732d8f0d0 100644 --- a/test/ruby/test_literal.rb +++ b/test/ruby/test_literal.rb @@ -640,11 +640,20 @@ class TestRubyLiteral < Test::Unit::TestCase end begin r2 = eval(s) - rescue NameError, SyntaxError + rescue ArgumentError + # Debug log for a random failure: ArgumentError: SyntaxError#path changed + $stderr.puts "TestRubyLiteral#test_float failed: %p" % s + raise + rescue SyntaxError => e + r2 = :err + rescue NameError r2 = :err end r2 = :err if Range === r2 - assert_equal(r1, r2, "Float(#{s.inspect}) != eval(#{s.inspect})") + s = s.inspect + mesg = "Float(#{s}) != eval(#{s})" + mesg << ":" << e.message if e + assert_equal(r1, r2, mesg) } } } diff --git a/test/ruby/test_marshal.rb b/test/ruby/test_marshal.rb index 79d9577737..bcd8892f23 100644 --- a/test/ruby/test_marshal.rb +++ b/test/ruby/test_marshal.rb @@ -570,13 +570,19 @@ class TestMarshal < Test::Unit::TestCase def test_class_ivar assert_raise(TypeError) {Marshal.load("\x04\x08Ic\x1bTestMarshal::TestClass\x06:\x0e@ivar_bug\"\x08bug")} assert_raise(TypeError) {Marshal.load("\x04\x08IM\x1bTestMarshal::TestClass\x06:\x0e@ivar_bug\"\x08bug")} - assert_not_operator(TestClass, :instance_variable_defined?, :@bug) + assert_not_operator(TestClass, :instance_variable_defined?, :@ivar_bug) + + assert_raise(TypeError) {Marshal.load("\x04\x08[\x07c\x1bTestMarshal::TestClassI@\x06\x06:\x0e@ivar_bug\"\x08bug")} + assert_not_operator(TestClass, :instance_variable_defined?, :@ivar_bug) end def test_module_ivar assert_raise(TypeError) {Marshal.load("\x04\x08Im\x1cTestMarshal::TestModule\x06:\x0e@ivar_bug\"\x08bug")} assert_raise(TypeError) {Marshal.load("\x04\x08IM\x1cTestMarshal::TestModule\x06:\x0e@ivar_bug\"\x08bug")} - assert_not_operator(TestModule, :instance_variable_defined?, :@bug) + assert_not_operator(TestModule, :instance_variable_defined?, :@ivar_bug) + + assert_raise(TypeError) {Marshal.load("\x04\x08[\x07m\x1cTestMarshal::TestModuleI@\x06\x06:\x0e@ivar_bug\"\x08bug")} + assert_not_operator(TestModule, :instance_variable_defined?, :@ivar_bug) end class TestForRespondToFalse diff --git a/test/ruby/test_module.rb b/test/ruby/test_module.rb index 75d8d909d7..370b7351c2 100644 --- a/test/ruby/test_module.rb +++ b/test/ruby/test_module.rb @@ -3163,6 +3163,19 @@ class TestModule < Test::Unit::TestCase end; end + def test_define_method_changes_visibility_with_existing_method_bug_19749 + c = Class.new do + def a; end + private def b; end + + define_method(:b, instance_method(:b)) + private + define_method(:a, instance_method(:a)) + end + assert_equal([:b], c.public_instance_methods(false)) + assert_equal([:a], c.private_instance_methods(false)) + end + def test_define_method_with_unbound_method # Passing an UnboundMethod to define_method succeeds if it is from an ancestor assert_nothing_raised do diff --git a/test/ruby/test_pack.rb b/test/ruby/test_pack.rb index 1ce46e8916..a0c66c188b 100644 --- a/test/ruby/test_pack.rb +++ b/test/ruby/test_pack.rb @@ -895,4 +895,22 @@ EXPECTED } assert_equal [nil], "a".unpack("C", offset: 1) end + + def test_monkey_pack + assert_separately([], <<-'end;') + $-w = false + class Array + alias :old_pack :pack + def pack _; "oh no"; end + end + + v = [2 ** 15].pack('n') + + class Array + alias :pack :old_pack + end + + assert_equal "oh no", v + end; + end end diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb index b8de2ba952..20364c5a0d 100644 --- a/test/ruby/test_parse.rb +++ b/test/ruby/test_parse.rb @@ -512,12 +512,12 @@ class TestParse < Test::Unit::TestCase def t.dummy(_) end - assert_syntax_error("#{<<~"begin;"}\n#{<<~'end;'}", /block arg given in index/) + assert_syntax_error("#{<<~"begin;"}\n#{<<~'end;'}", /block arg given in index assignment/) begin; t[42, &blk] ||= 42 end; - assert_syntax_error("#{<<~"begin;"}\n#{<<~'end;'}", /block arg given in index/) + assert_syntax_error("#{<<~"begin;"}\n#{<<~'end;'}", /block arg given in index assignment/) begin; t[42, &blk] ||= t.dummy 42 # command_asgn test end; @@ -555,34 +555,42 @@ class TestParse < Test::Unit::TestCase mesg = 'from the backslash through the invalid char' e = assert_syntax_error('"\xg1"', /hex escape/) - assert_equal(' ^~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{1234"', 'unterminated Unicode escape') - assert_equal(' ^'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{xxxx}"', 'invalid Unicode escape') - assert_equal(' ^'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{xxxx', 'Unicode escape') - assert_pattern_list([ - /.*: invalid Unicode escape\n.*\n/, - / \^/, - /\n/, - /.*: unterminated Unicode escape\n.*\n/, - / \^/, - /\n/, - /.*: unterminated string.*\n.*\n/, - / \^\n/, - ], e.message) + if e.message.lines.first == "#{__FILE__}:#{__LINE__ - 1}: syntax errors found\n" + assert_pattern_list([ + /\s+\| \^ unterminated string;.+\n/, + /\s+\| \^ unterminated Unicode escape\n/, + /\s+\| \^ invalid Unicode escape sequence\n/, + ], e.message.lines[2..-1].join) + else + assert_pattern_list([ + /.*: invalid Unicode escape\n.*\n/, + / \^/, + /\n/, + /.*: unterminated Unicode escape\n.*\n/, + / \^/, + /\n/, + /.*: unterminated string.*\n.*\n/, + / \^\n/, + ], e.message) + end e = assert_syntax_error('"\M1"', /escape character syntax/) - assert_equal(' ^~~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~~(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\C1"', /escape character syntax/) - assert_equal(' ^~~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~~(?!~)/, e.message.lines.last, mesg) src = '"\xD0\u{90'"\n""000000000000000000000000" - assert_syntax_error(src, /:#{__LINE__}: unterminated/o) + assert_syntax_error(src, /(:#{__LINE__}:|> #{__LINE__} \|.+) unterminated/om) assert_syntax_error('"\u{100000000}"', /invalid Unicode escape/) assert_equal("", eval('"\u{}"')) @@ -605,22 +613,22 @@ class TestParse < Test::Unit::TestCase assert_syntax_error("\"\\C-\\M-\x01\"", 'Invalid escape character syntax') e = assert_syntax_error('"\c\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\c\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\C-\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\C-\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\M-\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\M-\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error(%["\\C-\u3042"], 'Invalid escape character syntax') - assert_match(/^\s \^(?# \\ ) ~(?# C ) ~(?# - ) ~+(?# U+3042 )$/x, e.message.lines.last) + assert_match(/(^|\|\s)\s \^(?# \\ ) ~(?# C ) ~(?# - ) ~+(?# U+3042 )($|\s)/x, e.message.lines.last) assert_not_include(e.message, "invalid multibyte char") end @@ -875,7 +883,8 @@ x = __ENCODING__ $test_parse_foobarbazqux = nil assert_equal(nil, $&) assert_equal(nil, eval('alias $& $preserve_last_match')) - assert_syntax_error('a = $#', /as a global variable name\na = \$\#\n \^~$/) + assert_syntax_error('a = $#', /as a global variable name/) + assert_syntax_error('a = $#', /a = \$\#\n(^|.+?\| ) \^~(?!~)/) end def test_invalid_instance_variable @@ -1259,8 +1268,8 @@ x = __ENCODING__ assert_syntax_error("def f r:def d; def f 0end", /unexpected/) end; - assert_syntax_error("def\nf(000)end", /^ \^~~/) - assert_syntax_error("def\nf(&0)end", /^ \^/) + assert_syntax_error("def\nf(000)end", /(^|\| ) \^~~/) + assert_syntax_error("def\nf(&0)end", /(^|\| ) \^/) end def test_method_location_in_rescue @@ -1336,17 +1345,21 @@ x = __ENCODING__ end def test_unexpected_token_after_numeric - assert_syntax_error('0000xyz', /^ \^~~\Z/) - assert_syntax_error('1.2i1.1', /^ \^~~\Z/) - assert_syntax_error('1.2.3', /^ \^~\Z/) + assert_syntax_error('0000xyz', /(^|\| ) \^~~(?!~)/) + assert_syntax_error('1.2i1.1', /(^|\| ) \^~~(?!~)/) + assert_syntax_error('1.2.3', /(^|\| ) \^~(?!~)/) assert_syntax_error('1.', /unexpected end-of-input/) assert_syntax_error('1e', /expecting end-of-input/) end def test_truncated_source_line - e = assert_syntax_error("'0123456789012345678901234567890123456789' abcdefghijklmnopqrstuvwxyz0123456789 0123456789012345678901234567890123456789", + lineno = __LINE__ + 1 + e = assert_syntax_error("'0123456789012345678901234567890123456789' abcdefghijklmnopqrstuvwxyz0123456789 123456789012345678901234567890123456789", /unexpected local variable or method/) + line = e.message.lines[1] + line.delete_prefix!("> #{lineno} | ") if line.start_with?(">") + assert_operator(line, :start_with?, "...") assert_operator(line, :end_with?, "...\n") end @@ -1390,11 +1403,11 @@ x = __ENCODING__ end def test_unexpected_eof - assert_syntax_error('unless', /^ \^\Z/) + assert_syntax_error('unless', /(^|\| ) \^(?!~)/) end def test_location_of_invalid_token - assert_syntax_error('class xxx end', /^ \^~~\Z/) + assert_syntax_error('class xxx end', /(^|\| ) \^~~(?!~)/) end def test_whitespace_warning diff --git a/test/ruby/test_pattern_matching.rb b/test/ruby/test_pattern_matching.rb index db6ad06b82..cfe3bd1e19 100644 --- a/test/ruby/test_pattern_matching.rb +++ b/test/ruby/test_pattern_matching.rb @@ -1331,7 +1331,7 @@ END end assert_block do - case {} + case C.new({}) in {} C.keys == nil end diff --git a/test/ruby/test_regexp.rb b/test/ruby/test_regexp.rb index c72029ca80..828117f516 100644 --- a/test/ruby/test_regexp.rb +++ b/test/ruby/test_regexp.rb @@ -1783,7 +1783,7 @@ class TestRegexp < Test::Unit::TestCase end t = Time.now - t - assert_in_delta(timeout, t, timeout / 2) + assert_operator(timeout, :<=, [timeout * 1.5, 1].max) end; end @@ -1862,7 +1862,7 @@ class TestRegexp < Test::Unit::TestCase def test_timeout_shorter_than_global omit "timeout test is too unstable on s390x" if RUBY_PLATFORM =~ /s390x/ - per_instance_redos_test(10, 0.2, 0.2) + per_instance_redos_test(10, 0.5, 0.5) end def test_timeout_longer_than_global @@ -1929,7 +1929,7 @@ class TestRegexp < Test::Unit::TestCase end def test_match_cache_positive_look_ahead - assert_separately([], "#{<<-"begin;"}\n#{<<-'end;'}") + assert_separately([], "#{<<-"begin;"}\n#{<<-'end;'}", timeout: 30) timeout = #{ EnvUtil.apply_timeout_scale(10).inspect } begin; Regexp.timeout = timeout diff --git a/test/ruby/test_require.rb b/test/ruby/test_require.rb index ef33928376..5aadf779fb 100644 --- a/test/ruby/test_require.rb +++ b/test/ruby/test_require.rb @@ -847,7 +847,7 @@ class TestRequire < Test::Unit::TestCase f.close File.unlink(f.path) File.mkfifo(f.path) - assert_separately(["-", f.path], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 3) + assert_separately(["-", f.path], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; th = Thread.current Thread.start {begin sleep(0.001) end until th.stop?; th.raise(IOError)} @@ -866,7 +866,7 @@ class TestRequire < Test::Unit::TestCase File.unlink(f.path) File.mkfifo(f.path) - assert_separately(["-", f.path], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 3) + assert_separately(["-", f.path], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; path = ARGV[0] th = Thread.current diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 76be9152a7..22663ed007 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -302,11 +302,8 @@ class TestRubyOptions < Test::Unit::TestCase end def test_parser_flag - warning = /compiler based on the Prism parser is currently experimental/ - - assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), warning) - assert_in_out_err(%w(--parser=prism -W:no-experimental -e) + ["puts :hi"], "", %w(hi), []) - assert_in_out_err(%w(--parser=prism -W:no-experimental --dump=parsetree -e _=:hi), "", /"hi"/, []) + assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), []) + assert_in_out_err(%w(--parser=prism --dump=parsetree -e _=:hi), "", /"hi"/, []) assert_in_out_err(%w(--parser=parse.y -e) + ["puts :hi"], "", %w(hi), []) assert_norun_with_rflag('--parser=parse.y', '--version', "") @@ -1208,15 +1205,12 @@ class TestRubyOptions < Test::Unit::TestCase end def test_frozen_string_literal_debug - default_frozen = eval("'test'").frozen? - with_debug_pat = /created at/ wo_debug_pat = /can\'t modify frozen String: "\w+" \(FrozenError\)\n\z/ frozen = [ ["--enable-frozen-string-literal", true], ["--disable-frozen-string-literal", false], ] - frozen << [nil, false] unless default_frozen debugs = [ ["--debug-frozen-string-literal", true], diff --git a/test/ruby/test_sprintf.rb b/test/ruby/test_sprintf.rb index c453ecd350..8cf2c63a20 100644 --- a/test/ruby/test_sprintf.rb +++ b/test/ruby/test_sprintf.rb @@ -266,8 +266,8 @@ class TestSprintf < Test::Unit::TestCase # Specifying the precision multiple times with negative star arguments: assert_raise(ArgumentError, "[ruby-core:11570]") {sprintf("%.*.*.*.*f", -1, -1, -1, 5, 1)} - # Null bytes after percent signs are removed: - assert_equal("%\0x hello", sprintf("%\0x hello"), "[ruby-core:11571]") + assert_raise(ArgumentError) {sprintf("%\0x hello")} + assert_raise(ArgumentError) {sprintf("%\nx hello")} assert_raise(ArgumentError, "[ruby-core:11573]") {sprintf("%.25555555555555555555555555555555555555s", "hello")} @@ -279,10 +279,9 @@ class TestSprintf < Test::Unit::TestCase assert_raise_with_message(ArgumentError, /unnumbered\(1\) mixed with numbered/) { sprintf("%1$*d", 3) } assert_raise_with_message(ArgumentError, /unnumbered\(1\) mixed with numbered/) { sprintf("%1$.*d", 3) } - verbose, $VERBOSE = $VERBOSE, nil - assert_nothing_raised { sprintf("", 1) } - ensure - $VERBOSE = verbose + assert_warning(/too many arguments/) do + sprintf("", 1) + end end def test_float diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index ebe85dac82..9bd86dfb78 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3617,17 +3617,16 @@ CODE def test_chilled_string chilled_string = eval('"chilled"') - # Chilled strings pretend to be frozen - assert_predicate chilled_string, :frozen? + assert_not_predicate chilled_string, :frozen? assert_not_predicate chilled_string.dup, :frozen? - assert_predicate chilled_string.clone, :frozen? + assert_not_predicate chilled_string.clone, :frozen? # @+ treat the original string as frozen assert_not_predicate(+chilled_string, :frozen?) assert_not_same chilled_string, +chilled_string - # @- the original string as mutable + # @- treat the original string as mutable assert_predicate(-chilled_string, :frozen?) assert_not_same chilled_string, -chilled_string end diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 44162f06cb..8aa3a54084 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -392,12 +392,11 @@ class TestSyntax < Test::Unit::TestCase end def test_keyword_self_reference - message = /circular argument reference - var/ - assert_syntax_error("def foo(var: defined?(var)) var end", message) - assert_syntax_error("def foo(var: var) var end", message) - assert_syntax_error("def foo(var: bar(var)) var end", message) - assert_syntax_error("def foo(var: bar {var}) var end", message) - assert_syntax_error("def foo(var: (1 in ^var)); end", message) + assert_valid_syntax("def foo(var: defined?(var)) var end") + assert_valid_syntax("def foo(var: var) var end") + assert_valid_syntax("def foo(var: bar(var)) var end") + assert_valid_syntax("def foo(var: bar {var}) var end") + assert_valid_syntax("def foo(var: (1 in ^var)); end") o = Object.new assert_warn("") do @@ -423,6 +422,9 @@ class TestSyntax < Test::Unit::TestCase assert_warn("") do o.instance_eval("proc {|var: 1| var}") end + + o = Object.new + assert_nil(o.instance_eval("def foo(bar: bar) = bar; foo")) end def test_keyword_invalid_name @@ -456,14 +458,13 @@ class TestSyntax < Test::Unit::TestCase end def test_optional_self_reference - message = /circular argument reference - var/ - assert_syntax_error("def foo(var = defined?(var)) var end", message) - assert_syntax_error("def foo(var = var) var end", message) - assert_syntax_error("def foo(var = bar(var)) var end", message) - assert_syntax_error("def foo(var = bar {var}) var end", message) - assert_syntax_error("def foo(var = (def bar;end; var)) var end", message) - assert_syntax_error("def foo(var = (def self.bar;end; var)) var end", message) - assert_syntax_error("def foo(var = (1 in ^var)); end", message) + assert_valid_syntax("def foo(var = defined?(var)) var end") + assert_valid_syntax("def foo(var = var) var end") + assert_valid_syntax("def foo(var = bar(var)) var end") + assert_valid_syntax("def foo(var = bar {var}) var end") + assert_valid_syntax("def foo(var = (def bar;end; var)) var end") + assert_valid_syntax("def foo(var = (def self.bar;end; var)) var end") + assert_valid_syntax("def foo(var = (1 in ^var)); end") o = Object.new assert_warn("") do @@ -489,6 +490,9 @@ class TestSyntax < Test::Unit::TestCase assert_warn("") do o.instance_eval("proc {|var = 1| var}") end + + o = Object.new + assert_nil(o.instance_eval("def foo(bar: bar) = bar; foo")) end def test_warn_grouped_expression @@ -506,10 +510,6 @@ class TestSyntax < Test::Unit::TestCase end def test_warn_balanced - warning = <<WARN -test:1: warning: '%s' after local variable or literal is interpreted as binary operator -test:1: warning: even though it seems like %s -WARN [ [:**, "argument prefix"], [:*, "argument prefix"], @@ -523,7 +523,9 @@ WARN all_assertions do |a| ["puts 1 #{op}0", "puts :a #{op}0", "m = 1; puts m #{op}0"].each do |src| a.for(src) do - assert_warning(warning % [op, syn], src) do + warning = /'#{Regexp.escape(op)}' after local variable or literal is interpreted as binary operator.+?even though it seems like #{syn}/m + + assert_warning(warning, src) do assert_valid_syntax(src, "test", verbose: true) end end @@ -715,8 +717,8 @@ WARN end def test_duplicated_when - w = 'warning: duplicated \'when\' clause with line 3 is ignored' - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + w = ->(line) { "warning: 'when' clause on line #{line} duplicates 'when' clause on line 3 and is ignored" } + assert_warning(/#{w[3]}.+#{w[4]}.+#{w[4]}.+#{w[5]}.+#{w[5]}/m) { eval %q{ case 1 when 1, 1 @@ -725,7 +727,7 @@ WARN end } } - assert_warning(/#{w}/) {#/3: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m){ + assert_warning(/#{w[3]}.+#{w[4]}.+#{w[5]}.+#{w[5]}/m) { a = a = 1 eval %q{ case 1 @@ -735,7 +737,7 @@ WARN end } } - assert_warning(/3: #{w}/m) { + assert_warning(/#{w[3]}/) { eval %q{ case 1 when __LINE__, __LINE__ @@ -744,7 +746,7 @@ WARN end } } - assert_warning(/3: #{w}/m) { + assert_warning(/#{w[3]}/) { eval %q{ case 1 when __FILE__, __FILE__ @@ -756,7 +758,7 @@ WARN end def test_duplicated_when_check_option - w = /duplicated \'when\' clause with line 3 is ignored/ + w = /'when' clause on line 4 duplicates 'when' clause on line 3 and is ignored/ assert_in_out_err(%[-wc], "#{<<~"begin;"}\n#{<<~'end;'}", ["Syntax OK"], w) begin; case 1 @@ -889,6 +891,16 @@ e" assert_dedented_heredoc(expect, result) end + def test_dedented_heredoc_with_leading_blank_line + # the blank line has six leading spaces + result = " \n" \ + " b\n" + expect = " \n" \ + "b\n" + assert_dedented_heredoc(expect, result) + end + + def test_dedented_heredoc_with_blank_more_indented_line_escaped result = " a\n" \ "\\ \\ \\ \\ \\ \\ \n" \ @@ -996,7 +1008,7 @@ eom end def test_dedented_heredoc_concatenation - assert_equal("\n0\n1", eval("<<~0 '1'\n \n0\#{}\n0")) + assert_equal(" \n0\n1", eval("<<~0 '1'\n \n0\#{}\n0")) end def test_heredoc_mixed_encoding @@ -1238,6 +1250,20 @@ eom assert_syntax_error("a&.x,=0", /multiple assignment destination/) end + def test_safe_call_in_for_variable + assert_valid_syntax("for x&.bar in []; end") + assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}") + begin; + foo = nil + for foo&.bar in [1]; end + assert_nil(foo) + + foo = Struct.new(:bar).new + for foo&.bar in [1]; end + assert_equal(1, foo.bar) + end; + end + def test_no_warning_logop_literal assert_warning("") do eval("true||raise;nil") @@ -1577,7 +1603,7 @@ eom end def test_syntax_error_at_newline - expected = "\n ^" + expected = /(\n|\| ) \^/ assert_syntax_error("%[abcdef", expected) assert_syntax_error("%[abcdef\n", expected) end @@ -1755,8 +1781,8 @@ eom assert_equal("instance ok", k.new.rescued("ok")) # Current technical limitation: cannot prepend "private" or something for command endless def - error = /syntax error, unexpected string literal/ - error2 = /syntax error, unexpected local variable or method/ + error = /(syntax error,|\^~*) unexpected string literal/ + error2 = /(syntax error,|\^~*) unexpected local variable or method/ assert_syntax_error('private def foo = puts "Hello"', error) assert_syntax_error('private def foo() = puts "Hello"', error) assert_syntax_error('private def foo(x) = puts x', error2) @@ -1900,7 +1926,7 @@ eom ] end assert_valid_syntax('proc {def foo(_);end;it}') - assert_syntax_error('p { [it **2] }', /unexpected \*\*arg/) + assert_syntax_error('p { [it **2] }', /unexpected \*\*/) end def test_value_expr_in_condition diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index da14c429e6..6620ccbf33 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -3,7 +3,6 @@ require 'test/unit' require "rbconfig/sizeof" require "timeout" -require "fiddle" class TestThread < Test::Unit::TestCase class Thread < ::Thread @@ -1446,13 +1445,16 @@ q.pop end def test_thread_native_thread_id_across_fork_on_linux - rtld_default = Fiddle.dlopen(nil) - omit "this test is only for Linux" unless rtld_default.sym_defined?('gettid') - - gettid = Fiddle::Function.new(rtld_default['gettid'], [], Fiddle::TYPE_INT) + begin + require '-test-/thread/id' + rescue LoadError + omit "this test is only for Linux" + else + extend Bug::ThreadID + end parent_thread_id = Thread.main.native_thread_id - real_parent_thread_id = gettid.call + real_parent_thread_id = gettid assert_equal real_parent_thread_id, parent_thread_id @@ -1464,7 +1466,7 @@ q.pop else # child puts Thread.main.native_thread_id - puts gettid.call + puts gettid end end child_thread_id = child_lines[0].chomp.to_i diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index 796787e355..44b935758d 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -56,14 +56,26 @@ class TestYJIT < Test::Unit::TestCase def test_yjit_enable args = [] args << "--disable=yjit" if RubyVM::YJIT.enabled? - assert_separately(args, <<~RUBY) - assert_false RubyVM::YJIT.enabled? - assert_false RUBY_DESCRIPTION.include?("+YJIT") + assert_separately(args, <<~'RUBY') + refute_predicate RubyVM::YJIT, :enabled? + refute_includes RUBY_DESCRIPTION, "+YJIT" RubyVM::YJIT.enable - assert_true RubyVM::YJIT.enabled? - assert_true RUBY_DESCRIPTION.include?("+YJIT") + assert_predicate RubyVM::YJIT, :enabled? + assert_includes RUBY_DESCRIPTION, "+YJIT" + RUBY + end + + def test_yjit_disable + assert_separately(["--yjit", "--yjit-disable"], <<~'RUBY') + refute_predicate RubyVM::YJIT, :enabled? + refute_includes RUBY_DESCRIPTION, "+YJIT" + + RubyVM::YJIT.enable + + assert_predicate RubyVM::YJIT, :enabled? + assert_includes RUBY_DESCRIPTION, "+YJIT" RUBY end diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index f97306717d..7014843bba 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -76,8 +76,6 @@ class Gem::TestCase < Test::Unit::TestCase attr_accessor :uri # :nodoc: - @@tempdirs = [] - def assert_activate(expected, *specs) specs.each do |spec| case spec @@ -451,9 +449,7 @@ class Gem::TestCase < Test::Unit::TestCase Dir.chdir @current_dir - FileUtils.rm_rf @tempdir - - restore_env + ENV.replace(@orig_env) Gem::ConfigFile.send :remove_const, :SYSTEM_WIDE_CONFIG_FILE Gem::ConfigFile.send :const_set, :SYSTEM_WIDE_CONFIG_FILE, @@ -481,12 +477,9 @@ class Gem::TestCase < Test::Unit::TestCase @back_ui.close - refute_directory_exists @tempdir, "may be still in use" - ghosts = @@tempdirs.filter_map do |test_name, tempdir| - test_name if File.exist?(tempdir) - end - @@tempdirs << [method_name, @tempdir] - assert_empty ghosts + FileUtils.rm_rf @tempdir + + refute_directory_exists @tempdir, "#{@tempdir} used by test #{method_name} is still in use" end def credential_setup @@ -541,6 +534,16 @@ class Gem::TestCase < Test::Unit::TestCase ENV["BUNDLE_GEMFILE"] = File.join(@tempdir, "Gemfile") end + def with_env(overrides, &block) + orig_env = ENV.to_h + ENV.replace(overrides) + begin + block.call + ensure + ENV.replace(orig_env) + end + end + ## # A git_gem is used with a gem dependencies file. The gem created here # has no files, just a gem specification for the given +name+ and +version+. @@ -1526,23 +1529,6 @@ Also, a list: PUBLIC_KEY = nil PUBLIC_CERT = nil end if Gem::HAVE_OPENSSL - - private - - def restore_env - unless Gem.win_platform? - ENV.replace(@orig_env) - return - end - - # Fallback logic for Windows below to workaround - # https://bugs.ruby-lang.org/issues/16798. Can be dropped once all - # supported rubies include the fix for that. - - ENV.clear - - @orig_env.each {|k, v| ENV[k] = v } - end end # https://github.com/seattlerb/minitest/blob/13c48a03d84a2a87855a4de0c959f96800100357/lib/minitest/mock.rb#L192 diff --git a/test/rubygems/test_bundled_ca.rb b/test/rubygems/test_bundled_ca.rb index 50e621f22b..a737185681 100644 --- a/test/rubygems/test_bundled_ca.rb +++ b/test/rubygems/test_bundled_ca.rb @@ -33,7 +33,7 @@ class TestGemBundledCA < Gem::TestCase http.verify_mode = OpenSSL::SSL::VERIFY_PEER http.cert_store = bundled_certificate_store http.get("/") - rescue Errno::ENOENT, Errno::ETIMEDOUT, SocketError, Net::OpenTimeout + rescue Errno::ENOENT, Errno::ETIMEDOUT, SocketError, Gem::Net::OpenTimeout pend "#{host} seems offline, I can't tell whether ssl would work." rescue OpenSSL::SSL::SSLError => e # Only fail for certificate verification errors diff --git a/test/rubygems/test_gem.rb b/test/rubygems/test_gem.rb index 3fb66e1407..e8a294d65c 100644 --- a/test/rubygems/test_gem.rb +++ b/test/rubygems/test_gem.rb @@ -516,7 +516,10 @@ class TestGem < Gem::TestCase Gem.clear_paths - assert_nil Gem::Specification.send(:class_variable_get, :@@all) + with_env("GEM_HOME" => "foo", "GEM_PATH" => "bar") do + assert_equal("foo", Gem.dir) + assert_equal("bar", Gem.path.first) + end end def test_self_configuration @@ -1559,21 +1562,20 @@ class TestGem < Gem::TestCase assert_equal m1.gem_dir, File.join(Gem.user_dir, "gems", "m-1") tests = [ - [:dir0, [Gem.dir, Gem.user_dir], m0], - [:dir1, [Gem.user_dir, Gem.dir], m1], + [:dir0, [Gem.dir, Gem.user_dir]], + [:dir1, [Gem.user_dir, Gem.dir]], ] - tests.each do |name, paths, expected| + tests.each do |name, paths| Gem.use_paths paths.first, paths - Gem::Specification.reset Gem.searcher = nil assert_equal Gem::Dependency.new("m","1").to_specs, Gem::Dependency.new("m","1").to_specs.sort assert_equal \ - [expected.gem_dir], - Gem::Dependency.new("m","1").to_specs.map(&:gem_dir).sort, + [m0.gem_dir, m1.gem_dir], + Gem::Dependency.new("m","1").to_specs.map(&:gem_dir).uniq.sort, "Wrong specs for #{name}" spec = Gem::Dependency.new("m","1").to_spec @@ -1613,9 +1615,11 @@ class TestGem < Gem::TestCase Gem.use_paths Gem.dir, [Gem.dir, Gem.user_dir] + spec = Gem::Dependency.new("m", "1").to_spec + assert_equal \ File.join(Gem.dir, "gems", "m-1"), - Gem::Dependency.new("m","1").to_spec.gem_dir, + spec.gem_dir, "Wrong spec selected" end diff --git a/test/rubygems/test_gem_ci_detector.rb b/test/rubygems/test_gem_ci_detector.rb index 3caefce97d..a28ee49f4b 100644 --- a/test/rubygems/test_gem_ci_detector.rb +++ b/test/rubygems/test_gem_ci_detector.rb @@ -3,7 +3,7 @@ require_relative "helper" require "rubygems" -class TestCiDetector < Test::Unit::TestCase +class TestCiDetector < Gem::TestCase def test_ci? with_env("FOO" => "bar") { assert_equal(false, Gem::CIDetector.ci?) } with_env("CI" => "true") { assert_equal(true, Gem::CIDetector.ci?) } @@ -29,16 +29,4 @@ class TestCiDetector < Test::Unit::TestCase assert_equal(["dsari", "taskcluster"], Gem::CIDetector.ci_strings) end end - - private - - def with_env(overrides, &block) - @orig_env = ENV.to_h - ENV.replace(overrides) - begin - block.call - ensure - ENV.replace(@orig_env) - end - end end diff --git a/test/rubygems/test_gem_commands_pristine_command.rb b/test/rubygems/test_gem_commands_pristine_command.rb index a17d7837c9..b8b39133ff 100644 --- a/test/rubygems/test_gem_commands_pristine_command.rb +++ b/test/rubygems/test_gem_commands_pristine_command.rb @@ -96,7 +96,7 @@ class TestGemCommandsPristineCommand < Gem::TestCase out = @ui.output.split("\n") assert_equal "Restoring gems to pristine condition...", out.shift - assert_equal "Restored #{a.full_name}", out.shift + assert_equal "Restored #{a.full_name} in #{Gem.user_dir}", out.shift assert_empty out, out.inspect ensure FileUtils.chmod(0o755, @gemhome) @@ -404,7 +404,7 @@ class TestGemCommandsPristineCommand < Gem::TestCase out = @ui.output.split "\n" assert_equal "Restoring gems to pristine condition...", out.shift - assert_equal "Restored #{a.full_name}", out.shift + assert_equal "Restored #{a.full_name} in #{@gemhome}", out.shift assert_equal "Restored #{b.full_name}", out.shift assert_empty out, out.inspect @@ -476,8 +476,9 @@ class TestGemCommandsPristineCommand < Gem::TestCase [ "Restoring gems to pristine condition...", - "Cached gem for a-1 not found, attempting to fetch...", - "Restored a-1", + "Cached gem for a-1 in #{@gemhome} not found, attempting to fetch...", + "Restored a-1 in #{@gemhome}", + "Restored b-1 in #{@gemhome}", "Cached gem for b-1 not found, attempting to fetch...", "Restored b-1", ].each do |line| @@ -495,7 +496,7 @@ class TestGemCommandsPristineCommand < Gem::TestCase assert_path_exist File.join(gemhome2, "cache", "b-1.gem") assert_path_not_exist File.join(@gemhome, "cache", "b-2.gem") assert_path_exist File.join(gemhome2, "gems", "b-1") - assert_path_not_exist File.join(@gemhome, "gems", "b-1") + assert_path_exist File.join(@gemhome, "gems", "b-1") end def test_execute_no_gem diff --git a/test/rubygems/test_gem_commands_rebuild_command.rb b/test/rubygems/test_gem_commands_rebuild_command.rb index 5e8c797e2d..3b7927c44e 100644 --- a/test/rubygems/test_gem_commands_rebuild_command.rb +++ b/test/rubygems/test_gem_commands_rebuild_command.rb @@ -105,7 +105,7 @@ class TestGemCommandsRebuildCommand < Gem::TestCase assert_equal old_spec.name, new_spec.name assert_equal old_spec.summary, new_spec.summary - reproduced + [reproduced, original] end def test_build_is_reproducible @@ -134,12 +134,21 @@ class TestGemCommandsRebuildCommand < Gem::TestCase # also testing that `gem rebuild` overrides the value. ENV["SOURCE_DATE_EPOCH"] = Time.new(2007, 8, 9, 10, 11, 12).to_s - rebuild_gem_file = util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) + rebuild_gem_file, saved_gem_file = + util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) rebuild_contents = File.read(rebuild_gem_file) assert_equal build_contents, rebuild_contents ensure ENV["SOURCE_DATE_EPOCH"] = epoch + if rebuild_gem_file + File.unlink(rebuild_gem_file) + dir = File.dirname(rebuild_gem_file) + Dir.rmdir(dir) + File.unlink(saved_gem_file) + Dir.rmdir(File.dirname(saved_gem_file)) + Dir.rmdir(File.dirname(dir)) + end end end diff --git a/test/rubygems/test_gem_commands_setup_command.rb b/test/rubygems/test_gem_commands_setup_command.rb index 43f695f147..8eedb6c03a 100644 --- a/test/rubygems/test_gem_commands_setup_command.rb +++ b/test/rubygems/test_gem_commands_setup_command.rb @@ -159,6 +159,23 @@ class TestGemCommandsSetupCommand < Gem::TestCase end end + def test_destdir_flag_regenerates_binstubs + # install to destdir + destdir = File.join(@tempdir, "foo") + gem_bin_path = gem_install "destdir-only-gem", install_dir: destdir + + # change binstub manually + write_file gem_bin_path do |io| + io.puts "I changed it!" + end + + @cmd.options[:destdir] = destdir + @cmd.options[:prefix] = "/" + @cmd.execute + + assert_match(/\A#!/, File.read(gem_bin_path)) + end + def test_files_in assert_equal %w[rubygems.rb rubygems/requirement.rb rubygems/ssl_certs/rubygems.org/foo.pem], @cmd.files_in("lib").sort @@ -412,7 +429,7 @@ class TestGemCommandsSetupCommand < Gem::TestCase end end - def gem_install(name) + def gem_install(name, **options) gem = util_spec name do |s| s.executables = [name] s.files = %W[bin/#{name}] @@ -420,8 +437,8 @@ class TestGemCommandsSetupCommand < Gem::TestCase write_file File.join @tempdir, "bin", name do |f| f.puts "#!/usr/bin/ruby" end - install_gem gem - File.join @gemhome, "bin", name + install_gem gem, **options + File.join options[:install_dir] || @gemhome, "bin", name end def gem_install_with_plugin(name) diff --git a/test/rubygems/test_gem_commands_uninstall_command.rb b/test/rubygems/test_gem_commands_uninstall_command.rb index 4daa61cb0c..66885395d8 100644 --- a/test/rubygems/test_gem_commands_uninstall_command.rb +++ b/test/rubygems/test_gem_commands_uninstall_command.rb @@ -32,7 +32,7 @@ class TestGemCommandsUninstallCommand < Gem::InstallerTestCase @cmd.execute end - assert_equal %w[a_evil-9 b-2 c-1.2 default-1 dep_x-1 pl-1-x86-linux x-1], + assert_equal %w[a-4 a_evil-9 b-2 c-1.2 default-1 dep_x-1 pl-1-x86-linux x-1], Gem::Specification.all_names.sort end diff --git a/test/rubygems/test_gem_commands_update_command.rb b/test/rubygems/test_gem_commands_update_command.rb index 2683840f2e..194ebb030a 100644 --- a/test/rubygems/test_gem_commands_update_command.rb +++ b/test/rubygems/test_gem_commands_update_command.rb @@ -217,7 +217,15 @@ class TestGemCommandsUpdateCommand < Gem::TestCase end def test_execute_system_update_installed_in_non_default_gem_path - rubygems_update_spec = quick_gem "rubygems-update", 9 do |s| + rubygems_update_spec = Gem::Specification.new do |s| + s.name = "rubygems-update" + s.version = "9" + s.author = "A User" + s.email = "example@example.com" + s.homepage = "http://example.com" + s.summary = "this is a summary" + s.description = "This is a test description" + write_file File.join(@tempdir, "setup.rb") s.files += %w[setup.rb] diff --git a/test/rubygems/test_gem_package_tar_header.rb b/test/rubygems/test_gem_package_tar_header.rb index 4469750f9a..a3f95bb770 100644 --- a/test/rubygems/test_gem_package_tar_header.rb +++ b/test/rubygems/test_gem_package_tar_header.rb @@ -99,6 +99,31 @@ class TestGemPackageTarHeader < Gem::Package::TarTestCase assert_empty @tar_header end + def test_empty + @tar_header = Gem::Package::TarHeader.from(StringIO.new(Gem::Package::TarHeader::EMPTY_HEADER)) + + assert_empty @tar_header + assert_equal Gem::Package::TarHeader.new( + checksum: 0, + devmajor: 0, + devminor: 0, + empty: true, + gid: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + mtime: 0, + name: "", + prefix: "", + size: 0, + typeflag: "0", + uid: 0, + uname: "", + version: 0, + ), @tar_header + end + def test_equals2 assert_equal @tar_header, @tar_header assert_equal @tar_header, @tar_header.dup diff --git a/test/rubygems/test_gem_platform.rb b/test/rubygems/test_gem_platform.rb index e4bf882317..00e48498c6 100644 --- a/test/rubygems/test_gem_platform.rb +++ b/test/rubygems/test_gem_platform.rb @@ -145,6 +145,9 @@ class TestGemPlatform < Gem::TestCase "x86_64-openbsd3.9" => ["x86_64", "openbsd", "3.9"], "x86_64-openbsd4.0" => ["x86_64", "openbsd", "4.0"], "x86_64-openbsd" => ["x86_64", "openbsd", nil], + "wasm32-wasi" => ["wasm32", "wasi", nil], + "wasm32-wasip1" => ["wasm32", "wasi", nil], + "wasm32-wasip2" => ["wasm32", "wasi", nil], } test_cases.each do |arch, expected| diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index f9f063c797..9395e34f75 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -967,7 +967,10 @@ dependencies: [] def test_self_stubs_for_lazy_loading Gem.loaded_specs.clear - Gem::Specification.class_variable_set(:@@stubs, nil) + + specification_record = Gem::Specification.specification_record + + specification_record.instance_variable_set(:@stubs, nil) dir_standard_specs = File.join Gem.dir, "specifications" @@ -975,9 +978,9 @@ dependencies: [] save_gemspec("b-1", "1", dir_standard_specs) {|s| s.name = "b" } assert_equal ["a-1"], Gem::Specification.stubs_for("a").map(&:full_name) - assert_equal 1, Gem::Specification.class_variable_get(:@@stubs_by_name).length + assert_equal 1, specification_record.instance_variable_get(:@stubs_by_name).length assert_equal ["b-1"], Gem::Specification.stubs_for("b").map(&:full_name) - assert_equal 2, Gem::Specification.class_variable_get(:@@stubs_by_name).length + assert_equal 2, specification_record.instance_variable_get(:@stubs_by_name).length assert_equal( Gem::Specification.stubs_for("a").map(&:object_id), @@ -986,7 +989,7 @@ dependencies: [] Gem.loaded_specs.delete "a" Gem.loaded_specs.delete "b" - Gem::Specification.class_variable_set(:@@stubs, nil) + specification_record.instance_variable_set(:@stubs, nil) end def test_self_stubs_for_no_lazy_loading_after_all_specs_setup diff --git a/test/rubygems/test_gem_uninstaller.rb b/test/rubygems/test_gem_uninstaller.rb index 9e0c1aa3d8..794aaf1aec 100644 --- a/test/rubygems/test_gem_uninstaller.rb +++ b/test/rubygems/test_gem_uninstaller.rb @@ -19,8 +19,6 @@ class TestGemUninstaller < Gem::InstallerTestCase @user_spec = @user_installer.spec end end - - Gem::Specification.reset end def test_initialize_expand_path @@ -188,22 +186,81 @@ class TestGemUninstaller < Gem::InstallerTestCase refute File.exist?(plugin_path), "plugin not removed" end - def test_remove_plugins_with_install_dir + def test_uninstall_with_install_dir_removes_plugins write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| io.write "# do nothing" end @spec.files += %w[lib/rubygems_plugin.rb] - Gem::Installer.at(Gem::Package.build(@spec), force: true).install + package = Gem::Package.build(@spec) + + Gem::Installer.at(package, force: true).install plugin_path = File.join Gem.plugindir, "a_plugin.rb" assert File.exist?(plugin_path), "plugin not written" - Dir.mkdir "#{@gemhome}2" - Gem::Uninstaller.new(nil, install_dir: "#{@gemhome}2").remove_plugins @spec + install_dir = "#{@gemhome}2" + + Gem::Installer.at(package, force: true, install_dir: install_dir).install + + install_dir_plugin_path = File.join install_dir, "plugins/a_plugin.rb" + assert File.exist?(install_dir_plugin_path), "plugin not written" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, executables: true, install_dir: install_dir).uninstall assert File.exist?(plugin_path), "plugin unintentionally removed" + refute File.exist?(install_dir_plugin_path), "plugin not removed" + end + + def test_uninstall_with_install_dir_regenerates_plugins + write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| + io.write "# do nothing" + end + + @spec.files += %w[lib/rubygems_plugin.rb] + + install_dir = "#{@gemhome}2" + + package = Gem::Package.build(@spec) + + spec_v9 = @spec.dup + spec_v9.version = "9" + package_v9 = Gem::Package.build(spec_v9) + + Gem::Installer.at(package, force: true, install_dir: install_dir).install + Gem::Installer.at(package_v9, force: true, install_dir: install_dir).install + + install_dir_plugin_path = File.join install_dir, "plugins/a_plugin.rb" + assert File.exist?(install_dir_plugin_path), "plugin not written" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, version: "9", executables: true, install_dir: install_dir).uninstall + assert File.exist?(install_dir_plugin_path), "plugin unintentionally removed" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, executables: true, install_dir: install_dir).uninstall + refute File.exist?(install_dir_plugin_path), "plugin not removed" + end + + def test_remove_plugins_user_installed + write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| + io.write "# do nothing" + end + + @spec.files += %w[lib/rubygems_plugin.rb] + + Gem::Installer.at(Gem::Package.build(@spec), force: true, user_install: true).install + + plugin_path = File.join Gem.user_dir, "plugins/a_plugin.rb" + assert File.exist?(plugin_path), "plugin not written" + + Gem::Specification.dirs = [Gem.dir, Gem.user_dir] + + Gem::Uninstaller.new(@spec.name, executables: true, force: true, user_install: true).uninstall + + refute File.exist?(plugin_path), "plugin not removed" end def test_regenerate_plugins_for @@ -370,7 +427,7 @@ create_makefile '#{@spec.name}' end def test_uninstall_user_install - @user_spec = Gem::Specification.find_by_name "b" + Gem::Specification.dirs = [Gem.user_dir] uninstaller = Gem::Uninstaller.new(@user_spec.name, executables: true, @@ -394,6 +451,32 @@ create_makefile '#{@spec.name}' assert_same uninstaller, @post_uninstall_hook_arg end + def test_uninstall_user_install_with_symlinked_home + pend "Symlinks not supported or not enabled" unless symlink_supported? + + Gem::Specification.dirs = [Gem.user_dir] + + symlinked_home = File.join(@tempdir, "new-home") + FileUtils.ln_s(Gem.user_home, symlinked_home) + + ENV["HOME"] = symlinked_home + Gem.instance_variable_set(:@user_home, nil) + Gem.instance_variable_set(:@data_home, nil) + + uninstaller = Gem::Uninstaller.new(@user_spec.name, + executables: true, + user_install: true, + force: true) + + gem_dir = File.join @user_spec.gem_dir + + assert_path_exist gem_dir + + uninstaller.uninstall + + assert_path_not_exist gem_dir + end + def test_uninstall_wrong_repo Dir.mkdir "#{@gemhome}2" Gem.use_paths "#{@gemhome}2", [@gemhome] diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 9031650581..aeccac2577 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -990,7 +990,8 @@ class TestStringIO < Test::Unit::TestCase assert_predicate(s.string, :ascii_only?) end - if eval(%{ "test".frozen? && !"test".equal?("test") }) # Ruby 3.4+ chilled strings + require "objspace" + if ObjectSpace.respond_to?(:dump) && ObjectSpace.dump(eval(%{"test"})).include?('"chilled":true') # Ruby 3.4+ chilled strings def test_chilled_string chilled_string = eval(%{""}) io = StringIO.new(chilled_string) diff --git a/test/test_tempfile.rb b/test/test_tempfile.rb index eddbac5d75..d4ae7d4b3f 100644 --- a/test/test_tempfile.rb +++ b/test/test_tempfile.rb @@ -425,4 +425,54 @@ puts Tempfile.new('foo').path assert_not_send([File.absolute_path(actual), :start_with?, target]) end end + + def test_create_anonymous_without_block + t = Tempfile.create(anonymous: true) + assert_equal(File, t.class) + assert_equal(0600, t.stat.mode & 0777) unless /mswin|mingw/ =~ RUBY_PLATFORM + t.puts "foo" + t.rewind + assert_equal("foo\n", t.read) + t.close + ensure + t.close if t + end + + def test_create_anonymous_with_block + result = Tempfile.create(anonymous: true) {|t| + assert_equal(File, t.class) + assert_equal(0600, t.stat.mode & 0777) unless /mswin|mingw/ =~ RUBY_PLATFORM + t.puts "foo" + t.rewind + assert_equal("foo\n", t.read) + :result + } + assert_equal(:result, result) + end + + def test_create_anonymous_removes_file + Dir.mktmpdir {|d| + t = Tempfile.create("", d, anonymous: true) + t.close + assert_equal([], Dir.children(d)) + } + end + + def test_create_anonymous_path + Dir.mktmpdir {|d| + begin + t = Tempfile.create("", d, anonymous: true) + assert_equal(File.join(d, ""), t.path) + ensure + t.close if t + end + } + end + + def test_create_anonymous_autoclose + Tempfile.create(anonymous: true) {|t| + assert_equal(true, t.autoclose?) + } + end + end diff --git a/test/test_timeout.rb b/test/test_timeout.rb index e900b10cd7..34966f92a4 100644 --- a/test/test_timeout.rb +++ b/test/test_timeout.rb @@ -66,7 +66,7 @@ class TestTimeout < Test::Unit::TestCase a = nil assert_raise(Timeout::Error) do Timeout.timeout(0.1) { - Timeout.timeout(1) { + Timeout.timeout(30) { nil while true } a = 1 @@ -84,7 +84,7 @@ class TestTimeout < Test::Unit::TestCase def test_nested_timeout_error_identity begin Timeout.timeout(0.1, MyNewErrorOuter) { - Timeout.timeout(1, MyNewErrorInner) { + Timeout.timeout(30, MyNewErrorInner) { nil while true } } diff --git a/test/zlib/test_zlib.rb b/test/zlib/test_zlib.rb index ae4adc21fe..15e5bd852f 100644 --- a/test/zlib/test_zlib.rb +++ b/test/zlib/test_zlib.rb @@ -991,6 +991,25 @@ if defined? Zlib assert_raise(ArgumentError) { f.read(-1) } assert_equal(str, f.read) end + + Zlib::GzipReader.open(t.path) do |f| + s = "".b + + assert_raise(ArgumentError) { f.read(-1, s) } + + assert_same s, f.read(1, s) + assert_equal "\xE3".b, s + + assert_same s, f.read(2, s) + assert_equal "\x81\x82".b, s + + assert_same s, f.read(6, s) + assert_equal "\u3044\u3046".b, s + + assert_nil f.read(1, s) + assert_equal "".b, s + assert_predicate f, :eof? + end } end @@ -1005,10 +1024,14 @@ if defined? Zlib Zlib::GzipReader.open(t.path) do |f| s = "".dup - f.readpartial(3, s) + assert_same s, f.readpartial(3, s) assert("foo".start_with?(s)) assert_raise(ArgumentError) { f.readpartial(-1) } + + assert_same s, f.readpartial(3, s) + + assert_predicate f, :eof? end } end @@ -197,6 +197,10 @@ static inline void blocking_region_end(rb_thread_t *th, struct rb_blocking_regio if (blocking_region_begin(th, &__region, (ubf), (ubfarg), fail_if_interrupted) || \ /* always return true unless fail_if_interrupted */ \ !only_if_constant(fail_if_interrupted, TRUE)) { \ + /* Important that this is inlined into the macro, and not part of \ + * blocking_region_begin - see bug #20493 */ \ + RB_VM_SAVE_MACHINE_CONTEXT(th); \ + thread_sched_to_waiting(TH_SCHED(th), th); \ exec; \ blocking_region_end(th, &__region); \ }; \ @@ -1482,9 +1486,6 @@ blocking_region_begin(rb_thread_t *th, struct rb_blocking_region_buffer *region, rb_ractor_blocking_threads_inc(th->ractor, __FILE__, __LINE__); RUBY_DEBUG_LOG("thread_id:%p", (void *)th->nt->thread_id); - - RB_VM_SAVE_MACHINE_CONTEXT(th); - thread_sched_to_waiting(TH_SCHED(th), th); return TRUE; } else { @@ -1540,10 +1541,12 @@ rb_nogvl(void *(*func)(void *), void *data1, } } + rb_vm_t *volatile saved_vm = vm; BLOCKING_REGION(th, { val = func(data1); saved_errno = rb_errno(); }, ubf, data2, flags & RB_NOGVL_INTR_FAIL); + vm = saved_vm; if (is_main_thread) vm->ubf_async_safe = 0; @@ -1767,7 +1770,7 @@ rb_thread_mn_schedulable(VALUE thval) VALUE rb_thread_io_blocking_call(rb_blocking_function_t *func, void *data1, int fd, int events) { - rb_execution_context_t * volatile ec = GET_EC(); + rb_execution_context_t *volatile ec = GET_EC(); rb_thread_t *th = rb_ec_thread_ptr(ec); RUBY_DEBUG_LOG("th:%u fd:%d ev:%d", rb_th_serial(th), fd, events); @@ -1789,21 +1792,25 @@ rb_thread_io_blocking_call(rb_blocking_function_t *func, void *data1, int fd, in { EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { + volatile enum ruby_tag_type saved_state = state; /* for BLOCKING_REGION */ retry: BLOCKING_REGION(waiting_fd.th, { val = func(data1); saved_errno = errno; }, ubf_select, waiting_fd.th, FALSE); + th = rb_ec_thread_ptr(ec); if (events && blocking_call_retryable_p((int)val, saved_errno) && thread_io_wait_events(th, fd, events, NULL)) { RUBY_VM_CHECK_INTS_BLOCKING(ec); goto retry; } + state = saved_state; } EC_POP_TAG(); + th = rb_ec_thread_ptr(ec); th->mn_schedulable = prev_mn_schedulable; } /* @@ -1893,6 +1900,8 @@ rb_thread_call_with_gvl(void *(*func)(void *), void *data1) /* leave from Ruby world: You can not access Ruby values, etc. */ int released = blocking_region_begin(th, brb, prev_unblock.func, prev_unblock.arg, FALSE); RUBY_ASSERT_ALWAYS(released); + RB_VM_SAVE_MACHINE_CONTEXT(th); + thread_sched_to_waiting(TH_SCHED(th), th); return r; } @@ -4205,9 +4214,10 @@ rb_fd_set(int fd, rb_fdset_t *set) #endif static int -wait_retryable(int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) +wait_retryable(volatile int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) { - if (*result < 0) { + int r = *result; + if (r < 0) { switch (errnum) { case EINTR: #ifdef ERESTART @@ -4221,7 +4231,7 @@ wait_retryable(int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) } return FALSE; } - else if (*result == 0) { + else if (r == 0) { /* check for spurious wakeup */ if (rel) { return !hrtime_update_expire(rel, end); @@ -4259,11 +4269,12 @@ static VALUE do_select(VALUE p) { struct select_set *set = (struct select_set *)p; - int result = 0; + volatile int result = 0; int lerrno; rb_hrtime_t *to, rel, end = 0; timeout_prepare(&to, &rel, &end, set->timeout); + volatile rb_hrtime_t endtime = end; #define restore_fdset(dst, src) \ ((dst) ? rb_fd_dup(dst, src) : (void)0) #define do_select_update() \ @@ -4279,15 +4290,15 @@ do_select(VALUE p) struct timeval tv; if (!RUBY_VM_INTERRUPTED(set->th->ec)) { - result = native_fd_select(set->max, - set->rset, set->wset, set->eset, - rb_hrtime2timeval(&tv, to), set->th); + result = native_fd_select(set->max, + set->rset, set->wset, set->eset, + rb_hrtime2timeval(&tv, to), set->th); if (result < 0) lerrno = errno; } }, ubf_select, set->th, TRUE); RUBY_VM_CHECK_INTS_BLOCKING(set->th->ec); /* may raise */ - } while (wait_retryable(&result, lerrno, to, end) && do_select_update()); + } while (wait_retryable(&result, lerrno, to, endtime) && do_select_update()); if (result < 0) { errno = lerrno; @@ -4349,6 +4360,23 @@ rb_thread_fd_select(int max, rb_fdset_t * read, rb_fdset_t * write, rb_fdset_t * # define POLLERR_SET (0) #endif +static int +wait_for_single_fd_blocking_region(rb_thread_t *th, struct pollfd *fds, nfds_t nfds, + rb_hrtime_t *const to, volatile int *lerrno) +{ + struct timespec ts; + volatile int result = 0; + + *lerrno = 0; + BLOCKING_REGION(th, { + if (!RUBY_VM_INTERRUPTED(th->ec)) { + result = ppoll(fds, nfds, rb_hrtime2timespec(&ts, to), 0); + if (result < 0) *lerrno = errno; + } + }, ubf_select, th, TRUE); + return result; +} + /* * returns a mask of events */ @@ -4360,7 +4388,7 @@ rb_thread_wait_for_single_fd(int fd, int events, struct timeval *timeout) .events = (short)events, .revents = 0, }}; - int result = 0; + volatile int result = 0; nfds_t nfds; struct waiting_fd wfd; enum ruby_tag_type state; @@ -4384,17 +4412,8 @@ rb_thread_wait_for_single_fd(int fd, int events, struct timeval *timeout) RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); timeout_prepare(&to, &rel, &end, timeout); do { - nfds = 1; - - lerrno = 0; - BLOCKING_REGION(wfd.th, { - struct timespec ts; - - if (!RUBY_VM_INTERRUPTED(wfd.th->ec)) { - result = ppoll(fds, nfds, rb_hrtime2timespec(&ts, to), 0); - if (result < 0) lerrno = errno; - } - }, ubf_select, wfd.th, TRUE); + nfds = numberof(fds); + result = wait_for_single_fd_blocking_region(wfd.th, fds, nfds, to, &lerrno); RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); } while (wait_retryable(&result, lerrno, to, end)); diff --git a/thread_pthread.c b/thread_pthread.c index 82b5e362cc..b9421559f2 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -2569,10 +2569,7 @@ ubf_wakeup_thread(rb_thread_t *th) { RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); - int r = pthread_kill(th->nt->thread_id, SIGVTALRM); - if (r != 0) { - rb_bug_errno("pthread_kill", r); - } + pthread_kill(th->nt->thread_id, SIGVTALRM); } static void @@ -5725,16 +5725,6 @@ rb_time_zone_abbreviation(VALUE zone, VALUE time) return rb_obj_as_string(abbr); } -/* Internal Details: - * - * Since Ruby 1.9.2, Time implementation uses a signed 63 bit integer or - * Integer(T_BIGNUM), Rational. - * The integer is a number of nanoseconds since the _Epoch_ which can - * represent 1823-11-12 to 2116-02-20. - * When Integer(T_BIGNUM) or Rational is used (before 1823, after 2116, under - * nanosecond), Time works slower than when integer is used. - */ - // void Init_Time(void) @@ -45,6 +45,46 @@ # (for example, by method Time.now) # has the resolution supported by the system. # +# == \Time Internal Representation +# +# Time implementation uses a signed 63 bit integer, Integer(T_BIGNUM), or +# Rational. +# It is a number of nanoseconds since the _Epoch_. +# The signed 63 bit integer can represent 1823-11-12 to 2116-02-20. +# When Integer or Rational is used (before 1823, after 2116, under +# nanosecond), Time works slower than when the signed 63 bit integer is used. +# +# Ruby uses the C function "localtime" and "gmtime" to map between the number +# and 6-tuple (year,month,day,hour,minute,second). +# "localtime" is used for local time and "gmtime" is used for UTC. +# +# Integer(T_BIGNUM) and Rational has no range limit, +# but the localtime and gmtime has range limits +# due to the C types "time_t" and "struct tm". +# If that limit is exceeded, Ruby extrapolates the localtime function. +# +# The Time class always uses the Gregorian calendar. +# I.e. the proleptic Gregorian calendar is used. +# Other calendars, such as Julian calendar, are not supported. +# +# "time_t" can represent 1901-12-14 to 2038-01-19 if it is 32 bit signed integer, +# -292277022657-01-27 to 292277026596-12-05 if it is 64 bit signed integer. +# However "localtime" on some platforms doesn't supports negative time_t (before 1970). +# +# "struct tm" has tm_year member to represent years. +# (tm_year = 0 means the year 1900.) +# It is defined as int in the C standard. +# tm_year can represent between -2147481748 to 2147485547 if int is 32 bit. +# +# Ruby supports leap seconds as far as if the C function "localtime" and +# "gmtime" supports it. +# They use the tz database in most Unix systems. +# The tz database has timezones which supports leap seconds. +# For example, "Asia/Tokyo" doesn't support leap seconds but +# "right/Asia/Tokyo" supports leap seconds. +# So, Ruby supports leap seconds if the TZ environment variable is +# set to "right/Asia/Tokyo" in most Unix systems. +# # == Examples # # All of these examples were done using the EST timezone which is GMT-5. diff --git a/tool/leaked-globals b/tool/leaked-globals index 4aa2aff996..05ce02658b 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -89,6 +89,7 @@ Pipe.new(NM + ARGV).each do |line| next if n.include?(".") next if !so and n.start_with?("___asan_") next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") case n when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ next diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index 3ba27f6d64..8730f0fb3a 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -11,7 +11,8 @@ module BundledGem "time", # net-ftp "singleton", # prime "ipaddr", # rinda - "forwardable" # prime, rinda + "forwardable", # prime, rinda + "strscan" # rexml ] module_function diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index d758b5fb02..2b0856b822 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -37,6 +37,26 @@ module Test class PendedError < AssertionFailedError; end + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + module Order class NoSort def initialize(seed) @@ -1778,15 +1798,7 @@ module Test end def location e # :nodoc: - last_before_assertion = "" - - return '<empty>' unless e&.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(?:Test::Unit::(?:Core)?Assertions#)?(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index b4f1dbc176..aad422f7e7 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -768,7 +768,14 @@ EOT e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/merger.rb b/tool/merger.rb index 0d9957074f..e45a3ac586 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,9 +2,8 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 2.0, Subversion and Git. -# As a Ruby committer, run this in an SVN repository -# to commit a change. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' require 'net/http' @@ -15,20 +14,11 @@ ENV['LC_ALL'] = 'C' ORIGIN = 'git@git.ruby-lang.org:ruby.git' GITHUB = 'git@github.com:ruby/ruby.git' -module Merger - REPOS = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' -end - -class << Merger - include Merger - +class << Merger = Object.new def help puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -37,16 +27,16 @@ class << Merger ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 \e[1mremove tag\e[0m - ruby #$0 removetag 2.2.9 + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m HELP @@ -69,11 +59,7 @@ class << Merger def version_up(teeny: false) now = Time.now now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 - if svn_mode? - system('svn', 'revert', 'version.h') - else - system('git', 'checkout', 'HEAD', 'version.h') - end + system('git', 'checkout', 'HEAD', 'version.h') v, pl = version if teeny @@ -129,31 +115,10 @@ class << Merger end tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" - if svn_mode? - if relname - branch_url = `svn info`[/URL: (.*)/, 1] - else - branch_url = "#{REPOS}branches/ruby_" - if v[0] < '2' || (v[0] == '2' && v[1] < '1') - abort 'patchlevel must be greater than 0 for patch release' if pl == '0' - branch_url << v.join('_') - else - abort 'teeny must be greater than 0 for patch release' if v[2] == '0' - branch_url << v.join('_').sub(/_\d+\z/, '') - end - end - tag_url = "#{REPOS}tags/#{tagname}" - system('svn', 'info', tag_url, out: IO::NULL, err: IO::NULL) - if $?.success? - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('svn', 'cp', '-m', "add tag #{tagname}", branch_url, tag_url, interactive: true) - else - unless execute('git', 'tag', tagname) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('git', 'push', ORIGIN, tagname, interactive: true) + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' end + execute('git', 'push', ORIGIN, tagname, interactive: true) end def remove_tag(relname) @@ -173,64 +138,44 @@ class << Merger tagname = relname end - if svn_mode? - tag_url = "#{REPOS}tags/#{tagname}" - execute('svn', 'rm', '-m', "remove tag #{tagname}", tag_url, interactive: true) - else - execute('git', 'tag', '-d', tagname) - execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) - execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) - end + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end def update_revision_h - if svn_mode? - execute('svn', 'up') - end execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') execute('rm', '-f', 'revision.tmp') end def stat - if svn_mode? - `svn stat` - else - `git status --short` - end + `git status --short` end def diff(file = nil) - if svn_mode? - command = %w[svn diff --diff-cmd=diff -x -upw] - else - command = %w[git diff --color HEAD] - end + command = %w[git diff --color HEAD] IO.popen(command + [file].compact, &:read) end def commit(file) - if svn_mode? - begin - execute('svn', 'ci', '-F', file) - execute('svn', 'update') # svn ci doesn't update revision info on working copy - ensure - execute('rm', '-f', 'subversion.commitlog') - end - else - current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip - execute('git', 'add', '.') && - execute('git', 'commit', '-F', file) - end + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) end - private - - def svn_mode? - return @svn_mode if defined?(@svn_mode) - @svn_mode = system("svn info", %i(out err) => IO::NULL) + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? end + private + # Prints the version of Ruby found in version.h def version v = p = nil @@ -289,10 +234,11 @@ else case ARGV[0] when /--ticket=(.*)/ - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + tickets = $1.split(/,/) ARGV.shift else - tickets = '' + tickets = [] + detect_ticket = true end revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') @@ -303,8 +249,6 @@ else revs.each do |rev| git_rev = nil case rev - when /\A\d{1,6}\z/ - svn_rev = rev when /\A\h{7,40}\z/ git_rev = rev when nil then @@ -315,30 +259,24 @@ else exit end - # Merge revision from Git patch or SVN - if git_rev - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" - resp = Net::HTTP.get_response(URI(git_uri)) - if resp.code != '200' - abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" - end - patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') + # Merge revision from Git patch + git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" - message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") - message = "\n\n#{message}" + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) + end - puts '+ git apply' - IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } - else - default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') - svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" - message = IO.popen(['svn', 'log', '-c', svn_rev, svn_src], &:read) + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" - cmd = ['svn', 'merge', '--accept=postpone', '-c', svn_rev, svn_src] - puts "+ #{cmd.join(' ')}" - system(*cmd) - end + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end @@ -349,20 +287,22 @@ else Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s", revstr, tickets + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join f.write commit_message f.flush f.close - Merger.interactive('conflicts resolved?', f.path) do - IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| - g << Merger.stat - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << Merger.diff + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 63f4beb943..db01e0b79d 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -1205,7 +1205,6 @@ installs = $install.map do |inst| end installs.flatten! installs -= $exclude.map {|exc| $install_procs[exc]}.flatten -puts "Installing to #$destdir" unless installs.empty? installs.each do |block| dir = Dir.pwd begin @@ -1214,5 +1213,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index 44b44920d4..843132ab3a 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -10,13 +10,7 @@ require 'optparse' require 'abbrev' require 'pp' require 'shellwords' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'reline' opts = OptionParser.new target_version = nil @@ -24,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -35,11 +28,11 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { @@ -70,28 +63,28 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" + seq = seq + self + "\e[0m" end end end @@ -162,84 +155,6 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - _, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while true - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` -end - def find_git_log(pattern) `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end @@ -276,10 +191,12 @@ def backport_command_string # check if the Git revision is included in master has_commit(c, "master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -294,7 +211,7 @@ end console = IO.console row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { @@ -306,10 +223,11 @@ commands = { @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, @@ -376,9 +294,6 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API case args - when /\Ar?(\d+)\z/ # SVN - rev = $1 - uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/trunk/revisions/#{rev}/issues.json") when /\A\h{7,40}\z/ # Git rev = args uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") @@ -436,23 +351,17 @@ eom next end - if rev - elsif system("svn info #{RUBY_REPO_PATH&.shellescape}", %i(out err) => IO::NULL) # SVN - if (log = find_svn_log("##@issue]")) && (/revision="(?<rev>\d+)/ =~ log) - rev = "r#{rev}" - end - else # Git - if log = find_git_log("##@issue]") - /^commit (?<rev>\h{40})$/ =~ log - end + if rev.nil? && log = find_git_log("##@issue]") + /^commit (?<rev>\h{40})$/ =~ log end if log && rev str = log[/merge revision\(s\) ([^:]+)(?=:)/] if str - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev} #{str}." + str.sub!(/\Amerge/, 'merged') + str.gsub!(/\h{40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev} #{str}." else - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev}." + str = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." end if notes str << "\n" @@ -571,7 +480,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end diff --git a/tool/release.sh b/tool/release.sh index 0988dc2a67..6b8cf3ea50 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,7 +1,10 @@ #!/bin/bash # Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' ver=$1 if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then diff --git a/tool/rjit/bindgen.rb b/tool/rjit/bindgen.rb index fb6653ed9c..2fc4576216 100755 --- a/tool/rjit/bindgen.rb +++ b/tool/rjit/bindgen.rb @@ -540,6 +540,7 @@ generator = BindingGenerator.new( rb_vm_opt_newarray_min rb_vm_opt_newarray_max rb_vm_opt_newarray_hash + rb_vm_opt_newarray_pack rb_vm_setinstancevariable rb_vm_splat_array rjit_full_cfunc_return diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 0307028eb7..b624282314 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -273,6 +273,7 @@ module SyncDefaultGems cp_r("#{upstream}/ext/strscan", "ext") cp_r("#{upstream}/test/strscan", "test") cp_r("#{upstream}/strscan.gemspec", "ext/strscan") + cp_r("#{upstream}/doc/strscan", "doc") rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) `git checkout ext/strscan/depend` when "cgi" diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 709b495572..1e19c102b8 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -50,4 +50,17 @@ class TestAssertion < Test::Unit::TestCase assert_pattern_list(pattern_list, actual, message) end end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end end diff --git a/universal_parser.c b/universal_parser.c index 05445587ba..d2105a9465 100644 --- a/universal_parser.c +++ b/universal_parser.c @@ -59,7 +59,7 @@ #undef st_lookup #define st_lookup rb_parser_st_lookup -#define rb_encoding void +#define rb_encoding const void #undef xmalloc #define xmalloc p->config->malloc @@ -131,8 +131,6 @@ #define is_ascii_string p->config->is_ascii_string #define rb_enc_str_new p->config->enc_str_new #define rb_str_vcatf p->config->str_vcatf -#undef StringValueCStr -#define StringValueCStr(v) p->config->string_value_cstr(&(v)) #define rb_sprintf p->config->rb_sprintf #undef RSTRING_PTR #define RSTRING_PTR p->config->rstring_ptr @@ -168,7 +166,6 @@ #define rb_ascii8bit_encoding p->config->ascii8bit_encoding #define rb_enc_codelen p->config->enc_codelen #define rb_enc_mbcput p->config->enc_mbcput -#define rb_enc_mbclen p->config->enc_mbclen #define rb_enc_find_index p->config->enc_find_index #define rb_enc_from_index p->config->enc_from_index #define rb_enc_isspace p->config->enc_isspace @@ -31,6 +31,7 @@ #include "internal.h" #include "internal/sanitizers.h" +#include "internal/imemo.h" #include "internal/util.h" #include "ruby/util.h" #include "ruby_atomic.h" @@ -543,41 +544,63 @@ ruby_strdup(const char *str) return tmp; } +#if defined HAVE_GETCWD +# if defined NO_GETCWD_MALLOC + char * ruby_getcwd(void) { -#if defined HAVE_GETCWD -# undef RUBY_UNTYPED_DATA_WARNING -# define RUBY_UNTYPED_DATA_WARNING 0 -# if defined NO_GETCWD_MALLOC - VALUE guard = Data_Wrap_Struct((VALUE)0, NULL, RUBY_DEFAULT_FREE, NULL); + VALUE guard = rb_imemo_tmpbuf_auto_free_pointer(); int size = 200; char *buf = xmalloc(size); while (!getcwd(buf, size)) { int e = errno; if (e != ERANGE) { - xfree(buf); - DATA_PTR(guard) = NULL; + rb_free_tmp_buffer(&guard); rb_syserr_fail(e, "getcwd"); } size *= 2; - DATA_PTR(guard) = buf; + rb_imemo_tmpbuf_set_ptr(guard, buf); buf = xrealloc(buf, size); } + rb_free_tmp_buffer(&guard); + return buf; +} + # else - VALUE guard = Data_Wrap_Struct((VALUE)0, NULL, free, NULL); + +static const rb_data_type_t getcwd_buffer_guard_type = { + .wrap_struct_name = "ruby_getcwd_guard", + .function = { + .dfree = free // not xfree. + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED +}; + +char * +ruby_getcwd(void) +{ + VALUE guard = TypedData_Wrap_Struct((VALUE)0, &getcwd_buffer_guard_type, NULL); char *buf, *cwd = getcwd(NULL, 0); - DATA_PTR(guard) = cwd; + RTYPEDDATA_DATA(guard) = cwd; if (!cwd) rb_sys_fail("getcwd"); buf = ruby_strdup(cwd); /* allocate by xmalloc */ free(cwd); + RTYPEDDATA_DATA(RB_GC_GUARD(guard)) = NULL; + return buf; +} + # endif - DATA_PTR(RB_GC_GUARD(guard)) = NULL; #else + # ifndef PATH_MAX # define PATH_MAX 8192 # endif + +char * +ruby_getcwd(void) +{ char *buf = xmalloc(PATH_MAX+1); if (!getwd(buf)) { @@ -585,10 +608,11 @@ ruby_getcwd(void) xfree(buf); rb_syserr_fail(e, "getwd"); } -#endif return buf; } +#endif + void ruby_each_words(const char *str, void (*func)(const char*, int, void*), void *arg) { diff --git a/variable.c b/variable.c index fcde8a603e..657cbdc889 100644 --- a/variable.c +++ b/variable.c @@ -1761,7 +1761,7 @@ rb_obj_ivar_set(VALUE obj, ID id, VALUE val) VALUE rb_vm_set_ivar_id(VALUE obj, ID id, VALUE val) { - rb_check_frozen_internal(obj); + rb_check_frozen(obj); rb_obj_ivar_set(obj, id, val); return val; } diff --git a/vcpkg.json b/vcpkg.json index 6d2ee3a6bc..d8b9c68807 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -7,5 +7,5 @@ "openssl", "zlib" ], - "builtin-baseline": "53bef8994c541b6561884a8395ea35715ece75db" + "builtin-baseline": "a4275b7eee79fb24ec2e135481ef5fce8b41c339" } @@ -2278,6 +2278,7 @@ vm_redefinition_bop_for_id(ID mid) OP(NilP, NIL_P); OP(Cmp, CMP); OP(Default, DEFAULT); + OP(Pack, PACK); #undef OP } return -1; @@ -684,15 +684,20 @@ setup_parameters_complex(rb_execution_context_t * const ec, const rb_iseq_t * co if (RB_TYPE_P(rest_last, T_HASH) && FL_TEST_RAW(rest_last, RHASH_PASS_AS_KEYWORDS)) { // def f(**kw); a = [..., kw]; g(*a) splat_flagged_keyword_hash = rest_last; - rest_last = rb_hash_dup(rest_last); + if (!RHASH_EMPTY_P(rest_last) || (ISEQ_BODY(iseq)->param.flags.has_kwrest)) { + rest_last = rb_hash_dup(rest_last); + } kw_flag |= VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT; // Unset rest_dupped set by anon_rest as we may need to modify splat in this case args->rest_dupped = false; if (ignore_keyword_hash_p(rest_last, iseq, &kw_flag, &converted_keyword_hash)) { - arg_rest_dup(args); - rb_ary_pop(args->rest); + if (ISEQ_BODY(iseq)->param.flags.has_rest || arg_setup_type != arg_setup_method) { + // Only duplicate/modify splat array if it will be used + arg_rest_dup(args); + rb_ary_pop(args->rest); + } given_argc--; kw_flag &= ~(VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT); } @@ -1629,6 +1629,10 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, const rb_iseq_t *const parent = vm_block_iseq(base_block); const rb_iseq_t *iseq = parent; VALUE name = rb_fstring_lit("<compiled>"); + + // Conditionally enable coverage depending on the current mode: + int coverage_enabled = ((rb_get_coverage_mode() & COVERAGE_TARGET_EVAL) != 0) ? 1 : 0; + if (!fname) { fname = rb_source_location(&line); } @@ -1638,10 +1642,12 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, } else { fname = get_eval_default_path(); + coverage_enabled = 0; } pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, line); + result.node.coverage_enabled = coverage_enabled; // Cout scopes, one for each parent iseq, plus one for our local scope int scopes_count = 0; @@ -1703,6 +1709,7 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, RUBY_ASSERT(parent_scope != NULL); pm_options_scope_t *options_scope = &result.options.scopes[scopes_count - scopes_index - 1]; + parent_scope->coverage_enabled = coverage_enabled; parent_scope->parser = &result.parser; parent_scope->index_lookup_table = st_init_numtable(); diff --git a/vm_insnhelper.c b/vm_insnhelper.c index a1893b1ba2..46737145ca 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1419,7 +1419,7 @@ vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, RB_DEBUG_COUNTER_INC(ivar_set_ic_miss); if (BUILTIN_TYPE(obj) == T_OBJECT) { - rb_check_frozen_internal(obj); + rb_check_frozen(obj); attr_index_t index = rb_obj_ivar_set(obj, id, val); @@ -2209,7 +2209,7 @@ vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass) } #if USE_DEBUG_COUNTER - if (old_cc == empty_cc) { + if (!old_cc || old_cc == empty_cc) { // empty RB_DEBUG_COUNTER_INC(mc_inline_miss_empty); } @@ -3008,7 +3008,7 @@ warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, if (0) { fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE); - fprintf(stderr, "pc:%p def:%p\n", pc, cme->def); + fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def); fprintf(stderr, "key:%p\n", (void *)key); } @@ -3767,7 +3767,7 @@ vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, cons attr_index_t index = vm_cc_attr_index(cc); shape_id_t dest_shape_id = vm_cc_attr_index_dest_shape_id(cc); ID id = vm_cc_cme(cc)->def->body.attr.id; - rb_check_frozen_internal(obj); + rb_check_frozen(obj); VALUE res = vm_setivar(obj, id, val, dest_shape_id, index); if (UNDEF_P(res)) { switch (BUILTIN_TYPE(obj)) { @@ -5932,6 +5932,22 @@ rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t num, const VALUE *p return vm_opt_newarray_hash(ec, num, ptr); } +VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len, bool freeze); +VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer); + +VALUE +rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr, VALUE fmt) +{ + if (BASIC_OP_UNREDEFINED_P(BOP_PACK, ARRAY_REDEFINED_OP_FLAG)) { + struct RArray fake_ary; + VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, num, true); + return rb_ec_pack_ary(ec, ary, fmt, Qnil); + } + else { + return rb_vm_call_with_refinements(ec, rb_ary_new4(num, ptr), idPack, 1, &fmt, RB_PASS_CALLED_KEYWORDS); + } +} + #undef id_cmp #define IMEMO_CONST_CACHE_SHAREABLE IMEMO_FL_USER0 diff --git a/vm_method.c b/vm_method.c index 3cacc010b8..4cf03fafde 100644 --- a/vm_method.c +++ b/vm_method.c @@ -1250,6 +1250,7 @@ method_entry_set(VALUE klass, ID mid, const rb_method_entry_t *me, me->def->type, me->def, 0, NULL); if (newme == me) { me->def->no_redef_warning = TRUE; + METHOD_ENTRY_FLAGS_SET(newme, visi, FALSE); } method_added(klass, mid); diff --git a/win32/Makefile.sub b/win32/Makefile.sub index 7dbb020eee..c73a8a4ba5 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -160,6 +160,9 @@ OPTFLAGS = -O2b2xg- OPTFLAGS = -O2sy- !endif !endif +!if !defined(incflags) +incflags = +!endif !if !defined(PLATFORM) PLATFORM = mswin32 !endif @@ -294,9 +297,7 @@ CXXFLAGS = $(CFLAGS) !if !defined(LDFLAGS) LDFLAGS = -incremental:no -debug -opt:ref -opt:icf !endif -!if !defined(XLDFLAGS) -XLDFLAGS = -stack:$(STACK) -!endif +XLDFLAGS = -stack:$(STACK) $(XLDFLAGS) !if !defined(RFLAGS) RFLAGS = -r !endif @@ -745,6 +746,7 @@ $(CONFIG_H): $(MKFILES) $(srcdir)/win32/Makefile.sub $(win_srcdir)/Makefile.sub #define RBIMPL_ATTR_PACKED_STRUCT_END() __pragma(pack(pop)) !endif #define RUBY_EXTERN extern __declspec(dllimport) +#define RUBY_FUNC_EXPORTED extern __declspec(dllexport) #define RUBY_ALIGNAS(n) __declspec(align(n)) #define RUBY_ALIGNOF __alignof #define HAVE_DECL_SYS_NERR 1 @@ -970,6 +972,7 @@ s,@WERRORFLAG@,$(WERRORFLAG),;t t s,@DEFS@,$(DEFS),;t t s,@CPPFLAGS@,$(CPPFLAGS),;t t s,@CXXFLAGS@,$(CXXFLAGS),;t t +s,@incflags@,$(incflags),;t t s,@FFLAGS@,$(FFLAGS),;t t s,@LDFLAGS@,$(LDFLAGS),;t t s,@LIBS@,user32.lib,;t t @@ -1143,7 +1146,8 @@ miniruby: miniruby$(EXEEXT) miniruby$(EXEEXT): @echo $(LIBS) $(ECHO) linking $(@:\=/) - $(Q) $(PURIFY) $(CC) $(MAINOBJ) $(MINIOBJS) $(COMMONOBJS) $(LIBS) -Fe$@ -link $(LDFLAGS) + $(Q) $(PURIFY) $(CC) $(MAINOBJ) $(MINIOBJS) $(COMMONOBJS) $(LIBS) \ + $(OUTFLAG)$@ -link $(LDFLAGS) $(XLDFLAGS) @$(RM) miniruby.lib miniruby.exp $(Q) miniruby.exe -v $(Q) $(LDSHARED_1) @@ -1213,7 +1217,8 @@ $(LIBRUBY_SO): $(LIBRUBY_A) $(DLDOBJS) $(RUBYDEF) $(RUBY_SO_NAME).res !endif $(ECHO) linking shared-library $(@:\=/) $(Q) $(LDSHARED) $(DLDOBJS) $(LIBRUBY_A) \ - $(RUBY_SO_NAME).res $(SOLIBS) $(EXTSOLIBS) $(LIBS) -Fe$@ -link $(LDFLAGS) \ + $(RUBY_SO_NAME).res $(SOLIBS) $(EXTSOLIBS) $(LIBS) \ + $(OUTFLAG)$@ -link $(LDFLAGS) $(XLDFLAGS) \ $(LIBRUBY_DLDFLAGS) @$(RM) dummy.lib dummy.exp !if defined(LDSHARED_0) @@ -1426,7 +1431,7 @@ rubyspec-capiext: $(RUBYSPEC_CAPIEXT_EXTS) $(Q)$(MAKEDIRS) $(@D) $(Q)echo> $*.def EXPORTS $(Q)echo>> $*.def Init_$(*F) - $(Q)$(LDSHARED) -Fe$(@) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) $< $(LIBRUBYARG) -link $(DLDFLAGS) $(LIBS) $(LOCAL_LIBS) -implib:$*.lib -pdb:$*.pdb -def:$*.def + $(Q)$(LDSHARED) -Fe$(@) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) $< $(LIBRUBYARG) -link $(DLDFLAGS) $(XLDFLAGS) $(LIBS) $(LOCAL_LIBS) -implib:$*.lib -pdb:$*.pdb -def:$*.def !if defined(LDSHARED_0) $(Q)$(LDSHARED_0) $(Q)$(LDSHARED_1) @@ -1434,3 +1439,8 @@ rubyspec-capiext: $(RUBYSPEC_CAPIEXT_EXTS) !endif exts: rubyspec-capiext + +yesterday: + for /f "usebackq" %H in \ + (`$(GIT) -C $(srcdir) log -1 "--before=00:00+0900" "--format=%H"`) do \ + $(GIT) -C $(srcdir) reset --hard %%H diff --git a/win32/configure.bat b/win32/configure.bat index 7253ade28b..dd1a917adc 100755 --- a/win32/configure.bat +++ b/win32/configure.bat @@ -7,6 +7,9 @@ for %%I in (%0) do if /%%~dpI/ == /%CD%\/ ( exit /b 999
)
+set XINCFLAGS=
+set XLDFLAGS=
+
echo> ~tmp~.mak ####
echo>> ~tmp~.mak conf = %0
echo>> ~tmp~.mak $(conf): nul
@@ -48,7 +51,9 @@ if "%1" == "--with-git" goto :git if "%1" == "--without-git" goto :nogit
if "%1" == "--without-ext" goto :witharg
if "%1" == "--without-extensions" goto :witharg
+if "%1" == "--with-opt-dir" goto :opt-dir
if "%1" == "--with-gmp" goto :gmp
+if "%1" == "--with-gmp-dir" goto :gmp-dir
if "%opt:~0,10%" == "--without-" goto :withoutarg
if "%opt:~0,7%" == "--with-" goto :witharg
if "%1" == "-h" goto :help
@@ -217,6 +222,10 @@ goto :loop ; shift
shift
goto :loop ;
+:gmp-dir
+:opt-dir
+ set XINCFLAGS=%XINCFLAGS% -I%2/include
+ set XLDFLAGS=%XLDFLAGS% -libpath:%2/lib
:witharg
echo>>confargs.tmp %1=%2\
set witharg=1
@@ -263,6 +272,8 @@ cl -EP confargs.c > ~setup~.mak 2>nul if exist pathlist.tmp echo>>~setup~.mak PATH = $(pathlist:;=/bin;)$(PATH)
if exist pathlist.tmp echo>>~setup~.mak INCLUDE = $(pathlist:;=/include;)
if exist pathlist.tmp echo>>~setup~.mak LIB = $(pathlist:;=/lib;)
+echo>>~setup~.mak XINCFLAGS = %XINCFLAGS%
+echo>>~setup~.mak XLDFLAGS = %XLDFLAGS%
type>>~setup~.mak ~tmp~.mak
del *.tmp > nul
del ~tmp~.mak > nul
diff --git a/win32/setup.mak b/win32/setup.mak index 8c27994821..3b822a33c0 100644 --- a/win32/setup.mak +++ b/win32/setup.mak @@ -62,6 +62,12 @@ ENABLE_DEBUG_ENV = $(ENABLE_DEBUG_ENV) !if defined(RJIT_SUPPORT) RJIT_SUPPORT = $(RJIT_SUPPORT) !endif +!if defined(XINCFLAGS) +CPPFLAGS = $(XINCFLAGS) +!endif +!if defined(XLDFLAGS) +XLDFLAGS = $(XLDFLAGS) +!endif # TOOLS << @@ -79,8 +85,13 @@ RJIT_SUPPORT = $(RJIT_SUPPORT) @echo HAVE_GIT = $(HAVE_GIT)>> $(MAKEFILE) !endif -!if "$(WITH_GMP)" == "yes" - @echo>>$(MAKEFILE) USE_GMP = 1 +!if "$(WITH_GMP)" != "no" + @($(CC) $(XINCFLAGS) <<conftest.c -link $(XLDFLAGS) gmp.lib > nul && (echo USE_GMP = yes) || exit /b 0) >>$(MAKEFILE) +#include <gmp.h> +mpz_t x; +int main(void) {mpz_init(x); return 0;} +<< + @$(WIN32DIR:/=\)\rm.bat conftest.* !endif -osname-section-: diff --git a/win32/win32.c b/win32/win32.c index c51d53595f..0d9cc087dc 100644 --- a/win32/win32.c +++ b/win32/win32.c @@ -1950,7 +1950,7 @@ w32_cmdvector(const WCHAR *cmd, char ***vec, UINT cp, rb_encoding *enc) } } - curr = (NtCmdLineElement *)calloc(sizeof(NtCmdLineElement), 1); + curr = (NtCmdLineElement *)calloc(1, sizeof(NtCmdLineElement)); if (!curr) goto do_nothing; curr->str = rb_w32_wstr_to_mbstr(cp, base, len, &curr->len); curr->flags |= NTMALLOC; @@ -2156,7 +2156,7 @@ w32_wopendir(const WCHAR *wpath) // // Get us a DIR structure // - p = calloc(sizeof(DIR), 1); + p = calloc(1, sizeof(DIR)); if (p == NULL) return NULL; @@ -1245,7 +1245,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_print_stats_p(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self); -VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self, VALUE context); +VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_disasm_iseq(rb_execution_context_t *ec, VALUE self, VALUE iseq); VALUE rb_yjit_insns_compiled(rb_execution_context_t *ec, VALUE self, VALUE iseq); @@ -155,8 +155,8 @@ module RubyVM::YJIT # Return a hash for statistics generated for the `--yjit-stats` command line option. # Return `nil` when option is not passed or unavailable. - def self.runtime_stats(context: false) - stats = Primitive.rb_yjit_get_stats(context) + def self.runtime_stats() + stats = Primitive.rb_yjit_get_stats() return stats if stats.nil? stats[:object_shape_count] = Primitive.object_shape_count @@ -313,7 +313,7 @@ module RubyVM::YJIT # Format and print out counters def _print_stats(out: $stderr) # :nodoc: - stats = runtime_stats(context: true) + stats = runtime_stats() return unless Primitive.rb_yjit_stats_enabled_p out.puts("***YJIT: Printing YJIT statistics on exit***") @@ -388,8 +388,12 @@ module RubyVM::YJIT out.puts "freed_code_size: " + format_number(13, stats[:freed_code_size]) out.puts "yjit_alloc_size: " + format_number(13, stats[:yjit_alloc_size]) if stats.key?(:yjit_alloc_size) - out.puts "live_context_size: " + format_number(13, stats[:live_context_size]) - out.puts "live_context_count: " + format_number(13, stats[:live_context_count]) + + bytes_per_context = stats[:context_data_bytes].fdiv(stats[:num_contexts_encoded]) + out.puts "context_data_bytes: " + format_number(13, stats[:context_data_bytes]) + out.puts "num_contexts_encoded: " + format_number(13, stats[:num_contexts_encoded]) + out.puts "bytes_per_context: " + ("%13.2f" % bytes_per_context) + out.puts "live_page_count: " + format_number(13, stats[:live_page_count]) out.puts "freed_page_count: " + format_number(13, stats[:freed_page_count]) out.puts "code_gc_count: " + format_number(13, stats[:code_gc_count]) diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index a7473c1bf6..e76d9b3063 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -100,7 +100,7 @@ fn main() { .allowlist_function("rb_shape_get_shape_by_id") .allowlist_function("rb_shape_id_offset") .allowlist_function("rb_shape_get_iv_index") - .allowlist_function("rb_shape_get_next") + .allowlist_function("rb_shape_get_next_no_warnings") .allowlist_function("rb_shape_id") .allowlist_function("rb_shape_obj_too_complex") .allowlist_var("SHAPE_ID_NUM_BITS") @@ -223,6 +223,7 @@ fn main() { .allowlist_function("rb_float_div") // From internal/string.h + .allowlist_type("ruby_rstring_private_flags") .allowlist_function("rb_ec_str_resurrect") .allowlist_function("rb_str_concat_literals") .allowlist_function("rb_obj_as_string_result") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 072d96f1b0..755e64c244 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -30,7 +30,6 @@ pub use crate::virtualmem::CodePtr; /// Status returned by code generation functions #[derive(PartialEq, Debug)] enum CodegenStatus { - SkipNextInsn, KeepCompiling, EndBlock, } @@ -197,6 +196,13 @@ impl JITState { self.insn_idx + insn_len(self.get_opcode()) as u16 } + /// Get the index of the next instruction of the next instruction + fn next_next_insn_idx(&self) -> u16 { + let next_pc = unsafe { rb_iseq_pc_at_idx(self.iseq, self.next_insn_idx().into()) }; + let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(self.iseq, next_pc) }.try_into().unwrap(); + self.next_insn_idx() + insn_len(next_opcode) as u16 + } + // Check if we are compiling the instruction at the stub PC // Meaning we are compiling the instruction that is next to execute pub fn at_current_insn(&self) -> bool { @@ -1098,7 +1104,16 @@ fn jump_to_next_insn( jit: &mut JITState, asm: &mut Assembler, ocb: &mut OutlinedCb, -) -> Option<()> { +) -> Option<CodegenStatus> { + end_block_with_jump(jit, asm, ocb, jit.next_insn_idx()) +} + +fn end_block_with_jump( + jit: &mut JITState, + asm: &mut Assembler, + ocb: &mut OutlinedCb, + continuation_insn_idx: u16, +) -> Option<CodegenStatus> { // Reset the depth since in current usages we only ever jump to // chain_depth > 0 from the same instruction. let mut reset_depth = asm.ctx; @@ -1106,20 +1121,20 @@ fn jump_to_next_insn( let jump_block = BlockId { iseq: jit.iseq, - idx: jit.next_insn_idx(), + idx: continuation_insn_idx, }; // We are at the end of the current instruction. Record the boundary. if jit.record_boundary_patch_point { jit.record_boundary_patch_point = false; - let exit_pc = unsafe { jit.pc.offset(insn_len(jit.opcode).try_into().unwrap()) }; + let exit_pc = unsafe { rb_iseq_pc_at_idx(jit.iseq, continuation_insn_idx.into())}; let exit_pos = gen_outlined_exit(exit_pc, &reset_depth, ocb); record_global_inval_patch(asm, exit_pos?); } // Generate the jump instruction gen_direct_jump(jit, &reset_depth, jump_block, asm); - Some(()) + Some(EndBlock) } // Compile a sequence of bytecode instructions for a given basic block version. @@ -1283,13 +1298,6 @@ pub fn gen_single_block( // Move to the next instruction to compile insn_idx += insn_len(opcode) as u16; - // Move past next instruction when instructed - if status == Some(SkipNextInsn) { - let next_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx.into()) }; - let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(iseq, next_pc) }.try_into().unwrap(); - insn_idx += insn_len(next_opcode) as u16; - } - // If the instruction terminates this block if status == Some(EndBlock) { break; @@ -1519,7 +1527,7 @@ fn fuse_putobject_opt_ltlt( asm.stack_pop(1); fixnum_left_shift_body(asm, lhs, shift_amt as u64); - return Some(SkipNextInsn); + return end_block_with_jump(jit, asm, ocb, jit.next_next_insn_idx()); } return None; } @@ -2962,7 +2970,7 @@ fn gen_set_ivar( // The current shape doesn't contain this iv, we need to transition to another shape. let new_shape = if !shape_too_complex && receiver_t_object && ivar_index.is_none() { let current_shape = comptime_receiver.shape_of(); - let next_shape = unsafe { rb_shape_get_next(current_shape, comptime_receiver, ivar_name) }; + let next_shape = unsafe { rb_shape_get_next_no_warnings(current_shape, comptime_receiver, ivar_name) }; let next_shape_id = unsafe { rb_shape_id(next_shape) }; // If the VM ran out of shapes, or this class generated too many leaf, @@ -4235,11 +4243,50 @@ fn gen_opt_newarray_send( gen_opt_newarray_max(jit, asm, _ocb) } else if method == ID!(hash) { gen_opt_newarray_hash(jit, asm, _ocb) + } else if method == ID!(pack) { + gen_opt_newarray_pack(jit, asm, _ocb) } else { None } } +fn gen_opt_newarray_pack( + jit: &mut JITState, + asm: &mut Assembler, + _ocb: &mut OutlinedCb, +) -> Option<CodegenStatus> { + // num == 4 ( for this code ) + let num = jit.get_arg(0).as_u32(); + + // Save the PC and SP because we may call #pack + jit_prepare_non_leaf_call(jit, asm); + + extern "C" { + fn rb_vm_opt_newarray_pack(ec: EcPtr, num: u32, elts: *const VALUE, fmt: VALUE) -> VALUE; + } + + let values_opnd = asm.ctx.sp_opnd(-(num as i32)); + let values_ptr = asm.lea(values_opnd); + + let fmt_string = asm.ctx.sp_opnd(-1); + + let val_opnd = asm.ccall( + rb_vm_opt_newarray_pack as *const u8, + vec![ + EC, + (num - 1).into(), + values_ptr, + fmt_string + ], + ); + + asm.stack_pop(num.as_usize()); + let stack_ret = asm.stack_push(Type::CString); + asm.mov(stack_ret, val_opnd); + + Some(KeepCompiling) +} + fn gen_opt_newarray_hash( jit: &mut JITState, asm: &mut Assembler, @@ -5572,7 +5619,7 @@ fn jit_rb_str_uplus( let recv_opnd = asm.stack_pop(1); let recv_opnd = asm.load(recv_opnd); let flags_opnd = asm.load(Opnd::mem(64, recv_opnd, RUBY_OFFSET_RBASIC_FLAGS)); - asm.test(flags_opnd, Opnd::Imm(RUBY_FL_FREEZE as i64)); + asm.test(flags_opnd, Opnd::Imm(RUBY_FL_FREEZE as i64 | RSTRING_CHILLED as i64)); let ret_label = asm.new_label("stack_ret"); @@ -5742,7 +5789,7 @@ fn jit_rb_str_getbyte( RUBY_OFFSET_RSTRING_LEN as i32, ); - // Exit if the indes is out of bounds + // Exit if the index is out of bounds asm.cmp(idx, str_len_opnd); asm.jge(Target::side_exit(Counter::getbyte_idx_out_of_bounds)); @@ -6914,20 +6961,20 @@ fn push_splat_args(required_args: u32, asm: &mut Assembler) { asm.cmp(array_len_opnd, required_args.into()); asm.jne(Target::side_exit(Counter::guard_send_splatarray_length_not_equal)); - asm_comment!(asm, "Check last argument is not ruby2keyword hash"); - - // Need to repeat this here to deal with register allocation - let array_reg = asm.load(asm.stack_opnd(0)); - - let ary_opnd = get_array_ptr(asm, array_reg); - - let last_array_value = asm.load(Opnd::mem(64, ary_opnd, (required_args as i32 - 1) * (SIZEOF_VALUE as i32))); + // Check last element of array if present + if required_args > 0 { + asm_comment!(asm, "Check last argument is not ruby2keyword hash"); - guard_object_is_not_ruby2_keyword_hash( - asm, - last_array_value, - Counter::guard_send_splatarray_last_ruby2_keywords, - ); + // Need to repeat this here to deal with register allocation + let array_reg = asm.load(asm.stack_opnd(0)); + let ary_opnd = get_array_ptr(asm, array_reg); + let last_array_value = asm.load(Opnd::mem(64, ary_opnd, (required_args as i32 - 1) * (SIZEOF_VALUE as i32))); + guard_object_is_not_ruby2_keyword_hash( + asm, + last_array_value, + Counter::guard_send_splatarray_last_ruby2_keywords, + ); + } asm_comment!(asm, "Push arguments from array"); let array_opnd = asm.stack_pop(1); @@ -10286,6 +10333,9 @@ fn yjit_reg_method(klass: VALUE, mid_str: &str, gen_fn: MethodGenFn) { /// Global state needed for code generation pub struct CodegenGlobals { + /// Flat vector of bits to store compressed context data + context_data: BitVector, + /// Inline code block (fast path) inline_cb: CodeBlock, @@ -10401,6 +10451,7 @@ impl CodegenGlobals { ocb.unwrap().mark_all_executable(); let codegen_globals = CodegenGlobals { + context_data: BitVector::new(), inline_cb: cb, outlined_cb: ocb, leave_exit_code, @@ -10429,6 +10480,11 @@ impl CodegenGlobals { unsafe { CODEGEN_GLOBALS.as_mut().is_some() } } + /// Get a mutable reference to the context data + pub fn get_context_data() -> &'static mut BitVector { + &mut CodegenGlobals::get_instance().context_data + } + /// Get a mutable reference to the inline code block pub fn get_inline_cb() -> &'static mut CodeBlock { &mut CodegenGlobals::get_instance().inline_cb diff --git a/yjit/src/core.rs b/yjit/src/core.rs index cd6e649aa0..0cfd7d9124 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -15,12 +15,14 @@ use crate::utils::*; use crate::disasm::*; use core::ffi::c_void; use std::cell::*; -use std::collections::HashSet; use std::fmt; use std::mem; use std::mem::transmute; use std::ops::Range; use std::rc::Rc; +use std::collections::HashSet; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use mem::MaybeUninit; use std::ptr; use ptr::NonNull; @@ -457,8 +459,13 @@ const CHAIN_DEPTH_MASK: u8 = 0b00111111; // 63 /// Contains information we can use to specialize/optimize code /// There are a lot of context objects so we try to keep the size small. #[derive(Copy, Clone, Default, Eq, Hash, PartialEq, Debug)] -#[repr(packed)] pub struct Context { + // FIXME: decoded_from breaks == on contexts + /* + // Offset at which this context was previously encoded (zero if not) + decoded_from: u32, + */ + // Number of values currently on the temporary stack stack_size: u8, @@ -498,6 +505,595 @@ pub struct Context { inline_block: u64, } +#[derive(Clone)] +pub struct BitVector { + // Flat vector of bytes to write into + bytes: Vec<u8>, + + // Number of bits taken out of bytes allocated + num_bits: usize, +} + +impl BitVector { + pub fn new() -> Self { + Self { + bytes: Vec::with_capacity(4096), + num_bits: 0, + } + } + + #[allow(unused)] + pub fn num_bits(&self) -> usize { + self.num_bits + } + + // Total number of bytes taken + #[allow(unused)] + pub fn num_bytes(&self) -> usize { + (self.num_bits / 8) + if (self.num_bits % 8) != 0 { 1 } else { 0 } + } + + // Write/append an unsigned integer value + fn push_uint(&mut self, mut val: u64, mut num_bits: usize) { + assert!(num_bits <= 64); + + // Mask out bits above the number of bits requested + let mut val_bits = val; + if num_bits < 64 { + val_bits &= (1 << num_bits) - 1; + assert!(val == val_bits); + } + + // Number of bits encoded in the last byte + let rem_bits = self.num_bits % 8; + + // Encode as many bits as we can in this last byte + if rem_bits != 0 { + let num_enc = std::cmp::min(num_bits, 8 - rem_bits); + let bit_mask = (1 << num_enc) - 1; + let frac_bits = (val & bit_mask) << rem_bits; + let frac_bits: u8 = frac_bits.try_into().unwrap(); + let last_byte_idx = self.bytes.len() - 1; + self.bytes[last_byte_idx] |= frac_bits; + + self.num_bits += num_enc; + num_bits -= num_enc; + val >>= num_enc; + } + + // While we have bits left to encode + while num_bits > 0 { + // Grow with a 1.2x growth factor instead of 2x + assert!(self.num_bits % 8 == 0); + let num_bytes = self.num_bits / 8; + if num_bytes == self.bytes.capacity() { + self.bytes.reserve_exact(self.bytes.len() / 5); + } + + let bits = val & 0xFF; + let bits: u8 = bits.try_into().unwrap(); + self.bytes.push(bits); + + let bits_to_encode = std::cmp::min(num_bits, 8); + self.num_bits += bits_to_encode; + num_bits -= bits_to_encode; + val >>= bits_to_encode; + } + } + + fn push_u8(&mut self, val: u8) { + self.push_uint(val as u64, 8); + } + + fn push_u4(&mut self, val: u8) { + assert!(val < 16); + self.push_uint(val as u64, 4); + } + + fn push_u3(&mut self, val: u8) { + assert!(val < 8); + self.push_uint(val as u64, 3); + } + + fn push_u2(&mut self, val: u8) { + assert!(val < 4); + self.push_uint(val as u64, 2); + } + + fn push_u1(&mut self, val: u8) { + assert!(val < 2); + self.push_uint(val as u64, 1); + } + + // Push a context encoding opcode + fn push_op(&mut self, op: CtxOp) { + self.push_u4(op as u8); + } + + // Read a uint value at a given bit index + // The bit index is incremented after the value is read + fn read_uint(&self, bit_idx: &mut usize, mut num_bits: usize) -> u64 { + let start_bit_idx = *bit_idx; + let mut cur_idx = *bit_idx; + + // Read the bits in the first byte + let bit_mod = cur_idx % 8; + let bits_in_byte = self.bytes[cur_idx / 8] >> bit_mod; + + let num_bits_in_byte = std::cmp::min(num_bits, 8 - bit_mod); + cur_idx += num_bits_in_byte; + num_bits -= num_bits_in_byte; + + let mut out_bits = (bits_in_byte as u64) & ((1 << num_bits_in_byte) - 1); + + // While we have bits left to read + while num_bits > 0 { + let num_bits_in_byte = std::cmp::min(num_bits, 8); + assert!(cur_idx % 8 == 0); + let byte = self.bytes[cur_idx / 8] as u64; + + let bits_in_byte = byte & ((1 << num_bits) - 1); + out_bits |= bits_in_byte << (cur_idx - start_bit_idx); + + // Move to the next byte/offset + cur_idx += num_bits_in_byte; + num_bits -= num_bits_in_byte; + } + + // Update the read index + *bit_idx = cur_idx; + + out_bits + } + + fn read_u8(&self, bit_idx: &mut usize) -> u8 { + self.read_uint(bit_idx, 8) as u8 + } + + fn read_u4(&self, bit_idx: &mut usize) -> u8 { + self.read_uint(bit_idx, 4) as u8 + } + + fn read_u3(&self, bit_idx: &mut usize) -> u8 { + self.read_uint(bit_idx, 3) as u8 + } + + fn read_u2(&self, bit_idx: &mut usize) -> u8 { + self.read_uint(bit_idx, 2) as u8 + } + + fn read_u1(&self, bit_idx: &mut usize) -> u8 { + self.read_uint(bit_idx, 1) as u8 + } + + fn read_op(&self, bit_idx: &mut usize) -> CtxOp { + unsafe { std::mem::transmute(self.read_u4(bit_idx)) } + } +} + +impl fmt::Debug for BitVector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // We print the higher bytes first + for (idx, byte) in self.bytes.iter().enumerate().rev() { + write!(f, "{:08b}", byte)?; + + // Insert a separator between each byte + if idx > 0 { + write!(f, "|")?; + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod bitvector_tests { + use super::*; + + #[test] + fn write_3() { + let mut arr = BitVector::new(); + arr.push_uint(3, 2); + assert!(arr.read_uint(&mut 0, 2) == 3); + } + + #[test] + fn write_11() { + let mut arr = BitVector::new(); + arr.push_uint(1, 1); + arr.push_uint(1, 1); + assert!(arr.read_uint(&mut 0, 2) == 3); + } + + #[test] + fn write_11_overlap() { + let mut arr = BitVector::new(); + arr.push_uint(0, 7); + arr.push_uint(3, 2); + arr.push_uint(1, 1); + + //dbg!(arr.read_uint(7, 2)); + assert!(arr.read_uint(&mut 7, 2) == 3); + } + + #[test] + fn write_ff_0() { + let mut arr = BitVector::new(); + arr.push_uint(0xFF, 8); + assert!(arr.read_uint(&mut 0, 8) == 0xFF); + } + + #[test] + fn write_ff_3() { + // Write 0xFF at bit index 3 + let mut arr = BitVector::new(); + arr.push_uint(0, 3); + arr.push_uint(0xFF, 8); + assert!(arr.read_uint(&mut 3, 8) == 0xFF); + } + + #[test] + fn write_ff_sandwich() { + // Write 0xFF sandwiched between zeros + let mut arr = BitVector::new(); + arr.push_uint(0, 3); + arr.push_u8(0xFF); + arr.push_uint(0, 3); + assert!(arr.read_uint(&mut 3, 8) == 0xFF); + } + + #[test] + fn write_read_u32_max() { + let mut arr = BitVector::new(); + arr.push_uint(0xFF_FF_FF_FF, 32); + assert!(arr.read_uint(&mut 0, 32) == 0xFF_FF_FF_FF); + } + + #[test] + fn write_read_u32_max_64b() { + let mut arr = BitVector::new(); + arr.push_uint(0xFF_FF_FF_FF, 64); + assert!(arr.read_uint(&mut 0, 64) == 0xFF_FF_FF_FF); + } + + #[test] + fn write_read_u64_max() { + let mut arr = BitVector::new(); + arr.push_uint(u64::MAX, 64); + assert!(arr.read_uint(&mut 0, 64) == u64::MAX); + } + + #[test] + fn encode_default() { + let mut bits = BitVector::new(); + let ctx = Context::default(); + let start_idx = ctx.encode_into(&mut bits); + assert!(start_idx == 0); + assert!(bits.num_bits() > 0); + assert!(bits.num_bytes() > 0); + + // Make sure that the round trip matches the input + let ctx2 = Context::decode_from(&bits, 0); + assert!(ctx2 == ctx); + } + + #[test] + fn encode_default_2x() { + let mut bits = BitVector::new(); + + let ctx0 = Context::default(); + let idx0 = ctx0.encode_into(&mut bits); + + let mut ctx1 = Context::default(); + ctx1.reg_temps = RegTemps(1); + let idx1 = ctx1.encode_into(&mut bits); + + // Make sure that we can encode two contexts successively + let ctx0_dec = Context::decode_from(&bits, idx0); + let ctx1_dec = Context::decode_from(&bits, idx1); + assert!(ctx0_dec == ctx0); + assert!(ctx1_dec == ctx1); + } + + #[test] + fn regress_reg_temps() { + let mut bits = BitVector::new(); + let mut ctx = Context::default(); + ctx.reg_temps = RegTemps(1); + ctx.encode_into(&mut bits); + + let b0 = bits.read_u1(&mut 0); + assert!(b0 == 1); + + // Make sure that the round trip matches the input + let ctx2 = Context::decode_from(&bits, 0); + assert!(ctx2 == ctx); + } +} + +// Context encoding opcodes (4 bits) +#[derive(Debug, Copy, Clone)] +#[repr(u8)] +enum CtxOp { + // Self type (4 bits) + SetSelfType = 0, + + // Local idx (3 bits), temp type (4 bits) + SetLocalType, + + // Map stack temp to self with known type + // Temp idx (3 bits), known type (4 bits) + SetTempType, + + // Map stack temp to a local variable + // Temp idx (3 bits), local idx (3 bits) + MapTempLocal, + + // Map a stack temp to self + // Temp idx (3 bits) + MapTempSelf, + + // Set inline block pointer (8 bytes) + SetInlineBlock, + + // End of encoding + EndOfCode, +} + +const CTX_CACHE_SIZE: usize = 512; + +// Cache of the last contexts encoded +// Empirically this saves a few percent of memory +// We can experiment with varying the size of this cache +static mut CTX_CACHE: Option<[(Context, u32); CTX_CACHE_SIZE]> = None; + +impl Context { + pub fn encode(&self) -> u32 { + incr_counter!(num_contexts_encoded); + + if *self == Context::default() { + return 0; + } + + if let Some(idx) = Self::cache_get(self) { + return idx; + } + + let context_data = CodegenGlobals::get_context_data(); + + // Offset 0 is reserved for the default context + if context_data.num_bits() == 0 { + context_data.push_u1(0); + } + + let idx = self.encode_into(context_data); + let idx: u32 = idx.try_into().unwrap(); + + Self::cache_set(self, idx); + + // In debug mode, check that the round-trip decoding always matches + debug_assert!(Self::decode(idx) == *self); + + idx + } + + pub fn decode(start_idx: u32) -> Context { + if start_idx == 0 { + return Context::default(); + }; + + let context_data = CodegenGlobals::get_context_data(); + let ctx = Self::decode_from(context_data, start_idx as usize); + + Self::cache_set(&ctx, start_idx); + + ctx + } + + // Lookup the context in a cache of recently encoded/decoded contexts + fn cache_get(ctx: &Context) -> Option<u32> + { + unsafe { + if CTX_CACHE == None { + return None; + } + + let cache = CTX_CACHE.as_mut().unwrap(); + + let mut hasher = DefaultHasher::new(); + ctx.hash(&mut hasher); + let ctx_hash = hasher.finish() as usize; + let cache_entry = &cache[ctx_hash % CTX_CACHE_SIZE]; + + if cache_entry.0 == *ctx { + return Some(cache_entry.1); + } + + return None; + } + } + + // Store an entry in a cache of recently encoded/decoded contexts + fn cache_set(ctx: &Context, idx: u32) + { + unsafe { + if CTX_CACHE == None { + CTX_CACHE = Some( [(Context::default(), 0); CTX_CACHE_SIZE] ); + } + + let mut hasher = DefaultHasher::new(); + ctx.hash(&mut hasher); + let ctx_hash = hasher.finish() as usize; + + let cache = CTX_CACHE.as_mut().unwrap(); + cache[ctx_hash % CTX_CACHE_SIZE] = (*ctx, idx); + } + } + + // Encode into a compressed context representation in a bit vector + fn encode_into(&self, bits: &mut BitVector) -> usize { + let start_idx = bits.num_bits(); + + // NOTE: this value is often zero or falls within + // a small range, so could be compressed + //println!("stack_size={}", self.stack_size); + //println!("sp_offset={}", self.sp_offset); + //println!("chain_depth_and_flags={}", self.chain_depth_and_flags); + + // Most of the time, the stack size is small and sp offset has the same value + if (self.stack_size as i64) == (self.sp_offset as i64) && self.stack_size < 4 { + // One single bit to signify a compact stack_size/sp_offset encoding + bits.push_u1(1); + bits.push_u2(self.stack_size); + } else { + // Full stack size encoding + bits.push_u1(0); + + // Number of values currently on the temporary stack + bits.push_u8(self.stack_size); + + // sp_offset: i8, + bits.push_u8(self.sp_offset as u8); + } + + // Bitmap of which stack temps are in a register + let RegTemps(reg_temps) = self.reg_temps; + bits.push_u8(reg_temps); + + // chain_depth_and_flags: u8, + bits.push_u8(self.chain_depth_and_flags); + + // Encode the self type if known + if self.self_type != Type::Unknown { + bits.push_op(CtxOp::SetSelfType); + bits.push_u4(self.self_type as u8); + } + + // Encode the local types if known + for local_idx in 0..MAX_LOCAL_TYPES { + let t = self.get_local_type(local_idx); + if t != Type::Unknown { + bits.push_op(CtxOp::SetLocalType); + bits.push_u3(local_idx as u8); + bits.push_u4(t as u8); + } + } + + // Encode stack temps + for stack_idx in 0..MAX_TEMP_TYPES { + let mapping = self.get_temp_mapping(stack_idx); + + match mapping.get_kind() { + MapToStack => { + let t = mapping.get_type(); + if t != Type::Unknown { + // Temp idx (3 bits), known type (4 bits) + bits.push_op(CtxOp::SetTempType); + bits.push_u3(stack_idx as u8); + bits.push_u4(t as u8); + } + } + + MapToLocal => { + // Temp idx (3 bits), local idx (3 bits) + let local_idx = mapping.get_local_idx(); + bits.push_op(CtxOp::MapTempLocal); + bits.push_u3(stack_idx as u8); + bits.push_u3(local_idx as u8); + } + + MapToSelf => { + // Temp idx (3 bits) + bits.push_op(CtxOp::MapTempSelf); + bits.push_u3(stack_idx as u8); + } + } + } + + // Inline block pointer + if self.inline_block != 0 { + bits.push_op(CtxOp::SetInlineBlock); + bits.push_uint(self.inline_block, 64); + } + + // TODO: should we add an op for end-of-encoding, + // or store num ops at the beginning? + bits.push_op(CtxOp::EndOfCode); + + start_idx + } + + // Decode a compressed context representation from a bit vector + fn decode_from(bits: &BitVector, start_idx: usize) -> Context { + let mut ctx = Context::default(); + + let mut idx = start_idx; + + // Small vs large stack size encoding + if bits.read_u1(&mut idx) == 1 { + ctx.stack_size = bits.read_u2(&mut idx); + ctx.sp_offset = ctx.stack_size as i8; + } else { + ctx.stack_size = bits.read_u8(&mut idx); + ctx.sp_offset = bits.read_u8(&mut idx) as i8; + } + + // Bitmap of which stack temps are in a register + ctx.reg_temps = RegTemps(bits.read_u8(&mut idx)); + + // chain_depth_and_flags: u8 + ctx.chain_depth_and_flags = bits.read_u8(&mut idx); + + loop { + //println!("reading op"); + let op = bits.read_op(&mut idx); + //println!("got op {:?}", op); + + match op { + CtxOp::SetSelfType => { + ctx.self_type = unsafe { transmute(bits.read_u4(&mut idx)) }; + } + + CtxOp::SetLocalType => { + let local_idx = bits.read_u3(&mut idx) as usize; + let t = unsafe { transmute(bits.read_u4(&mut idx)) }; + ctx.set_local_type(local_idx, t); + } + + // Map temp to stack (known type) + CtxOp::SetTempType => { + let temp_idx = bits.read_u3(&mut idx) as usize; + let t = unsafe { transmute(bits.read_u4(&mut idx)) }; + ctx.set_temp_mapping(temp_idx, TempMapping::map_to_stack(t)); + } + + // Map temp to local + CtxOp::MapTempLocal => { + let temp_idx = bits.read_u3(&mut idx) as usize; + let local_idx = bits.read_u3(&mut idx); + ctx.set_temp_mapping(temp_idx, TempMapping::map_to_local(local_idx)); + } + + // Map temp to self + CtxOp::MapTempSelf => { + let temp_idx = bits.read_u3(&mut idx) as usize; + ctx.set_temp_mapping(temp_idx, TempMapping::map_to_self()); + } + + // Inline block pointer + CtxOp::SetInlineBlock => { + ctx.inline_block = bits.read_uint(&mut idx, 64); + } + + CtxOp::EndOfCode => break, + } + } + + ctx + } +} + /// Tuple of (iseq, idx) used to identify basic blocks /// There are a lot of blockid objects so we try to keep the size small. #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -659,7 +1255,7 @@ impl BranchTarget { } } - fn get_ctx(&self) -> Context { + fn get_ctx(&self) -> u32 { match self { BranchTarget::Stub(stub) => stub.ctx, BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.ctx, @@ -686,7 +1282,7 @@ struct BranchStub { address: Option<CodePtr>, iseq: Cell<IseqPtr>, iseq_idx: IseqIdx, - ctx: Context, + ctx: u32, } /// Store info about an outgoing branch in a code segment @@ -808,6 +1404,9 @@ impl PendingBranch { return Some(block.start_addr); } + // Compress/encode the context + let ctx = Context::encode(ctx); + // The branch struct is uninitialized right now but as a stable address. // We make sure the stub runs after the branch is initialized. let branch_struct_addr = self.uninit_branch.as_ptr() as usize; @@ -819,7 +1418,7 @@ impl PendingBranch { address: Some(stub_addr), iseq: Cell::new(target.iseq), iseq_idx: target.idx, - ctx: *ctx, + ctx, }))))); } @@ -912,7 +1511,7 @@ pub struct Block { // Context at the start of the block // This should never be mutated - ctx: Context, + ctx: u32, // Positions where the generated code starts and ends start_addr: CodePtr, @@ -1000,7 +1599,7 @@ impl fmt::Debug for MutableBranchList { // SAFETY: the derived Clone for boxed slices does not mutate this Cell let branches = unsafe { self.0.ref_unchecked().clone() }; - formatter.debug_list().entries(branches.into_iter()).finish() + formatter.debug_list().entries(branches.iter()).finish() } } @@ -1085,15 +1684,6 @@ pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { unsafe { rb_yjit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; } -/// Iterate over all ISEQ payloads -pub fn for_each_iseq_payload<F: FnMut(&IseqPayload)>(mut callback: F) { - for_each_iseq(|iseq| { - if let Some(iseq_payload) = get_iseq_payload(iseq) { - callback(iseq_payload); - } - }); -} - /// Iterate over all on-stack ISEQs pub fn for_each_on_stack_iseq<F: FnMut(IseqPtr)>(mut callback: F) { unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { @@ -1425,13 +2015,17 @@ pub fn take_version_list(blockid: BlockId) -> VersionList { fn get_num_versions(blockid: BlockId, inlined: bool) -> usize { let insn_idx = blockid.idx.as_usize(); match get_iseq_payload(blockid.iseq) { + + // FIXME: this counting logic is going to be expensive. + // We should avoid it if possible + Some(payload) => { payload .version_map .get(insn_idx) .map(|versions| { versions.iter().filter(|&&version| - unsafe { version.as_ref() }.ctx.inline() == inlined + Context::decode(unsafe { version.as_ref() }.ctx).inline() == inlined ).count() }) .unwrap_or(0) @@ -1476,10 +2070,11 @@ fn find_block_version(blockid: BlockId, ctx: &Context) -> Option<BlockRef> { // For each version matching the blockid for blockref in versions.iter() { let block = unsafe { blockref.as_ref() }; + let block_ctx = Context::decode(block.ctx); // Note that we always prefer the first matching // version found because of inline-cache chains - match ctx.diff(&block.ctx) { + match ctx.diff(&block_ctx) { TypeDiff::Compatible(diff) if diff < best_diff => { best_version = Some(*blockref); best_diff = diff; @@ -1561,7 +2156,7 @@ unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) { let block = unsafe { blockref.as_ref() }; // Function entry blocks must have stack size 0 - assert!(!(block.iseq_range.start == 0 && block.ctx.stack_size > 0)); + debug_assert!(!(block.iseq_range.start == 0 && Context::decode(block.ctx).stack_size > 0)); let version_list = get_or_create_version_list(block.get_blockid()); @@ -1620,12 +2215,14 @@ impl JITState { incr_counter_by!(num_gc_obj_refs, gc_obj_offsets.len()); + let ctx = Context::encode(&self.get_starting_ctx()); + // Make the new block let block = MaybeUninit::new(Block { start_addr, iseq: Cell::new(self.get_iseq()), iseq_range: self.get_starting_insn_idx()..end_insn_idx, - ctx: self.get_starting_ctx(), + ctx, end_addr: Cell::new(end_addr), incoming: MutableBranchList(Cell::default()), gc_obj_offsets: gc_obj_offsets.into_boxed_slice(), @@ -2382,6 +2979,7 @@ fn gen_block_series_body( }; // Generate new block using context from the last branch. + let requested_ctx = Context::decode(requested_ctx); let result = gen_single_block(requested_blockid, &requested_ctx, ec, cb, ocb); // If the block failed to compile @@ -2769,7 +3367,8 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - return target.get_address().unwrap().raw_ptr(cb); } - (target.get_blockid(), target.get_ctx()) + let target_ctx = Context::decode(target.get_ctx()); + (target.get_blockid(), target_ctx) }; let (cfp, original_interp_sp) = unsafe { @@ -2906,7 +3505,7 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - /// Generate a "stub", a piece of code that calls the compiler back when run. /// A piece of code that redeems for more code; a thunk for code. fn gen_branch_stub( - ctx: &Context, + ctx: u32, ocb: &mut OutlinedCb, branch_struct_address: usize, target_idx: u32, @@ -2914,8 +3513,8 @@ fn gen_branch_stub( let ocb = ocb.unwrap(); let mut asm = Assembler::new(); - asm.ctx = *ctx; - asm.set_reg_temps(ctx.reg_temps); + asm.ctx = Context::decode(ctx); + asm.set_reg_temps(asm.ctx.reg_temps); asm_comment!(asm, "branch stub hit"); if asm.ctx.is_return_landing() { @@ -3112,7 +3711,7 @@ pub fn gen_direct_jump(jit: &mut JITState, ctx: &Context, target0: BlockId, asm: // compile the target block right after this one (fallthrough). BranchTarget::Stub(Box::new(BranchStub { address: None, - ctx: *ctx, + ctx: Context::encode(ctx), iseq: Cell::new(target0.iseq), iseq_idx: target0.idx, })) @@ -3364,7 +3963,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) { } // Create a stub for this branch target - let stub_addr = gen_branch_stub(&block.ctx, ocb, branchref.as_ptr() as usize, target_idx as u32); + let stub_addr = gen_branch_stub(block.ctx, ocb, branchref.as_ptr() as usize, target_idx as u32); // In case we were unable to generate a stub (e.g. OOM). Use the block's // exit instead of a stub for the block. It's important that we @@ -3547,11 +4146,6 @@ mod tests { } #[test] - fn context_size() { - assert_eq!(mem::size_of::<Context>(), 23); - } - - #[test] fn types() { // Valid src => dst assert_eq!(Type::Unknown.diff(Type::Unknown), TypeDiff::Compatible(0)); @@ -3695,7 +4289,7 @@ mod tests { iseq: Cell::new(ptr::null()), iseq_idx: 0, address: None, - ctx: Context::default(), + ctx: 0, })))))] }; // For easier soundness reasoning, make sure the reference returned does not out live the @@ -3728,7 +4322,7 @@ mod tests { iseq: Cell::new(ptr::null()), iseq_idx: 0, address: None, - ctx: Context::default(), + ctx: 0, }))))); // Invalid ISeq; we never dereference it. let secret_iseq = NonNull::<rb_iseq_t>::dangling().as_ptr(); diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 68c0304b06..53586cb4f4 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -802,6 +802,7 @@ pub(crate) mod ids { name: min content: b"min" name: max content: b"max" name: hash content: b"hash" + name: pack content: b"pack" name: respond_to_missing content: b"respond_to_missing?" name: to_ary content: b"to_ary" name: eq content: b"==" diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index a03c2d0f00..1f128f5e7e 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -342,7 +342,8 @@ pub const BOP_AND: ruby_basic_operators = 28; pub const BOP_OR: ruby_basic_operators = 29; pub const BOP_CMP: ruby_basic_operators = 30; pub const BOP_DEFAULT: ruby_basic_operators = 31; -pub const BOP_LAST_: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 32; +pub const BOP_LAST_: ruby_basic_operators = 33; pub type ruby_basic_operators = u32; pub type rb_serial_t = ::std::os::raw::c_ulonglong; pub const imemo_env: imemo_type = 0; @@ -696,6 +697,8 @@ pub struct rb_call_data { pub ci: *const rb_callinfo, pub cc: *const rb_callcache, } +pub const RSTRING_CHILLED: ruby_rstring_private_flags = 32768; +pub type ruby_rstring_private_flags = u32; pub const RHASH_PASS_AS_KEYWORDS: ruby_rhash_flags = 8192; pub const RHASH_PROC_DEFAULT: ruby_rhash_flags = 16384; pub const RHASH_ST_TABLE_FLAG: ruby_rhash_flags = 32768; @@ -1060,7 +1063,11 @@ extern "C" { pub fn rb_shape_get_shape_id(obj: VALUE) -> shape_id_t; pub fn rb_shape_get_iv_index(shape: *mut rb_shape_t, id: ID, value: *mut attr_index_t) -> bool; pub fn rb_shape_obj_too_complex(obj: VALUE) -> bool; - pub fn rb_shape_get_next(shape: *mut rb_shape_t, obj: VALUE, id: ID) -> *mut rb_shape_t; + pub fn rb_shape_get_next_no_warnings( + shape: *mut rb_shape_t, + obj: VALUE, + id: ID, + ) -> *mut rb_shape_t; pub fn rb_shape_id(shape: *mut rb_shape_t) -> shape_id_t; pub fn rb_gvar_get(arg1: ID) -> VALUE; pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index 6ffe28f12a..6a7de68576 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -10,8 +10,6 @@ use std::time::Instant; use std::collections::HashMap; use crate::codegen::CodegenGlobals; -use crate::core::Context; -use crate::core::for_each_iseq_payload; use crate::cruby::*; use crate::options::*; use crate::yjit::yjit_enabled_p; @@ -557,6 +555,7 @@ make_counters! { branch_insn_count, branch_known_count, max_inline_versions, + num_contexts_encoded, freed_iseq_count, @@ -641,8 +640,8 @@ pub extern "C" fn rb_yjit_print_stats_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE /// Primitive called in yjit.rb. /// Export all YJIT statistics as a Ruby hash. #[no_mangle] -pub extern "C" fn rb_yjit_get_stats(_ec: EcPtr, _ruby_self: VALUE, context: VALUE) -> VALUE { - with_vm_lock(src_loc!(), || rb_yjit_gen_stats_dict(context == Qtrue)) +pub extern "C" fn rb_yjit_get_stats(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || rb_yjit_gen_stats_dict()) } /// Primitive called in yjit.rb @@ -701,7 +700,7 @@ pub extern "C" fn rb_yjit_incr_counter(counter_name: *const std::os::raw::c_char } /// Export all YJIT statistics as a Ruby hash. -fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { +fn rb_yjit_gen_stats_dict() -> VALUE { // If YJIT is not enabled, return Qnil if !yjit_enabled_p() { return Qnil; @@ -744,14 +743,9 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { // Rust global allocations in bytes hash_aset_usize!(hash, "yjit_alloc_size", GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst)); - // `context` is true at RubyVM::YJIT._print_stats for --yjit-stats. It's false by default - // for RubyVM::YJIT.runtime_stats because counting all Contexts could be expensive. - if context { - let live_context_count = get_live_context_count(); - let context_size = std::mem::size_of::<Context>(); - hash_aset_usize!(hash, "live_context_count", live_context_count); - hash_aset_usize!(hash, "live_context_size", live_context_count * context_size); - } + // How many bytes we are using to store context data + let context_data = CodegenGlobals::get_context_data(); + hash_aset_usize!(hash, "context_data_bytes", context_data.num_bytes()); // VM instructions count hash_aset_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize); @@ -802,16 +796,31 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { rb_hash_aset(hash, key, value); } + // Set method call counts in a Ruby dict fn set_call_counts( calls_hash: VALUE, method_name_to_idx: &mut Option<HashMap<String, usize>>, method_call_count: &mut Option<Vec<u64>>, ) { if let (Some(name_to_idx), Some(call_counts)) = (method_name_to_idx, method_call_count) { + // Create a list of (name, call_count) pairs + let mut pairs = Vec::new(); for (name, idx) in name_to_idx { let count = call_counts[*idx]; + pairs.push((name, count)); + } + + // Sort the vectors by decreasing call counts + pairs.sort_by_key(|e| -(e.1 as i64)); + + // Cap the number of counts reported to avoid + // bloating log files, etc. + pairs.truncate(20); + + // Add the pairs to the dict + for (name, call_count) in pairs { let key = rust_str_to_sym(name); - let value = VALUE::fixnum_from_usize(count as usize); + let value = VALUE::fixnum_from_usize(call_count as usize); unsafe { rb_hash_aset(calls_hash, key, value); } } } @@ -831,21 +840,6 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { hash } -fn get_live_context_count() -> usize { - let mut count = 0; - for_each_iseq_payload(|iseq_payload| { - for blocks in iseq_payload.version_map.iter() { - for block in blocks.iter() { - count += unsafe { block.as_ref() }.get_ctx_count(); - } - } - for block in iseq_payload.dead_blocks.iter() { - count += unsafe { block.as_ref() }.get_ctx_count(); - } - }); - count -} - /// Record the backtrace when a YJIT exit occurs. This functionality requires /// that the stats feature is enabled as well as the --yjit-trace-exits option. /// diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs index cc2c8fe066..b2429df61e 100644 --- a/yjit/src/yjit.rs +++ b/yjit/src/yjit.rs @@ -22,6 +22,11 @@ pub extern "C" fn rb_yjit_parse_option(str_ptr: *const raw::c_char) -> bool { return parse_option(str_ptr).is_some(); } +#[no_mangle] +pub extern "C" fn rb_yjit_option_disable() -> bool { + return get_option!(disable); +} + /// Like rb_yjit_enabled_p, but for Rust code. pub fn yjit_enabled_p() -> bool { unsafe { rb_yjit_enabled_p } @@ -34,7 +39,7 @@ pub extern "C" fn rb_yjit_init(yjit_enabled: bool) { yjit_reg_method_codegen_fns(); // If --yjit-disable, yjit_init() will not be called until RubyVM::YJIT.enable. - if yjit_enabled && !get_option!(disable) { + if yjit_enabled { yjit_init(); } } |