diff options
499 files changed, 10838 insertions, 6595 deletions
diff --git a/.github/actions/setup/directories/action.yml b/.github/actions/setup/directories/action.yml index a59f575c99..5264e0e969 100644 --- a/.github/actions/setup/directories/action.yml +++ b/.github/actions/setup/directories/action.yml @@ -88,7 +88,7 @@ runs: git config --global init.defaultBranch garbage - if: inputs.checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: path: ${{ inputs.srcdir }} fetch-depth: ${{ inputs.fetch-depth }} diff --git a/.github/actions/setup/macos/action.yml b/.github/actions/setup/macos/action.yml index 896930de84..b96e959aa6 100644 --- a/.github/actions/setup/macos/action.yml +++ b/.github/actions/setup/macos/action.yml @@ -18,7 +18,11 @@ runs: - name: Set ENV shell: bash run: | - for lib in openssl@1.1 gmp; do + for lib in gmp; do + ruby_configure_args="${ruby_configure_args:+$ruby_configure_args }--with-${lib%@*}-dir=$(brew --prefix $lib)" + done + for lib in openssl@1.1; do CONFIGURE_ARGS="${CONFIGURE_ARGS:+$CONFIGURE_ARGS }--with-${lib%@*}-dir=$(brew --prefix $lib)" done + echo ruby_configure_args="${ruby_configure_args}" >> $GITHUB_ENV echo CONFIGURE_ARGS="${CONFIGURE_ARGS}" >> $GITHUB_ENV diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml index 589672b4bb..fe34ab0222 100644 --- a/.github/workflows/annocheck.yml +++ b/.github/workflows/annocheck.yml @@ -4,7 +4,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +12,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -63,7 +63,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -74,7 +74,7 @@ jobs: builddir: build makeup: true - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml index ac0d5f967f..09e28cb5f1 100644 --- a/.github/workflows/baseruby.yml +++ b/.github/workflows/baseruby.yml @@ -4,7 +4,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +12,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -51,12 +51,12 @@ jobs: - ruby-3.3 steps: - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: ${{ matrix.ruby }} bundler: none - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: ./.github/actions/setup/ubuntu diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml index fcc005239b..6f06451a99 100644 --- a/.github/workflows/bundled_gems.yml +++ b/.github/workflows/bundled_gems.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml index 076af08aed..c52e30a5c6 100644 --- a/.github/workflows/check_dependencies.yml +++ b/.github/workflows/check_dependencies.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -45,7 +45,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: ./.github/actions/setup/ubuntu if: ${{ contains(matrix.os, 'ubuntu') }} @@ -55,7 +55,7 @@ jobs: - uses: ./.github/actions/setup/directories - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/check_misc.yml b/.github/workflows/check_misc.yml index 48434a47a9..f26319f448 100644 --- a/.github/workflows/check_misc.yml +++ b/.github/workflows/check_misc.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c7d0def7ae..7d12c1fe81 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -5,7 +5,7 @@ on: branches: ['master'] paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -13,7 +13,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Install libraries if: ${{ contains(matrix.os, 'macos') }} @@ -80,15 +80,15 @@ jobs: run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - name: Initialize CodeQL - uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/init@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/autobuild@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/analyze@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 with: category: '/language:${{ matrix.language }}' upload: False @@ -118,7 +118,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 + uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 5e5eb69b6f..8937a5b15f 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -4,7 +4,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -12,7 +12,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -233,7 +233,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 54e30129e4..151df8126b 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -52,7 +52,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -70,8 +70,13 @@ jobs: # Set fetch-depth: 0 so that Launchable can receive commits information. fetch-depth: 10 + - name: make sure that kern.coredump=1 + run: | + sysctl -n kern.coredump + sudo sysctl -w kern.coredump=1 + - name: Run configure - run: ../src/configure -C --disable-install-doc + run: ../src/configure -C --disable-install-doc ${ruby_configure_args} - run: make prepare-gems if: ${{ matrix.test_task == 'test-bundled-gems' }} @@ -124,6 +129,36 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot if: ${{ failure() }} + - name: Resolve job ID + id: job_id + uses: actions/github-script@main + env: + matrix: ${{ toJson(matrix) }} + with: + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `${context.job}${matrix ? ` (${Object.values(matrix).join(", ")})` : ""}`; + return workflow_run.jobs.find((job) => job.name === job_name).id; + + - name: upload /cores + run: | + ls -l /cores + CORES_TAR_GZ="cores-${{ github.run_id }}-${{ steps.job_id.outputs.result }}.tar.gz" + echo "test: $CORES_TAR_GZ" + if [ -n "$(ls /cores)" ]; then + tar czf /cores "$CORES_TAR_GZ" + aws s3 cp "$CORES_TAR_GZ" "s3://ruby-core-files/$CORES_TAR_GZ" + fi + env: + AWS_ACCESS_KEY_ID: ${{ secrets.RUBY_CORE_FILES_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RUBY_CORE_FILES_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ap-northeast-1 + result: if: ${{ always() }} name: ${{ github.workflow }} result diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 75cdfbab65..9328b72608 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -66,7 +66,7 @@ jobs: steps: - name: Set up Ruby & MSYS2 - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: ${{ matrix.baseruby }} @@ -97,7 +97,7 @@ jobs: $result working-directory: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 95a36f1634..63abd09cb8 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -55,7 +55,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -92,12 +92,12 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb" --exclude="prism/locals_test.rb" --exclude="prism/newline_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' RUN_OPTS: ${{ matrix.run_opts }} - - name: make test-prism-spec + - name: make test-spec run: | - $SETARCH make -s test-prism-spec SPECOPTS="$SPECOPTS" + $SETARCH make -s test-spec SPECOPTS="$SPECOPTS" timeout-minutes: 10 env: GNUMAKEFLAGS: '' diff --git a/.github/workflows/rjit-bindgen.yml b/.github/workflows/rjit-bindgen.yml index 1263c20505..531d1e95a8 100644 --- a/.github/workflows/rjit-bindgen.yml +++ b/.github/workflows/rjit-bindgen.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -47,11 +47,11 @@ jobs: steps: - name: Set up Ruby - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.1' - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index 7a5e6cf7c0..509088db7d 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -55,7 +55,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 00a0638e96..eeff61d558 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -32,12 +32,12 @@ jobs: steps: - name: 'Checkout code' - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: persist-credentials: false - name: 'Run analysis' - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3 with: results_file: results.sarif results_format: sarif @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: 'Upload to code-scanning' - uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v2.1.27 + uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v2.1.27 with: sarif_file: results.sarif diff --git a/.github/workflows/spec_guards.yml b/.github/workflows/spec_guards.yml index 165a0150b5..55e7348bc5 100644 --- a/.github/workflows/spec_guards.yml +++ b/.github/workflows/spec_guards.yml @@ -45,9 +45,9 @@ jobs: - ruby-3.3 steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: ${{ matrix.ruby }} bundler: none diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index b6f989a03d..7fbf0504cc 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -58,7 +58,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -67,7 +67,7 @@ jobs: with: arch: ${{ matrix.arch }} - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index e0b67496ac..617ce41b37 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -61,7 +61,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -100,7 +100,7 @@ jobs: run: | echo "WASI_SDK_PATH=/opt/wasi-sdk" >> $GITHUB_ENV - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 67cca44fa6..49313bc118 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -11,7 +11,7 @@ on: pull_request: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -91,7 +91,7 @@ jobs: ${{ steps.find-tools.outputs.needs }} if: ${{ steps.find-tools.outputs.needs != '' }} - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none @@ -123,7 +123,7 @@ jobs: Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH shell: pwsh - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml index 948644d06e..bd17b302ae 100644 --- a/.github/workflows/yjit-macos.yml +++ b/.github/workflows/yjit-macos.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -37,7 +37,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - run: RUST_BACKTRACE=1 cargo test working-directory: yjit @@ -60,11 +60,13 @@ jobs: - test_task: 'check' configure: '--enable-yjit=dev' yjit_opts: '--yjit-call-threshold=1 --yjit-verify-ctx --yjit-code-gc' + specopts: '-T --yjit-call-threshold=1 -T --yjit-verify-ctx -T --yjit-code-gc' fail-fast: false env: GITPULLOPTIONS: --no-tags origin ${{ github.ref }} RUN_OPTS: ${{ matrix.yjit_opts }} + SPECOPTS: ${{ matrix.specopts }} runs-on: ${{ github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14' }} @@ -79,7 +81,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -128,6 +130,7 @@ jobs: run: >- make -s ${{ matrix.test_task }} ${TESTS:+TESTS="$TESTS"} RUN_OPTS="$RUN_OPTS" + SPECOPTS="$SPECOPTS" timeout-minutes: 60 env: RUBY_TESTOPTS: '-q --tty=no' diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 02d0ac7536..45700224fa 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -3,7 +3,7 @@ on: push: paths-ignore: - 'doc/**' - - '**/man' + - '**/man/*' - '**.md' - '**.rdoc' - '**/.document' @@ -36,7 +36,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 # For now we can't run cargo test --offline because it complains about the # capstone dependency, even though the dependency is optional @@ -68,7 +68,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 # Check that we don't have linting errors in release mode, too - run: cargo clippy --all-targets --all-features @@ -95,6 +95,7 @@ jobs: - test_task: 'check' configure: '--enable-yjit=dev' yjit_opts: '--yjit-call-threshold=1 --yjit-verify-ctx --yjit-code-gc' + specopts: '-T --yjit-call-threshold=1 -T --yjit-verify-ctx -T --yjit-code-gc' - test_task: 'test-bundled-gems' configure: '--enable-yjit=dev' @@ -108,6 +109,7 @@ jobs: GITPULLOPTIONS: --no-tags origin ${{ github.ref }} RUN_OPTS: ${{ matrix.yjit_opts }} YJIT_BENCH_OPTS: ${{ matrix.yjit_bench_opts }} + SPECOPTS: ${{ matrix.specopts }} RUBY_DEBUG: ci BUNDLE_JOBS: 8 # for yjit-bench RUST_BACKTRACE: 1 @@ -125,7 +127,7 @@ jobs: )}} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: sparse-checkout-cone-mode: false sparse-checkout: /.github @@ -145,7 +147,7 @@ jobs: if: ${{ matrix.rust_version }} run: rustup install ${{ matrix.rust_version }} --profile minimal - - uses: ruby/setup-ruby@70da3bbf44ac06db1b0547ce2acc9380a5270d1e # v1.175.0 + - uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: '3.0' bundler: none @@ -180,7 +182,7 @@ jobs: - name: make ${{ matrix.test_task }} run: >- make -s ${{ matrix.test_task }} ${TESTS:+TESTS="$TESTS"} - RUN_OPTS="$RUN_OPTS" MSPECOPT=--debug + RUN_OPTS="$RUN_OPTS" MSPECOPT=--debug SPECOPTS="$SPECOPTS" YJIT_BENCH_OPTS="$YJIT_BENCH_OPTS" YJIT_BINDGEN_DIFF_OPTS="$YJIT_BINDGEN_DIFF_OPTS" timeout-minutes: 90 env: diff --git a/.gitignore b/.gitignore index d7a8b978c9..b6beba3b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,7 @@ lcov*.info # /coroutine/ !/coroutine/**/*.s +!/coroutine/**/*.S # /enc/trans/ /enc/trans/*.c @@ -45,35 +45,37 @@ The following default gems are updated. * erb 4.0.4 * fiddle 1.1.3 * io-console 0.7.2 -* irb 1.12.0 +* irb 1.13.1 * json 2.7.2 * net-http 0.4.1 * optparse 0.5.0 -* prism 0.27.0 -* rdoc 6.6.3.1 -* reline 0.5.3 +* prism 0.29.0 +* rdoc 6.7.0 +* reline 0.5.7 * resolv 0.4.0 * stringio 3.1.1 * strscan 3.1.1 The following bundled gems are updated. -* minitest 5.22.3 +* minitest 5.23.1 * rake 13.2.1 * test-unit 3.6.2 +* rexml 3.2.8 * net-ftp 0.3.4 -* net-imap 0.4.10 +* net-imap 0.4.11 * net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 * debug 1.9.2 +* racc 1.8.0 The following bundled gems are promoted from default gems. * mutex_m 0.2.0 * getoptlong 0.2.1 * base64 0.2.0 -* bigdecimal 3.1.7 +* bigdecimal 3.1.8 * observer 0.1.2 * abbrev 0.1.2 * resolv-replace 0.1.1 @@ -122,7 +124,7 @@ See GitHub releases like [GitHub Releases of Logger](https://github.com/ruby/log a warning on verbose mode (`-w`). [[Feature #15554]] -* Redefining some core methods that are specially optimized by the interpeter +* Redefining some core methods that are specially optimized by the interpreter and JIT like `String.freeze` or `Integer#+` now emits a performance class warning (`-W:performance` or `Warning[:performance] = true`). [[Feature #20429]] @@ -350,7 +350,7 @@ ary_memcpy(VALUE ary, long beg, long argc, const VALUE *argv) } static VALUE * -ary_heap_alloc(size_t capa) +ary_heap_alloc_buffer(size_t capa) { return ALLOC_N(VALUE, capa); } @@ -404,7 +404,7 @@ ary_resize_capa(VALUE ary, long capacity) size_t new_capa = capacity; if (ARY_EMBED_P(ary)) { long len = ARY_EMBED_LEN(ary); - VALUE *ptr = ary_heap_alloc(capacity); + VALUE *ptr = ary_heap_alloc_buffer(capacity); MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len); FL_UNSET_EMBED(ary); @@ -555,7 +555,7 @@ rb_ary_cancel_sharing(VALUE ary) rb_ary_decrement_share(shared_root); } else { - VALUE *ptr = ary_heap_alloc(len); + VALUE *ptr = ary_heap_alloc_buffer(len); MEMCPY(ptr, ARY_HEAP_PTR(ary), VALUE, len); rb_ary_unshare(ary); ARY_SET_CAPA(ary, len); @@ -714,7 +714,7 @@ ary_new(VALUE klass, long capa) ARY_SET_CAPA(ary, capa); RUBY_ASSERT(!ARY_EMBED_P(ary)); - ARY_SET_PTR(ary, ary_heap_alloc(capa)); + ARY_SET_PTR(ary, ary_heap_alloc_buffer(capa)); ARY_SET_HEAP_LEN(ary, 0); } @@ -818,7 +818,7 @@ ec_ary_new(rb_execution_context_t *ec, VALUE klass, long capa) ARY_SET_CAPA(ary, capa); RUBY_ASSERT(!ARY_EMBED_P(ary)); - ARY_SET_PTR(ary, ary_heap_alloc(capa)); + ARY_SET_PTR(ary, ary_heap_alloc_buffer(capa)); ARY_SET_HEAP_LEN(ary, 0); } @@ -880,6 +880,19 @@ rb_ary_free(VALUE ary) } } +VALUE +rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len, bool freeze) +{ + fake_ary->basic.flags = T_ARRAY; + VALUE ary = (VALUE)fake_ary; + RBASIC_CLEAR_CLASS(ary); + ARY_SET_PTR(ary, list); + ARY_SET_HEAP_LEN(ary, len); + ARY_SET_CAPA(ary, len); + if (freeze) OBJ_FREEZE(ary); + return ary; +} + size_t rb_ary_memsize(VALUE ary) { @@ -918,7 +931,7 @@ ary_make_shared(VALUE ary) FL_SET_SHARED_ROOT(shared); if (ARY_EMBED_P(ary)) { - VALUE *ptr = ary_heap_alloc(capa); + VALUE *ptr = ary_heap_alloc_buffer(capa); ARY_SET_PTR(shared, ptr); ary_memcpy(shared, 0, len, RARRAY_CONST_PTR(ary)); @@ -4559,7 +4572,7 @@ rb_ary_replace(VALUE copy, VALUE orig) * contents of orig. */ else if (ARY_EMBED_P(orig)) { long len = ARY_EMBED_LEN(orig); - VALUE *ptr = ary_heap_alloc(len); + VALUE *ptr = ary_heap_alloc_buffer(len); FL_UNSET_EMBED(copy); ARY_SET_PTR(copy, ptr); @@ -16,7 +16,7 @@ static VALUE rb_mAST; static VALUE rb_cNode; struct ASTNodeData { - VALUE vast; + VALUE ast_value; const NODE *node; }; @@ -24,14 +24,14 @@ static void node_gc_mark(void *ptr) { struct ASTNodeData *data = (struct ASTNodeData *)ptr; - rb_gc_mark(data->vast); + rb_gc_mark(data->ast_value); } static size_t node_memsize(const void *ptr) { struct ASTNodeData *data = (struct ASTNodeData *)ptr; - rb_ast_t *ast = rb_ruby_ast_data_get(data->vast); + rb_ast_t *ast = rb_ruby_ast_data_get(data->ast_value); return sizeof(struct ASTNodeData) + rb_ast_memsize(ast); } @@ -46,22 +46,22 @@ static const rb_data_type_t rb_node_type = { static VALUE rb_ast_node_alloc(VALUE klass); static void -setup_node(VALUE obj, VALUE vast, const NODE *node) +setup_node(VALUE obj, VALUE ast_value, const NODE *node) { struct ASTNodeData *data; TypedData_Get_Struct(obj, struct ASTNodeData, &rb_node_type, data); - data->vast = vast; + data->ast_value = ast_value; data->node = node; } static VALUE -ast_new_internal(VALUE vast, const NODE *node) +ast_new_internal(VALUE ast_value, const NODE *node) { VALUE obj; obj = rb_ast_node_alloc(rb_cNode); - setup_node(obj, vast, node); + setup_node(obj, ast_value, node); return obj; } @@ -76,16 +76,16 @@ ast_parse_new(void) } static VALUE -ast_parse_done(VALUE vast) +ast_parse_done(VALUE ast_value) { - rb_ast_t *ast = rb_ruby_ast_data_get(vast); + rb_ast_t *ast = rb_ruby_ast_data_get(ast_value); if (!ast->body.root) { rb_ast_dispose(ast); rb_exc_raise(GET_EC()->errinfo); } - return ast_new_internal(vast, (NODE *)ast->body.root); + return ast_new_internal(ast_value, (NODE *)ast->body.root); } static VALUE @@ -97,15 +97,15 @@ ast_s_parse(rb_execution_context_t *ec, VALUE module, VALUE str, VALUE keep_scri static VALUE rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens) { - VALUE vast; + VALUE ast_value; StringValue(str); VALUE vparser = ast_parse_new(); if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); - vast = rb_parser_compile_string_path(vparser, Qnil, str, 1); - return ast_parse_done(vast); + ast_value = rb_parser_compile_string_path(vparser, Qnil, str, 1); + return ast_parse_done(ast_value); } static VALUE @@ -118,7 +118,7 @@ static VALUE rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens) { VALUE f; - VALUE vast = Qnil; + VALUE ast_value = Qnil; rb_encoding *enc = rb_utf8_encoding(); f = rb_file_open_str(path, "r"); @@ -127,23 +127,23 @@ rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VAL if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); - vast = rb_parser_compile_file_path(vparser, Qnil, f, 1); + ast_value = rb_parser_compile_file_path(vparser, Qnil, f, 1); rb_io_close(f); - return ast_parse_done(vast); + return ast_parse_done(ast_value); } static VALUE rb_ast_parse_array(VALUE array, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens) { - VALUE vast = Qnil; + VALUE ast_value = Qnil; array = rb_check_array_type(array); VALUE vparser = ast_parse_new(); if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); - vast = rb_parser_compile_array(vparser, Qnil, array, 1); - return ast_parse_done(vast); + ast_value = rb_parser_compile_array(vparser, Qnil, array, 1); + return ast_parse_done(ast_value); } static VALUE node_children(VALUE, const NODE*); @@ -158,7 +158,7 @@ node_find(VALUE self, const int node_id) if (nd_node_id(data->node) == node_id) return self; - ary = node_children(data->vast, data->node); + ary = node_children(data->ast_value, data->node); for (i = 0; i < RARRAY_LEN(ary); i++) { VALUE child = RARRAY_AREF(ary, i); @@ -281,10 +281,10 @@ ast_node_node_id(rb_execution_context_t *ec, VALUE self) return INT2FIX(nd_node_id(data->node)); } -#define NEW_CHILD(vast, node) (node ? ast_new_internal(vast, node) : Qnil) +#define NEW_CHILD(ast_value, node) (node ? ast_new_internal(ast_value, node) : Qnil) static VALUE -rb_ary_new_from_node_args(VALUE vast, long n, ...) +rb_ary_new_from_node_args(VALUE ast_value, long n, ...) { va_list ar; VALUE ary; @@ -296,39 +296,39 @@ rb_ary_new_from_node_args(VALUE vast, long n, ...) for (i=0; i<n; i++) { NODE *node; node = va_arg(ar, NODE *); - rb_ary_push(ary, NEW_CHILD(vast, node)); + rb_ary_push(ary, NEW_CHILD(ast_value, node)); } va_end(ar); return ary; } static VALUE -dump_block(VALUE vast, const struct RNode_BLOCK *node) +dump_block(VALUE ast_value, const struct RNode_BLOCK *node) { VALUE ary = rb_ary_new(); do { - rb_ary_push(ary, NEW_CHILD(vast, node->nd_head)); + rb_ary_push(ary, NEW_CHILD(ast_value, node->nd_head)); } while (node->nd_next && nd_type_p(node->nd_next, NODE_BLOCK) && (node = RNODE_BLOCK(node->nd_next), 1)); if (node->nd_next) { - rb_ary_push(ary, NEW_CHILD(vast, node->nd_next)); + rb_ary_push(ary, NEW_CHILD(ast_value, node->nd_next)); } return ary; } static VALUE -dump_array(VALUE vast, const struct RNode_LIST *node) +dump_array(VALUE ast_value, const struct RNode_LIST *node) { VALUE ary = rb_ary_new(); - rb_ary_push(ary, NEW_CHILD(vast, node->nd_head)); + rb_ary_push(ary, NEW_CHILD(ast_value, node->nd_head)); while (node->nd_next && nd_type_p(node->nd_next, NODE_LIST)) { node = RNODE_LIST(node->nd_next); - rb_ary_push(ary, NEW_CHILD(vast, node->nd_head)); + rb_ary_push(ary, NEW_CHILD(ast_value, node->nd_head)); } - rb_ary_push(ary, NEW_CHILD(vast, node->nd_next)); + rb_ary_push(ary, NEW_CHILD(ast_value, node->nd_next)); return ary; } @@ -350,155 +350,155 @@ no_name_rest(void) } static VALUE -rest_arg(VALUE vast, const NODE *rest_arg) +rest_arg(VALUE ast_value, const NODE *rest_arg) { - return NODE_NAMED_REST_P(rest_arg) ? NEW_CHILD(vast, rest_arg) : no_name_rest(); + return NODE_NAMED_REST_P(rest_arg) ? NEW_CHILD(ast_value, rest_arg) : no_name_rest(); } static VALUE -node_children(VALUE vast, const NODE *node) +node_children(VALUE ast_value, const NODE *node) { char name[sizeof("$") + DECIMAL_SIZE_OF(long)]; enum node_type type = nd_type(node); switch (type) { case NODE_BLOCK: - return dump_block(vast, RNODE_BLOCK(node)); + return dump_block(ast_value, RNODE_BLOCK(node)); case NODE_IF: - return rb_ary_new_from_node_args(vast, 3, RNODE_IF(node)->nd_cond, RNODE_IF(node)->nd_body, RNODE_IF(node)->nd_else); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_IF(node)->nd_cond, RNODE_IF(node)->nd_body, RNODE_IF(node)->nd_else); case NODE_UNLESS: - return rb_ary_new_from_node_args(vast, 3, RNODE_UNLESS(node)->nd_cond, RNODE_UNLESS(node)->nd_body, RNODE_UNLESS(node)->nd_else); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_UNLESS(node)->nd_cond, RNODE_UNLESS(node)->nd_body, RNODE_UNLESS(node)->nd_else); case NODE_CASE: - return rb_ary_new_from_node_args(vast, 2, RNODE_CASE(node)->nd_head, RNODE_CASE(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_CASE(node)->nd_head, RNODE_CASE(node)->nd_body); case NODE_CASE2: - return rb_ary_new_from_node_args(vast, 2, RNODE_CASE2(node)->nd_head, RNODE_CASE2(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_CASE2(node)->nd_head, RNODE_CASE2(node)->nd_body); case NODE_CASE3: - return rb_ary_new_from_node_args(vast, 2, RNODE_CASE3(node)->nd_head, RNODE_CASE3(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_CASE3(node)->nd_head, RNODE_CASE3(node)->nd_body); case NODE_WHEN: - return rb_ary_new_from_node_args(vast, 3, RNODE_WHEN(node)->nd_head, RNODE_WHEN(node)->nd_body, RNODE_WHEN(node)->nd_next); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_WHEN(node)->nd_head, RNODE_WHEN(node)->nd_body, RNODE_WHEN(node)->nd_next); case NODE_IN: - return rb_ary_new_from_node_args(vast, 3, RNODE_IN(node)->nd_head, RNODE_IN(node)->nd_body, RNODE_IN(node)->nd_next); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_IN(node)->nd_head, RNODE_IN(node)->nd_body, RNODE_IN(node)->nd_next); case NODE_WHILE: case NODE_UNTIL: - return rb_ary_push(rb_ary_new_from_node_args(vast, 2, RNODE_WHILE(node)->nd_cond, RNODE_WHILE(node)->nd_body), + return rb_ary_push(rb_ary_new_from_node_args(ast_value, 2, RNODE_WHILE(node)->nd_cond, RNODE_WHILE(node)->nd_body), RBOOL(RNODE_WHILE(node)->nd_state)); case NODE_ITER: case NODE_FOR: - return rb_ary_new_from_node_args(vast, 2, RNODE_ITER(node)->nd_iter, RNODE_ITER(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_ITER(node)->nd_iter, RNODE_ITER(node)->nd_body); case NODE_FOR_MASGN: - return rb_ary_new_from_node_args(vast, 1, RNODE_FOR_MASGN(node)->nd_var); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_FOR_MASGN(node)->nd_var); case NODE_BREAK: - return rb_ary_new_from_node_args(vast, 1, RNODE_BREAK(node)->nd_stts); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_BREAK(node)->nd_stts); case NODE_NEXT: - return rb_ary_new_from_node_args(vast, 1, RNODE_NEXT(node)->nd_stts); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_NEXT(node)->nd_stts); case NODE_RETURN: - return rb_ary_new_from_node_args(vast, 1, RNODE_RETURN(node)->nd_stts); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_RETURN(node)->nd_stts); case NODE_REDO: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_RETRY: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_BEGIN: - return rb_ary_new_from_node_args(vast, 1, RNODE_BEGIN(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_BEGIN(node)->nd_body); case NODE_RESCUE: - return rb_ary_new_from_node_args(vast, 3, RNODE_RESCUE(node)->nd_head, RNODE_RESCUE(node)->nd_resq, RNODE_RESCUE(node)->nd_else); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_RESCUE(node)->nd_head, RNODE_RESCUE(node)->nd_resq, RNODE_RESCUE(node)->nd_else); case NODE_RESBODY: - return rb_ary_new_from_node_args(vast, 3, RNODE_RESBODY(node)->nd_args, RNODE_RESBODY(node)->nd_body, RNODE_RESBODY(node)->nd_next); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_RESBODY(node)->nd_args, RNODE_RESBODY(node)->nd_body, RNODE_RESBODY(node)->nd_next); case NODE_ENSURE: - return rb_ary_new_from_node_args(vast, 2, RNODE_ENSURE(node)->nd_head, RNODE_ENSURE(node)->nd_ensr); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_ENSURE(node)->nd_head, RNODE_ENSURE(node)->nd_ensr); case NODE_AND: case NODE_OR: { VALUE ary = rb_ary_new(); while (1) { - rb_ary_push(ary, NEW_CHILD(vast, RNODE_AND(node)->nd_1st)); + rb_ary_push(ary, NEW_CHILD(ast_value, RNODE_AND(node)->nd_1st)); if (!RNODE_AND(node)->nd_2nd || !nd_type_p(RNODE_AND(node)->nd_2nd, type)) break; node = RNODE_AND(node)->nd_2nd; } - rb_ary_push(ary, NEW_CHILD(vast, RNODE_AND(node)->nd_2nd)); + rb_ary_push(ary, NEW_CHILD(ast_value, RNODE_AND(node)->nd_2nd)); return ary; } case NODE_MASGN: if (NODE_NAMED_REST_P(RNODE_MASGN(node)->nd_args)) { - return rb_ary_new_from_node_args(vast, 3, RNODE_MASGN(node)->nd_value, RNODE_MASGN(node)->nd_head, RNODE_MASGN(node)->nd_args); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_MASGN(node)->nd_value, RNODE_MASGN(node)->nd_head, RNODE_MASGN(node)->nd_args); } else { - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_MASGN(node)->nd_value), - NEW_CHILD(vast, RNODE_MASGN(node)->nd_head), + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_MASGN(node)->nd_value), + NEW_CHILD(ast_value, RNODE_MASGN(node)->nd_head), no_name_rest()); } case NODE_LASGN: if (NODE_REQUIRED_KEYWORD_P(RNODE_LASGN(node)->nd_value)) { return rb_ary_new_from_args(2, var_name(RNODE_LASGN(node)->nd_vid), ID2SYM(rb_intern("NODE_SPECIAL_REQUIRED_KEYWORD"))); } - return rb_ary_new_from_args(2, var_name(RNODE_LASGN(node)->nd_vid), NEW_CHILD(vast, RNODE_LASGN(node)->nd_value)); + return rb_ary_new_from_args(2, var_name(RNODE_LASGN(node)->nd_vid), NEW_CHILD(ast_value, RNODE_LASGN(node)->nd_value)); case NODE_DASGN: if (NODE_REQUIRED_KEYWORD_P(RNODE_DASGN(node)->nd_value)) { return rb_ary_new_from_args(2, var_name(RNODE_DASGN(node)->nd_vid), ID2SYM(rb_intern("NODE_SPECIAL_REQUIRED_KEYWORD"))); } - return rb_ary_new_from_args(2, var_name(RNODE_DASGN(node)->nd_vid), NEW_CHILD(vast, RNODE_DASGN(node)->nd_value)); + return rb_ary_new_from_args(2, var_name(RNODE_DASGN(node)->nd_vid), NEW_CHILD(ast_value, RNODE_DASGN(node)->nd_value)); case NODE_IASGN: - return rb_ary_new_from_args(2, var_name(RNODE_IASGN(node)->nd_vid), NEW_CHILD(vast, RNODE_IASGN(node)->nd_value)); + return rb_ary_new_from_args(2, var_name(RNODE_IASGN(node)->nd_vid), NEW_CHILD(ast_value, RNODE_IASGN(node)->nd_value)); case NODE_CVASGN: - return rb_ary_new_from_args(2, var_name(RNODE_CVASGN(node)->nd_vid), NEW_CHILD(vast, RNODE_CVASGN(node)->nd_value)); + return rb_ary_new_from_args(2, var_name(RNODE_CVASGN(node)->nd_vid), NEW_CHILD(ast_value, RNODE_CVASGN(node)->nd_value)); case NODE_GASGN: - return rb_ary_new_from_args(2, var_name(RNODE_GASGN(node)->nd_vid), NEW_CHILD(vast, RNODE_GASGN(node)->nd_value)); + return rb_ary_new_from_args(2, var_name(RNODE_GASGN(node)->nd_vid), NEW_CHILD(ast_value, RNODE_GASGN(node)->nd_value)); case NODE_CDECL: if (RNODE_CDECL(node)->nd_vid) { - return rb_ary_new_from_args(2, ID2SYM(RNODE_CDECL(node)->nd_vid), NEW_CHILD(vast, RNODE_CDECL(node)->nd_value)); + return rb_ary_new_from_args(2, ID2SYM(RNODE_CDECL(node)->nd_vid), NEW_CHILD(ast_value, RNODE_CDECL(node)->nd_value)); } - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_CDECL(node)->nd_else), ID2SYM(RNODE_COLON2(RNODE_CDECL(node)->nd_else)->nd_mid), NEW_CHILD(vast, RNODE_CDECL(node)->nd_value)); + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_CDECL(node)->nd_else), ID2SYM(RNODE_COLON2(RNODE_CDECL(node)->nd_else)->nd_mid), NEW_CHILD(ast_value, RNODE_CDECL(node)->nd_value)); case NODE_OP_ASGN1: - return rb_ary_new_from_args(4, NEW_CHILD(vast, RNODE_OP_ASGN1(node)->nd_recv), + return rb_ary_new_from_args(4, NEW_CHILD(ast_value, RNODE_OP_ASGN1(node)->nd_recv), ID2SYM(RNODE_OP_ASGN1(node)->nd_mid), - NEW_CHILD(vast, RNODE_OP_ASGN1(node)->nd_index), - NEW_CHILD(vast, RNODE_OP_ASGN1(node)->nd_rvalue)); + NEW_CHILD(ast_value, RNODE_OP_ASGN1(node)->nd_index), + NEW_CHILD(ast_value, RNODE_OP_ASGN1(node)->nd_rvalue)); case NODE_OP_ASGN2: - return rb_ary_new_from_args(5, NEW_CHILD(vast, RNODE_OP_ASGN2(node)->nd_recv), + return rb_ary_new_from_args(5, NEW_CHILD(ast_value, RNODE_OP_ASGN2(node)->nd_recv), RBOOL(RNODE_OP_ASGN2(node)->nd_aid), ID2SYM(RNODE_OP_ASGN2(node)->nd_vid), ID2SYM(RNODE_OP_ASGN2(node)->nd_mid), - NEW_CHILD(vast, RNODE_OP_ASGN2(node)->nd_value)); + NEW_CHILD(ast_value, RNODE_OP_ASGN2(node)->nd_value)); case NODE_OP_ASGN_AND: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_OP_ASGN_AND(node)->nd_head), ID2SYM(idANDOP), - NEW_CHILD(vast, RNODE_OP_ASGN_AND(node)->nd_value)); + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_OP_ASGN_AND(node)->nd_head), ID2SYM(idANDOP), + NEW_CHILD(ast_value, RNODE_OP_ASGN_AND(node)->nd_value)); case NODE_OP_ASGN_OR: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_OP_ASGN_OR(node)->nd_head), ID2SYM(idOROP), - NEW_CHILD(vast, RNODE_OP_ASGN_OR(node)->nd_value)); + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_OP_ASGN_OR(node)->nd_head), ID2SYM(idOROP), + NEW_CHILD(ast_value, RNODE_OP_ASGN_OR(node)->nd_value)); case NODE_OP_CDECL: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_OP_CDECL(node)->nd_head), + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_OP_CDECL(node)->nd_head), ID2SYM(RNODE_OP_CDECL(node)->nd_aid), - NEW_CHILD(vast, RNODE_OP_CDECL(node)->nd_value)); + NEW_CHILD(ast_value, RNODE_OP_CDECL(node)->nd_value)); case NODE_CALL: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_CALL(node)->nd_recv), + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_CALL(node)->nd_recv), ID2SYM(RNODE_CALL(node)->nd_mid), - NEW_CHILD(vast, RNODE_CALL(node)->nd_args)); + NEW_CHILD(ast_value, RNODE_CALL(node)->nd_args)); case NODE_OPCALL: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_OPCALL(node)->nd_recv), + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_OPCALL(node)->nd_recv), ID2SYM(RNODE_OPCALL(node)->nd_mid), - NEW_CHILD(vast, RNODE_OPCALL(node)->nd_args)); + NEW_CHILD(ast_value, RNODE_OPCALL(node)->nd_args)); case NODE_QCALL: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_QCALL(node)->nd_recv), + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_QCALL(node)->nd_recv), ID2SYM(RNODE_QCALL(node)->nd_mid), - NEW_CHILD(vast, RNODE_QCALL(node)->nd_args)); + NEW_CHILD(ast_value, RNODE_QCALL(node)->nd_args)); case NODE_FCALL: return rb_ary_new_from_args(2, ID2SYM(RNODE_FCALL(node)->nd_mid), - NEW_CHILD(vast, RNODE_FCALL(node)->nd_args)); + NEW_CHILD(ast_value, RNODE_FCALL(node)->nd_args)); case NODE_VCALL: return rb_ary_new_from_args(1, ID2SYM(RNODE_VCALL(node)->nd_mid)); case NODE_SUPER: - return rb_ary_new_from_node_args(vast, 1, RNODE_SUPER(node)->nd_args); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_SUPER(node)->nd_args); case NODE_ZSUPER: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_LIST: - return dump_array(vast, RNODE_LIST(node)); + return dump_array(ast_value, RNODE_LIST(node)); case NODE_ZLIST: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_HASH: - return rb_ary_new_from_node_args(vast, 1, RNODE_HASH(node)->nd_head); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_HASH(node)->nd_head); case NODE_YIELD: - return rb_ary_new_from_node_args(vast, 1, RNODE_YIELD(node)->nd_head); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_YIELD(node)->nd_head); case NODE_LVAR: return rb_ary_new_from_args(1, var_name(RNODE_LVAR(node)->nd_vid)); case NODE_DVAR: @@ -523,11 +523,11 @@ node_children(VALUE vast, const NODE *node) return rb_ary_new_from_args(1, rb_node_regx_string_val(node)); case NODE_MATCH2: if (RNODE_MATCH2(node)->nd_args) { - return rb_ary_new_from_node_args(vast, 3, RNODE_MATCH2(node)->nd_recv, RNODE_MATCH2(node)->nd_value, RNODE_MATCH2(node)->nd_args); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_MATCH2(node)->nd_recv, RNODE_MATCH2(node)->nd_value, RNODE_MATCH2(node)->nd_args); } - return rb_ary_new_from_node_args(vast, 2, RNODE_MATCH2(node)->nd_recv, RNODE_MATCH2(node)->nd_value); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_MATCH2(node)->nd_recv, RNODE_MATCH2(node)->nd_value); case NODE_MATCH3: - return rb_ary_new_from_node_args(vast, 2, RNODE_MATCH3(node)->nd_recv, RNODE_MATCH3(node)->nd_value); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_MATCH3(node)->nd_recv, RNODE_MATCH3(node)->nd_value); case NODE_STR: case NODE_XSTR: return rb_ary_new_from_args(1, rb_node_str_string_val(node)); @@ -542,7 +542,7 @@ node_children(VALUE vast, const NODE *node) case NODE_REGX: return rb_ary_new_from_args(1, rb_node_regx_string_val(node)); case NODE_ONCE: - return rb_ary_new_from_node_args(vast, 1, RNODE_ONCE(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_ONCE(node)->nd_body); case NODE_DSTR: case NODE_DXSTR: case NODE_DREGX: @@ -551,91 +551,91 @@ node_children(VALUE vast, const NODE *node) struct RNode_LIST *n = RNODE_DSTR(node)->nd_next; VALUE head = Qnil, next = Qnil; if (n) { - head = NEW_CHILD(vast, n->nd_head); - next = NEW_CHILD(vast, n->nd_next); + head = NEW_CHILD(ast_value, n->nd_head); + next = NEW_CHILD(ast_value, n->nd_next); } return rb_ary_new_from_args(3, rb_node_dstr_string_val(node), head, next); } case NODE_SYM: return rb_ary_new_from_args(1, rb_node_sym_string_val(node)); case NODE_EVSTR: - return rb_ary_new_from_node_args(vast, 1, RNODE_EVSTR(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_EVSTR(node)->nd_body); case NODE_ARGSCAT: - return rb_ary_new_from_node_args(vast, 2, RNODE_ARGSCAT(node)->nd_head, RNODE_ARGSCAT(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_ARGSCAT(node)->nd_head, RNODE_ARGSCAT(node)->nd_body); case NODE_ARGSPUSH: - return rb_ary_new_from_node_args(vast, 2, RNODE_ARGSPUSH(node)->nd_head, RNODE_ARGSPUSH(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_ARGSPUSH(node)->nd_head, RNODE_ARGSPUSH(node)->nd_body); case NODE_SPLAT: - return rb_ary_new_from_node_args(vast, 1, RNODE_SPLAT(node)->nd_head); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_SPLAT(node)->nd_head); case NODE_BLOCK_PASS: - return rb_ary_new_from_node_args(vast, 2, RNODE_BLOCK_PASS(node)->nd_head, RNODE_BLOCK_PASS(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_BLOCK_PASS(node)->nd_head, RNODE_BLOCK_PASS(node)->nd_body); case NODE_DEFN: - return rb_ary_new_from_args(2, ID2SYM(RNODE_DEFN(node)->nd_mid), NEW_CHILD(vast, RNODE_DEFN(node)->nd_defn)); + return rb_ary_new_from_args(2, ID2SYM(RNODE_DEFN(node)->nd_mid), NEW_CHILD(ast_value, RNODE_DEFN(node)->nd_defn)); case NODE_DEFS: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_DEFS(node)->nd_recv), ID2SYM(RNODE_DEFS(node)->nd_mid), NEW_CHILD(vast, RNODE_DEFS(node)->nd_defn)); + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_DEFS(node)->nd_recv), ID2SYM(RNODE_DEFS(node)->nd_mid), NEW_CHILD(ast_value, RNODE_DEFS(node)->nd_defn)); case NODE_ALIAS: - return rb_ary_new_from_node_args(vast, 2, RNODE_ALIAS(node)->nd_1st, RNODE_ALIAS(node)->nd_2nd); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_ALIAS(node)->nd_1st, RNODE_ALIAS(node)->nd_2nd); case NODE_VALIAS: return rb_ary_new_from_args(2, ID2SYM(RNODE_VALIAS(node)->nd_alias), ID2SYM(RNODE_VALIAS(node)->nd_orig)); case NODE_UNDEF: - return rb_ary_new_from_node_args(vast, 1, RNODE_UNDEF(node)->nd_undef); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_UNDEF(node)->nd_undef); case NODE_CLASS: - return rb_ary_new_from_node_args(vast, 3, RNODE_CLASS(node)->nd_cpath, RNODE_CLASS(node)->nd_super, RNODE_CLASS(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 3, RNODE_CLASS(node)->nd_cpath, RNODE_CLASS(node)->nd_super, RNODE_CLASS(node)->nd_body); case NODE_MODULE: - return rb_ary_new_from_node_args(vast, 2, RNODE_MODULE(node)->nd_cpath, RNODE_MODULE(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_MODULE(node)->nd_cpath, RNODE_MODULE(node)->nd_body); case NODE_SCLASS: - return rb_ary_new_from_node_args(vast, 2, RNODE_SCLASS(node)->nd_recv, RNODE_SCLASS(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_SCLASS(node)->nd_recv, RNODE_SCLASS(node)->nd_body); case NODE_COLON2: - return rb_ary_new_from_args(2, NEW_CHILD(vast, RNODE_COLON2(node)->nd_head), ID2SYM(RNODE_COLON2(node)->nd_mid)); + return rb_ary_new_from_args(2, NEW_CHILD(ast_value, RNODE_COLON2(node)->nd_head), ID2SYM(RNODE_COLON2(node)->nd_mid)); case NODE_COLON3: return rb_ary_new_from_args(1, ID2SYM(RNODE_COLON3(node)->nd_mid)); case NODE_DOT2: case NODE_DOT3: case NODE_FLIP2: case NODE_FLIP3: - return rb_ary_new_from_node_args(vast, 2, RNODE_DOT2(node)->nd_beg, RNODE_DOT2(node)->nd_end); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_DOT2(node)->nd_beg, RNODE_DOT2(node)->nd_end); case NODE_SELF: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_NIL: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_TRUE: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_FALSE: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_ERRINFO: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_DEFINED: - return rb_ary_new_from_node_args(vast, 1, RNODE_DEFINED(node)->nd_head); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_DEFINED(node)->nd_head); case NODE_POSTEXE: - return rb_ary_new_from_node_args(vast, 1, RNODE_POSTEXE(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_POSTEXE(node)->nd_body); case NODE_ATTRASGN: - return rb_ary_new_from_args(3, NEW_CHILD(vast, RNODE_ATTRASGN(node)->nd_recv), ID2SYM(RNODE_ATTRASGN(node)->nd_mid), NEW_CHILD(vast, RNODE_ATTRASGN(node)->nd_args)); + return rb_ary_new_from_args(3, NEW_CHILD(ast_value, RNODE_ATTRASGN(node)->nd_recv), ID2SYM(RNODE_ATTRASGN(node)->nd_mid), NEW_CHILD(ast_value, RNODE_ATTRASGN(node)->nd_args)); case NODE_LAMBDA: - return rb_ary_new_from_node_args(vast, 1, RNODE_LAMBDA(node)->nd_body); + return rb_ary_new_from_node_args(ast_value, 1, RNODE_LAMBDA(node)->nd_body); case NODE_OPT_ARG: - return rb_ary_new_from_node_args(vast, 2, RNODE_OPT_ARG(node)->nd_body, RNODE_OPT_ARG(node)->nd_next); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_OPT_ARG(node)->nd_body, RNODE_OPT_ARG(node)->nd_next); case NODE_KW_ARG: - return rb_ary_new_from_node_args(vast, 2, RNODE_KW_ARG(node)->nd_body, RNODE_KW_ARG(node)->nd_next); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_KW_ARG(node)->nd_body, RNODE_KW_ARG(node)->nd_next); case NODE_POSTARG: if (NODE_NAMED_REST_P(RNODE_POSTARG(node)->nd_1st)) { - return rb_ary_new_from_node_args(vast, 2, RNODE_POSTARG(node)->nd_1st, RNODE_POSTARG(node)->nd_2nd); + return rb_ary_new_from_node_args(ast_value, 2, RNODE_POSTARG(node)->nd_1st, RNODE_POSTARG(node)->nd_2nd); } return rb_ary_new_from_args(2, no_name_rest(), - NEW_CHILD(vast, RNODE_POSTARG(node)->nd_2nd)); + NEW_CHILD(ast_value, RNODE_POSTARG(node)->nd_2nd)); case NODE_ARGS: { struct rb_args_info *ainfo = &RNODE_ARGS(node)->nd_ainfo; return rb_ary_new_from_args(10, INT2NUM(ainfo->pre_args_num), - NEW_CHILD(vast, ainfo->pre_init), - NEW_CHILD(vast, (NODE *)ainfo->opt_args), + NEW_CHILD(ast_value, ainfo->pre_init), + NEW_CHILD(ast_value, (NODE *)ainfo->opt_args), var_name(ainfo->first_post_arg), INT2NUM(ainfo->post_args_num), - NEW_CHILD(vast, ainfo->post_init), + NEW_CHILD(ast_value, ainfo->post_init), (ainfo->rest_arg == NODE_SPECIAL_EXCESSIVE_COMMA ? ID2SYM(rb_intern("NODE_SPECIAL_EXCESSIVE_COMMA")) : var_name(ainfo->rest_arg)), - (ainfo->no_kwarg ? Qfalse : NEW_CHILD(vast, (NODE *)ainfo->kw_args)), - (ainfo->no_kwarg ? Qfalse : NEW_CHILD(vast, ainfo->kw_rest_arg)), + (ainfo->no_kwarg ? Qfalse : NEW_CHILD(ast_value, (NODE *)ainfo->kw_args)), + (ainfo->no_kwarg ? Qfalse : NEW_CHILD(ast_value, ainfo->kw_rest_arg)), var_name(ainfo->block_arg)); } case NODE_SCOPE: @@ -646,35 +646,35 @@ node_children(VALUE vast, const NODE *node) for (i = 0; i < size; i++) { rb_ary_push(locals, var_name(tbl->ids[i])); } - return rb_ary_new_from_args(3, locals, NEW_CHILD(vast, (NODE *)RNODE_SCOPE(node)->nd_args), NEW_CHILD(vast, RNODE_SCOPE(node)->nd_body)); + return rb_ary_new_from_args(3, locals, NEW_CHILD(ast_value, (NODE *)RNODE_SCOPE(node)->nd_args), NEW_CHILD(ast_value, RNODE_SCOPE(node)->nd_body)); } case NODE_ARYPTN: { - VALUE rest = rest_arg(vast, RNODE_ARYPTN(node)->rest_arg); + VALUE rest = rest_arg(ast_value, RNODE_ARYPTN(node)->rest_arg); return rb_ary_new_from_args(4, - NEW_CHILD(vast, RNODE_ARYPTN(node)->nd_pconst), - NEW_CHILD(vast, RNODE_ARYPTN(node)->pre_args), + NEW_CHILD(ast_value, RNODE_ARYPTN(node)->nd_pconst), + NEW_CHILD(ast_value, RNODE_ARYPTN(node)->pre_args), rest, - NEW_CHILD(vast, RNODE_ARYPTN(node)->post_args)); + NEW_CHILD(ast_value, RNODE_ARYPTN(node)->post_args)); } case NODE_FNDPTN: { - VALUE pre_rest = rest_arg(vast, RNODE_FNDPTN(node)->pre_rest_arg); - VALUE post_rest = rest_arg(vast, RNODE_FNDPTN(node)->post_rest_arg); + VALUE pre_rest = rest_arg(ast_value, RNODE_FNDPTN(node)->pre_rest_arg); + VALUE post_rest = rest_arg(ast_value, RNODE_FNDPTN(node)->post_rest_arg); return rb_ary_new_from_args(4, - NEW_CHILD(vast, RNODE_FNDPTN(node)->nd_pconst), + NEW_CHILD(ast_value, RNODE_FNDPTN(node)->nd_pconst), pre_rest, - NEW_CHILD(vast, RNODE_FNDPTN(node)->args), + NEW_CHILD(ast_value, RNODE_FNDPTN(node)->args), post_rest); } case NODE_HSHPTN: { VALUE kwrest = RNODE_HSHPTN(node)->nd_pkwrestarg == NODE_SPECIAL_NO_REST_KEYWORD ? ID2SYM(rb_intern("NODE_SPECIAL_NO_REST_KEYWORD")) : - NEW_CHILD(vast, RNODE_HSHPTN(node)->nd_pkwrestarg); + NEW_CHILD(ast_value, RNODE_HSHPTN(node)->nd_pkwrestarg); return rb_ary_new_from_args(3, - NEW_CHILD(vast, RNODE_HSHPTN(node)->nd_pconst), - NEW_CHILD(vast, RNODE_HSHPTN(node)->nd_pkwargs), + NEW_CHILD(ast_value, RNODE_HSHPTN(node)->nd_pconst), + NEW_CHILD(ast_value, RNODE_HSHPTN(node)->nd_pkwargs), kwrest); } case NODE_LINE: @@ -684,7 +684,7 @@ node_children(VALUE vast, const NODE *node) case NODE_ENCODING: return rb_ary_new_from_args(1, rb_node_encoding_val(node)); case NODE_ERROR: - return rb_ary_new_from_node_args(vast, 0); + return rb_ary_new_from_node_args(ast_value, 0); case NODE_ARGS_AUX: case NODE_LAST: break; @@ -699,7 +699,7 @@ ast_node_children(rb_execution_context_t *ec, VALUE self) struct ASTNodeData *data; TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data); - return node_children(data->vast, data->node); + return node_children(data->ast_value, data->node); } static VALUE @@ -749,7 +749,7 @@ ast_node_all_tokens(rb_execution_context_t *ec, VALUE self) VALUE str, loc, token, all_tokens; TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data); - ast = rb_ruby_ast_data_get(data->vast); + ast = rb_ruby_ast_data_get(data->ast_value); parser_tokens = ast->node_buffer->tokens; if (parser_tokens == NULL) { @@ -800,7 +800,7 @@ ast_node_script_lines(rb_execution_context_t *ec, VALUE self) struct ASTNodeData *data; rb_ast_t *ast; TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data); - ast = rb_ruby_ast_data_get(data->vast); + ast = rb_ruby_ast_data_get(data->ast_value); rb_parser_ary_t *ret = ast->body.script_lines; return rb_parser_build_script_lines_from(ret); } diff --git a/bootstraptest/test_ractor.rb b/bootstraptest/test_ractor.rb index 451b58e793..0390d38f9c 100644 --- a/bootstraptest/test_ractor.rb +++ b/bootstraptest/test_ractor.rb @@ -1466,6 +1466,25 @@ assert_equal '[:ok, :ok]', %q{ end } +# Ractor.select is interruptible +assert_normal_exit %q{ + trap(:INT) do + exit + end + + r = Ractor.new do + loop do + sleep 1 + end + end + + Thread.new do + sleep 0.5 + Process.kill(:INT, Process.pid) + end + Ractor.select(r) +} + # Ractor-local storage assert_equal '[nil, "b", "a"]', %q{ ans = [] diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 31bb626690..8b8f7d1e39 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4859,3 +4859,124 @@ assert_equal '["raised", "Module", "Object"]', %q{ ret += [foo(Class), foo(Class.new)] } + +# test TrueClass#=== before and after redefining TrueClass#== +assert_equal '[[true, false, false], [true, true, false], [true, :error, :error]]', %q{ + def true_eqq(x) + true === x + rescue NoMethodError + :error + end + + def test + [ + # first one is always true because rb_equal does object comparison before calling #== + true_eqq(true), + # these will use TrueClass#== + true_eqq(false), + true_eqq(:truthy), + ] + end + + results = [test] + + class TrueClass + def ==(x) + !x + end + end + + results << test + + class TrueClass + undef_method :== + end + + results << test +} unless rjit_enabled? # Not yet working on RJIT + +# test FalseClass#=== before and after redefining FalseClass#== +assert_equal '[[true, false, false], [true, false, true], [true, :error, :error]]', %q{ + def case_equal(x, y) + x === y + rescue NoMethodError + :error + end + + def test + [ + # first one is always true because rb_equal does object comparison before calling #== + case_equal(false, false), + # these will use #== + case_equal(false, true), + case_equal(false, nil), + ] + end + + results = [test] + + class FalseClass + def ==(x) + !x + end + end + + results << test + + class FalseClass + undef_method :== + end + + results << test +} unless rjit_enabled? # Not yet working on RJIT + +# test NilClass#=== before and after redefining NilClass#== +assert_equal '[[true, false, false], [true, false, true], [true, :error, :error]]', %q{ + def case_equal(x, y) + x === y + rescue NoMethodError + :error + end + + def test + [ + # first one is always true because rb_equal does object comparison before calling #== + case_equal(nil, nil), + # these will use #== + case_equal(nil, true), + case_equal(nil, false), + ] + end + + results = [test] + + class NilClass + def ==(x) + !x + end + end + + results << test + + class NilClass + undef_method :== + end + + results << test +} unless rjit_enabled? # Not yet working on RJIT + +# test struct accessors fire c_call events +assert_equal '[[:c_call, :x=], [:c_call, :x]]', %q{ + c = Struct.new(:x) + obj = c.new + + events = [] + TracePoint.new(:c_call) do + events << [_1.event, _1.method_id] + end.enable do + obj.x = 100 + obj.x + end + + events +} @@ -1018,15 +1018,6 @@ yes-test-spec: yes-test-spec-precheck $(ACTIONS_ENDGROUP) no-test-spec: -test-prism-spec: $(TEST_RUNNABLE)-test-prism-spec -yes-test-prism-spec: yes-test-spec-precheck - $(ACTIONS_GROUP) - $(gnumake_recursive)$(Q) \ - $(RUNRUBY) -r./$(arch)-fake -r$(tooldir)/lib/_tmpdir \ - $(srcdir)/spec/mspec/bin/mspec run -B $(srcdir)/spec/default.mspec -B $(srcdir)/spec/prism.mspec $(MSPECOPT) $(SPECOPTS) - $(ACTIONS_ENDGROUP) -no-test-prism-spec: - check: $(DOT_WAIT) test-spec RUNNABLE = $(LIBRUBY_RELATIVE:no=un)-runnable @@ -1620,30 +1611,8 @@ yes-test-bundled-gems-spec: yes-test-spec-precheck $(PREPARE_BUNDLED_GEMS) $(ACTIONS_ENDGROUP) no-test-bundled-gems-spec: -test-syntax-suggest-precheck: $(TEST_RUNNABLE)-test-syntax-suggest-precheck -no-test-syntax-suggest-precheck: -yes-test-syntax-suggest-precheck: main - -test-syntax-suggest-prepare: $(TEST_RUNNABLE)-test-syntax-suggest-prepare -no-test-syntax-suggest-prepare: no-test-syntax-suggest-precheck -yes-test-syntax-suggest-prepare: yes-test-syntax-suggest-precheck - $(ACTIONS_GROUP) - $(XRUBY) -C "$(srcdir)" bin/gem install --no-document \ - --install-dir .bundle --conservative "rspec:~> 3" - $(ACTIONS_ENDGROUP) -RSPECOPTS = -SYNTAX_SUGGEST_SPECS = -PREPARE_SYNTAX_SUGGEST = $(TEST_RUNNABLE)-test-syntax-suggest-prepare -test-syntax-suggest: $(TEST_RUNNABLE)-test-syntax-suggest -yes-test-syntax-suggest: $(PREPARE_SYNTAX_SUGGEST) - $(ACTIONS_GROUP) - $(XRUBY) -C $(srcdir) -Ispec/syntax_suggest:spec/lib .bundle/bin/rspec \ - --require rspec/expectations \ - --require spec_helper --require formatter_overrides --require spec_coverage \ - $(RSPECOPTS) spec/syntax_suggest/$(SYNTAX_SUGGEST_SPECS) - $(ACTIONS_ENDGROUP) -no-test-syntax-suggest: +test-syntax-suggest: check: $(DOT_WAIT) $(PREPARE_SYNTAX_SUGGEST) test-syntax-suggest @@ -1479,11 +1479,11 @@ new_child_iseq(rb_iseq_t *iseq, const NODE *const node, VALUE name, const rb_iseq_t *parent, enum rb_iseq_type type, int line_no) { rb_iseq_t *ret_iseq; - VALUE vast = rb_ruby_ast_new(node); + VALUE ast_value = rb_ruby_ast_new(node); debugs("[new_child_iseq]> ---------------------------------------\n"); int isolated_depth = ISEQ_COMPILE_DATA(iseq)->isolated_depth; - ret_iseq = rb_iseq_new_with_opt(vast, name, + ret_iseq = rb_iseq_new_with_opt(ast_value, name, rb_iseq_path(iseq), rb_iseq_realpath(iseq), line_no, parent, isolated_depth ? isolated_depth + 1 : 0, @@ -4048,6 +4048,8 @@ insn_set_specialized_instruction(rb_iseq_t *iseq, INSN *iobj, int insn_id) return COMPILE_OK; } +#define vm_ci_simple(ci) (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) + static int iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) { @@ -4059,24 +4061,43 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) INSN *niobj = (INSN *)iobj->link.next; if (IS_INSN_ID(niobj, send)) { const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(niobj, 0); - if ((vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) && vm_ci_argc(ci) == 0) { + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0) { switch (vm_ci_mid(ci)) { case idMax: case idMin: case idHash: { VALUE num = iobj->operands[0]; + int operand_len = insn_len(BIN(opt_newarray_send)) - 1; iobj->insn_id = BIN(opt_newarray_send); - iobj->operands = compile_data_calloc2(iseq, insn_len(iobj->insn_id) - 1, sizeof(VALUE)); + iobj->operands = compile_data_calloc2(iseq, operand_len, sizeof(VALUE)); iobj->operands[0] = num; iobj->operands[1] = rb_id2sym(vm_ci_mid(ci)); - iobj->operand_size = insn_len(iobj->insn_id) - 1; + iobj->operand_size = operand_len; ELEM_REMOVE(&niobj->link); return COMPILE_OK; } } } } + else if ((IS_INSN_ID(niobj, putstring) || + (IS_INSN_ID(niobj, putobject) && RB_TYPE_P(OPERAND_AT(niobj, 0), T_STRING))) && + IS_NEXT_INSN_ID(&niobj->link, send)) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT((INSN *)niobj->link.next, 0); + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 1 && vm_ci_mid(ci) == idPack) { + VALUE num = iobj->operands[0]; + int operand_len = insn_len(BIN(opt_newarray_send)) - 1; + iobj->insn_id = BIN(opt_newarray_send); + iobj->operands = compile_data_calloc2(iseq, operand_len, sizeof(VALUE)); + iobj->operands[0] = FIXNUM_INC(num, 1); + iobj->operands[1] = rb_id2sym(vm_ci_mid(ci)); + iobj->operand_size = operand_len; + ELEM_REMOVE(&iobj->link); + ELEM_REMOVE(niobj->link.next); + ELEM_INSERT_NEXT(&niobj->link, &iobj->link); + return COMPILE_OK; + } + } } if (IS_INSN_ID(iobj, send)) { @@ -4084,7 +4105,7 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 1); #define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt)) - if (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) { + if (vm_ci_simple(ci)) { switch (vm_ci_argc(ci)) { case 0: switch (vm_ci_mid(ci)) { @@ -5380,12 +5401,17 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const CHECK(COMPILE_POPPED(pre, "masgn lhs (NODE_ATTRASGN)", node)); + bool safenav_call = false; LINK_ELEMENT *insn_element = LAST_ELEMENT(pre); iobj = (INSN *)get_prev_insn((INSN *)insn_element); /* send insn */ ASSUME(iobj); - ELEM_REMOVE(LAST_ELEMENT(pre)); - ELEM_REMOVE((LINK_ELEMENT *)iobj); - pre->last = iobj->link.prev; + ELEM_REMOVE(insn_element); + if (!IS_INSN_ID(iobj, send)) { + safenav_call = true; + iobj = (INSN *)get_prev_insn(iobj); + ELEM_INSERT_NEXT(&iobj->link, insn_element); + } + (pre->last = iobj->link.prev)->next = 0; const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(iobj, 0); int argc = vm_ci_argc(ci) + 1; @@ -5404,7 +5430,9 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const return COMPILE_NG; } - ADD_ELEM(lhs, (LINK_ELEMENT *)iobj); + iobj->link.prev = lhs->last; + lhs->last->next = &iobj->link; + for (lhs->last = &iobj->link; lhs->last->next; lhs->last = lhs->last->next); if (vm_ci_flag(ci) & VM_CALL_ARGS_SPLAT) { int argc = vm_ci_argc(ci); bool dupsplat = false; @@ -5437,9 +5465,11 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const } INSERT_BEFORE_INSN1(iobj, line_no, node_id, pushtoarray, INT2FIX(1)); } - ADD_INSN(lhs, line_node, pop); - if (argc != 1) { + if (!safenav_call) { ADD_INSN(lhs, line_node, pop); + if (argc != 1) { + ADD_INSN(lhs, line_node, pop); + } } for (int i=0; i < argc; i++) { ADD_INSN(post, line_node, pop); @@ -8771,10 +8801,10 @@ compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const N scope_node.nd_body = mandatory_node(iseq, node); scope_node.nd_args = &args_node; - VALUE vast = rb_ruby_ast_new(RNODE(&scope_node)); + VALUE ast_value = rb_ruby_ast_new(RNODE(&scope_node)); ISEQ_BODY(iseq)->mandatory_only_iseq = - rb_iseq_new_with_opt(vast, rb_iseq_base_label(iseq), + rb_iseq_new_with_opt(ast_value, rb_iseq_base_label(iseq), rb_iseq_path(iseq), rb_iseq_realpath(iseq), nd_line(line_node), NULL, 0, ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option, @@ -2470,7 +2470,7 @@ float_arg(VALUE self) * * The rectangular coordinates of a complex number * are called the _real_ and _imaginary_ parts; - * see {Complex number definition}[https://en.wikipedia.org/wiki/Complex_number#Definition]. + * see {Complex number definition}[https://en.wikipedia.org/wiki/Complex_number#Definition_and_basic_operations]. * * You can create a \Complex object from rectangular coordinates with: * @@ -2495,7 +2495,7 @@ float_arg(VALUE self) * * The polar coordinates of a complex number * are called the _absolute_ and _argument_ parts; - * see {Complex polar plane}[https://en.wikipedia.org/wiki/Complex_number#Polar_complex_plane]. + * see {Complex polar plane}[https://en.wikipedia.org/wiki/Complex_number#Polar_form]. * * In this class, the argument part * in expressed {radians}[https://en.wikipedia.org/wiki/Radian] diff --git a/configure.ac b/configure.ac index e9a452ebee..a7fb1ab56d 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,7 @@ m4_include([tool/m4/ruby_append_option.m4])dnl m4_include([tool/m4/ruby_append_options.m4])dnl m4_include([tool/m4/ruby_check_builtin_func.m4])dnl m4_include([tool/m4/ruby_check_builtin_setjmp.m4])dnl +m4_include([tool/m4/ruby_check_header.m4])dnl m4_include([tool/m4/ruby_check_printf_prefix.m4])dnl m4_include([tool/m4/ruby_check_setjmp.m4])dnl m4_include([tool/m4/ruby_check_signedness.m4])dnl @@ -226,15 +227,23 @@ AS_CASE(["/${rb_CC} "], [*clang*], [ # Ditto for LLVM. Note however that llvm-as is a LLVM-IR to LLVM bitcode # assembler that does not target your machine native binary. + + # Xcode has its own version tools that may be incompatible with + # genuine LLVM tools, use the tools in the same directory. + + AS_IF([$rb_CC -E -dM -xc - < /dev/null | grep -F __apple_build_version__ > /dev/null], + [llvm_prefix=], [llvm_prefix=llvm-]) + # AC_PREPROC_IFELSE cannot be used before AC_USE_SYSTEM_EXTENSIONS + RUBY_CHECK_PROG_FOR_CC([LD], [s/clang/ld/]) # ... maybe try lld ? - RUBY_CHECK_PROG_FOR_CC([AR], [s/clang/llvm-ar/]) -# RUBY_CHECK_PROG_FOR_CC([AS], [s/clang/llvm-as/]) + RUBY_CHECK_PROG_FOR_CC([AR], [s/clang/${llvm_prefix}ar/]) +# RUBY_CHECK_PROG_FOR_CC([AS], [s/clang/${llvm_prefix}as/]) RUBY_CHECK_PROG_FOR_CC([CXX], [s/clang/clang++/]) - RUBY_CHECK_PROG_FOR_CC([NM], [s/clang/llvm-nm/]) - RUBY_CHECK_PROG_FOR_CC([OBJCOPY], [s/clang/llvm-objcopy/]) - RUBY_CHECK_PROG_FOR_CC([OBJDUMP], [s/clang/llvm-objdump/]) - RUBY_CHECK_PROG_FOR_CC([RANLIB], [s/clang/llvm-ranlib/]) - RUBY_CHECK_PROG_FOR_CC([STRIP], [s/clang/llvm-strip/]) + RUBY_CHECK_PROG_FOR_CC([NM], [s/clang/${llvm_prefix}nm/]) + RUBY_CHECK_PROG_FOR_CC([OBJCOPY], [s/clang/${llvm_prefix}objcopy/]) + RUBY_CHECK_PROG_FOR_CC([OBJDUMP], [s/clang/${llvm_prefix}objdump/]) + RUBY_CHECK_PROG_FOR_CC([RANLIB], [s/clang/${llvm_prefix}ranlib/]) + RUBY_CHECK_PROG_FOR_CC([STRIP], [s/clang/${llvm_prefix}strip/]) ]) AS_UNSET(rb_CC) AS_UNSET(rb_dummy) @@ -245,12 +254,6 @@ AS_CASE(["${build_os}"], ], [aix*], [ AC_PATH_TOOL([NM], [nm], [/usr/ccs/bin/nm], [/usr/ccs/bin:$PATH]) -], -[darwin*], [ - # For Apple clang version 14.0.3 (clang-1403.0.22.14.1) - ac_cv_prog_ac_ct_AR=`$CC -print-prog-name=ar` - ac_cv_prog_ac_ct_LD=`$CC -print-prog-name=ld` - ac_cv_prog_ac_ct_NM=`$CC -print-prog-name=nm` ]) AS_CASE(["${target_os}"], [cygwin*|msys*|mingw*|darwin*], [ @@ -988,7 +991,6 @@ AC_SUBST(incflags, "$INCFLAGS") test -z "${ac_env_CFLAGS_set}" -a -n "${cflags+set}" && eval CFLAGS="\"$cflags $ARCH_FLAG\"" test -z "${ac_env_CXXFLAGS_set}" -a -n "${cxxflags+set}" && eval CXXFLAGS="\"$cxxflags $ARCH_FLAG\"" -} AC_CACHE_CHECK([whether compiler has statement and declarations in expressions], rb_cv_have_stmt_and_decl_in_expr, @@ -998,6 +1000,7 @@ AC_CACHE_CHECK([whether compiler has statement and declarations in expressions], AS_IF([test "$rb_cv_have_stmt_and_decl_in_expr" = yes], [ AC_DEFINE(HAVE_STMT_AND_DECL_IN_EXPR) ]) +} [begin]_group "header and library section" && { AC_ARG_WITH(winnt-ver, @@ -1375,7 +1378,7 @@ AS_CASE("$target_cpu", [x64|x86_64|i[3-6]86*], [ RUBY_UNIVERSAL_CHECK_HEADER([x86_64, i386], x86intrin.h) AS_IF([test "x$with_gmp" != xno], - [AC_CHECK_HEADERS(gmp.h) + [RUBY_CHECK_HEADER(gmp.h) AS_IF([test "x$ac_cv_header_gmp_h" != xno], AC_SEARCH_LIBS([__gmpz_init], [gmp], [AC_DEFINE(HAVE_LIBGMP, 1)]))]) @@ -1385,6 +1388,8 @@ AC_ARG_WITH([jemalloc], [with_jemalloc=$withval], [with_jemalloc=no]) AS_IF([test "x$with_jemalloc" != xno],[ # find jemalloc header first + save_CPPFLAGS="${CPPFLAGS}" + CPPFLAGS="${INCFLAGS} ${CPPFLAGS}" malloc_header= AC_CHECK_HEADER(jemalloc/jemalloc.h, [malloc_header=jemalloc/jemalloc.h], [ AC_CHECK_HEADER(jemalloc.h, [malloc_header=jemalloc.h]) @@ -1416,6 +1421,8 @@ AS_IF([test "x$with_jemalloc" != xno],[ done done ]) + CPPFLAGS="${save_CPPFLAGS}" + unset save_CPPFLAGS with_jemalloc=${rb_cv_jemalloc_library} AS_CASE(["$with_jemalloc"], [no], diff --git a/defs/id.def b/defs/id.def index 2ddde7be70..71baa4f968 100644 --- a/defs/id.def +++ b/defs/id.def @@ -59,6 +59,7 @@ firstline, predefined = __LINE__+1, %[\ name nil path + pack _ UScore diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 96cee40cb4..38c78e3cca 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -185,7 +185,7 @@ Using the address sanitizer (ASAN) is a great way to detect memory issues. It ca ``` shell ./autogen.sh mkdir build && cd build -../configure CC=clang cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like +../configure CC=clang-18 cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like make ``` The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. To run Ruby's test suite under ASAN, issue the following command. Note that this will take quite a long time (over two hours on my laptop); the `RUBY_TEST_TIMEOUT_SCALE` and `SYNTAX_SUGEST_TIMEOUT` variables are required to make sure tests don't spuriously fail with timeouts when in fact they're just slow. @@ -198,7 +198,7 @@ Please note, however, the following caveats! * ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch (and the whole test suite passes only as of commit [9d0a5148ae062a0481a4a18fbeb9cfd01dc10428](https://bugs.ruby-lang.org/projects/ruby-master/repository/git/revisions/9d0a5148ae062a0481a4a18fbeb9cfd01dc10428)) * Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now (with the `-DUSE_MN_THREADS=0` configure argument). -* Currently, ASAN will only work correctly when using a recent head build of LLVM/Clang - it requires [this bugfix](https://github.com/llvm/llvm-project/pull/75290) related to multithreaded `fork`, which is not yet in any released version. See [here](https://llvm.org/docs/CMake.html) for instructions on how to build LLVM/Clang from source (note you will need at least the `clang` and `compiler-rt` projects enabled). Then, you will need to replace `CC=clang` in the instructions with an explicit path to your built Clang binary. +* ASAN will only work when using Clang version 18 or later - it requires [this bugfix](https://github.com/llvm/llvm-project/pull/75290) related to multithreaded `fork`. * ASAN has only been tested so far with Clang on Linux. It may or may not work with other compilers or on other platforms - please file an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if you run into problems with such configurations (or, to report that they actually work properly!) * In particular, although I have not yet tried it, I have reason to believe ASAN will _not_ work properly on macOS yet - the fix for the multithreaded fork issue was actually reverted for macOS (see [here](https://github.com/llvm/llvm-project/commit/2a03854e4ce9bb1bcd79a211063bc63c4657f92c)). Please open an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if this is a problem for you. diff --git a/doc/encodings.rdoc b/doc/encodings.rdoc index 97c0d22616..d85099cdbc 100644 --- a/doc/encodings.rdoc +++ b/doc/encodings.rdoc @@ -419,7 +419,7 @@ These keyword-value pairs specify encoding options: hash = {"\u3042" => 'xyzzy'} hash.default = 'XYZZY' - s.encode('ASCII', fallback: h) # => "xyzzyfooXYZZY" + s.encode('ASCII', fallback: hash) # => "xyzzyfooXYZZY" def (fallback = "U+%.4X").escape(x) self % x.unpack("U") diff --git a/doc/syntax/pattern_matching.rdoc b/doc/syntax/pattern_matching.rdoc index e49c09a1f8..6a30380f46 100644 --- a/doc/syntax/pattern_matching.rdoc +++ b/doc/syntax/pattern_matching.rdoc @@ -422,7 +422,8 @@ These core and library classes implement deconstruction: == Guard clauses -+if+ can be used to attach an additional condition (guard clause) when the pattern matches. This condition may use bound variables: ++if+ can be used to attach an additional condition (guard clause) when the pattern matches in +case+/+in+ expressions. +This condition may use bound variables: case [1, 2] in a, b if b == a*2 @@ -450,6 +451,11 @@ These core and library classes implement deconstruction: end #=> "matched" +Note that <code>=></code> and +in+ operator can not have a guard clause. +The following examples is parsed as a standalone expression with modifier +if+. + + [1, 2] in a, b if b == a*2 + == Appendix A. Pattern syntax Approximate syntax is: diff --git a/enc/Makefile.in b/enc/Makefile.in index 6920bc9520..ce93fdd22d 100644 --- a/enc/Makefile.in +++ b/enc/Makefile.in @@ -52,7 +52,7 @@ optflags = @optflags@ debugflags = @debugflags@ warnflags = @warnflags@ CCDLFLAGS = @CCDLFLAGS@ -INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(top_srcdir) +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(top_srcdir) @incflags@ DEFS = @DEFS@ CPPFLAGS = @CPPFLAGS@ -DONIG_ENC_REGISTER=rb_enc_register LDFLAGS = @LDFLAGS@ @@ -870,134 +870,151 @@ ary_inject_op(VALUE ary, VALUE init, VALUE op) /* * call-seq: - * inject(symbol) -> object - * inject(initial_operand, symbol) -> object - * inject {|memo, operand| ... } -> object - * inject(initial_operand) {|memo, operand| ... } -> object - * - * Returns an object formed from operands via either: + * inject(symbol) -> object + * inject(initial_value, symbol) -> object + * inject {|memo, value| ... } -> object + * inject(initial_value) {|memo, value| ... } -> object * - * - A method named by +symbol+. - * - A block to which each operand is passed. - * - * With method-name argument +symbol+, - * combines operands using the method: - * - * # Sum, without initial_operand. - * (1..4).inject(:+) # => 10 - * # Sum, with initial_operand. - * (1..4).inject(10, :+) # => 20 + * Returns the result of applying a reducer to an initial value and + * the first element of the Enumerable. It then takes the result and applies the + * function to it and the second element of the collection, and so on. The + * return value is the result returned by the final call to the function. * - * With a block, passes each operand to the block: - * - * # Sum of squares, without initial_operand. - * (1..4).inject {|sum, n| sum + n*n } # => 30 - * # Sum of squares, with initial_operand. - * (1..4).inject(2) {|sum, n| sum + n*n } # => 32 + * You can think of * - * <b>Operands</b> + * [ a, b, c, d ].inject(i) { |r, v| fn(r, v) } * - * If argument +initial_operand+ is not given, - * the operands for +inject+ are simply the elements of +self+. - * Example calls and their operands: + * as being * - * - <tt>(1..4).inject(:+)</tt>:: <tt>[1, 2, 3, 4]</tt>. - * - <tt>(1...4).inject(:+)</tt>:: <tt>[1, 2, 3]</tt>. - * - <tt>('a'..'d').inject(:+)</tt>:: <tt>['a', 'b', 'c', 'd']</tt>. - * - <tt>('a'...'d').inject(:+)</tt>:: <tt>['a', 'b', 'c']</tt>. + * fn(fn(fn(fn(i, a), b), c), d) * - * Examples with first operand (which is <tt>self.first</tt>) of various types: + * In a way the +inject+ function _injects_ the function + * between the elements of the enumerable. * - * # Integer. - * (1..4).inject(:+) # => 10 - * # Float. - * [1.0, 2, 3, 4].inject(:+) # => 10.0 - * # Character. - * ('a'..'d').inject(:+) # => "abcd" - * # Complex. - * [Complex(1, 2), 3, 4].inject(:+) # => (8+2i) + * +inject+ is aliased as +reduce+. You use it when you want to + * _reduce_ a collection to a single value. * - * If argument +initial_operand+ is given, - * the operands for +inject+ are that value plus the elements of +self+. - * Example calls their operands: + * <b>The Calling Sequences</b> * - * - <tt>(1..4).inject(10, :+)</tt>:: <tt>[10, 1, 2, 3, 4]</tt>. - * - <tt>(1...4).inject(10, :+)</tt>:: <tt>[10, 1, 2, 3]</tt>. - * - <tt>('a'..'d').inject('e', :+)</tt>:: <tt>['e', 'a', 'b', 'c', 'd']</tt>. - * - <tt>('a'...'d').inject('e', :+)</tt>:: <tt>['e', 'a', 'b', 'c']</tt>. + * Let's start with the most verbose: * - * Examples with +initial_operand+ of various types: + * enum.inject(initial_value) do |result, next_value| + * # do something with +result+ and +next_value+ + * # the value returned by the block becomes the + * # value passed in to the next iteration + * # as +result+ + * end * - * # Integer. - * (1..4).inject(2, :+) # => 12 - * # Float. - * (1..4).inject(2.0, :+) # => 12.0 - * # String. - * ('a'..'d').inject('foo', :+) # => "fooabcd" - * # Array. - * %w[a b c].inject(['x'], :push) # => ["x", "a", "b", "c"] - * # Complex. - * (1..4).inject(Complex(2, 2), :+) # => (12+2i) + * For example: * - * <b>Combination by Given \Method</b> + * product = [ 2, 3, 4 ].inject(1) do |result, next_value| + * result * next_value + * end + * product #=> 24 * - * If the method-name argument +symbol+ is given, - * the operands are combined by that method: + * When this runs, the block is first called with +1+ (the initial value) and + * +2+ (the first element of the array). The block returns <tt>1*2</tt>, so on + * the next iteration the block is called with +2+ (the previous result) and + * +3+. The block returns +6+, and is called one last time with +6+ and +4+. + * The result of the block, +24+ becomes the value returned by +inject+. This + * code returns the product of the elements in the enumerable. * - * - The first and second operands are combined. - * - That result is combined with the third operand. - * - That result is combined with the fourth operand. - * - And so on. + * <b>First Shortcut: Default Initial value</b> * - * The return value from +inject+ is the result of the last combination. + * In the case of the previous example, the initial value, +1+, wasn't really + * necessary: the calculation of the product of a list of numbers is self-contained. * - * This call to +inject+ computes the sum of the operands: + * In these circumstances, you can omit the +initial_value+ parameter. +inject+ + * will then initially call the block with the first element of the collection + * as the +result+ parameter and the second element as the +next_value+. * - * (1..4).inject(:+) # => 10 + * [ 2, 3, 4 ].inject do |result, next_value| + * result * next_value + * end * - * Examples with various methods: + * This shortcut is convenient, but can only be used when the block produces a result + * which can be passed back to it as a first parameter. * - * # Integer addition. - * (1..4).inject(:+) # => 10 - * # Integer multiplication. - * (1..4).inject(:*) # => 24 - * # Character range concatenation. - * ('a'..'d').inject('', :+) # => "abcd" - * # String array concatenation. - * %w[foo bar baz].inject('', :+) # => "foobarbaz" - * # Hash update. - * h = [{foo: 0, bar: 1}, {baz: 2}, {bat: 3}].inject(:update) - * h # => {:foo=>0, :bar=>1, :baz=>2, :bat=>3} - * # Hash conversion to nested arrays. - * h = {foo: 0, bar: 1}.inject([], :push) - * h # => [[:foo, 0], [:bar, 1]] + * Here's an example where that's not the case: it returns a hash where the keys are words + * and the values are the number of occurrences of that word in the enumerable. * - * <b>Combination by Given Block</b> + * freqs = File.read("README.md") + * .scan(/\w{2,}/) + * .reduce(Hash.new(0)) do |counts, word| + * counts[word] += 1 + * counts + * end + * freqs #=> {"Actions"=>4, + * "Status"=>5, + * "MinGW"=>3, + * "https"=>27, + * "github"=>10, + * "com"=>15, ... * - * If a block is given, the operands are passed to the block: + * Note that the last line of the block is just the word +counts+. This ensures the + * return value of the block is the result that's being calculated. * - * - The first call passes the first and second operands. - * - The second call passes the result of the first call, - * along with the third operand. - * - The third call passes the result of the second call, - * along with the fourth operand. - * - And so on. + * <b>Second Shortcut: a Reducer function</b> * - * The return value from +inject+ is the return value from the last block call. - * - * This call to +inject+ gives a block - * that writes the memo and element, and also sums the elements: + * A <i>reducer function</i> is a function that takes a partial result and the next value, + * returning the next partial result. The block that is given to +inject+ is a reducer. * - * (1..4).inject do |memo, element| - * p "Memo: #{memo}; element: #{element}" - * memo + element - * end # => 10 + * You can also write a reducer as a function and pass the name of that function + * (as a symbol) to +inject+. However, for this to work, the function * - * Output: + * 1. Must be defined on the type of the result value + * 2. Must accept a single parameter, the next value in the collection, and + * 3. Must return an updated result which will also implement the function. + * + * Here's an example that adds elements to a string. The two calls invoke the functions + * String#concat and String#+ on the result so far, passing it the next value. + * + * s = [ "cat", " ", "dog" ].inject("", :concat) + * s #=> "cat dog" + * s = [ "cat", " ", "dog" ].inject("The result is:", :+) + * s #=> "The result is: cat dog" + * + * Here's a more complex example when the result object maintains + * state of a different type to the enumerable elements. + * + * class Turtle + * + * def initialize + * @x = @y = 0 + * end + * + * def move(dir) + * case dir + * when "n" then @y += 1 + * when "s" then @y -= 1 + * when "e" then @x += 1 + * when "w" then @x -= 1 + * end + * self + * end + * end + * + * position = "nnneesw".chars.reduce(Turtle.new, :move) + * position #=>> #<Turtle:0x00000001052f4698 @y=2, @x=1> + * + * <b>Third Shortcut: Reducer With no Initial Value</b> + * + * If your reducer returns a value that it can accept as a parameter, then you + * don't have to pass in an initial value. Here <tt>:*</tt> is the name of the + * _times_ function: + * + * product = [ 2, 3, 4 ].inject(:*) + * product # => 24 + * + * String concatenation again: + * + * s = [ "cat", " ", "dog" ].inject(:+) + * s #=> "cat dog" + * + * And an example that converts a hash to an array of two-element subarrays. * - * "Memo: 1; element: 2" - * "Memo: 3; element: 3" - * "Memo: 6; element: 4" + * nested = {foo: 0, bar: 1}.inject([], :push) + * nested # => [[:foo, 0], [:bar, 1]] * * */ @@ -386,18 +386,28 @@ warn_vsprintf(rb_encoding *enc, const char *file, int line, const char *fmt, va_ return rb_str_cat2(str, "\n"); } -#define with_warn_vsprintf(file, line, fmt) \ +#define with_warn_vsprintf(enc, file, line, fmt) \ VALUE str; \ va_list args; \ va_start(args, fmt); \ - str = warn_vsprintf(NULL, file, line, fmt, args); \ + str = warn_vsprintf(enc, file, line, fmt, args); \ va_end(args); void rb_compile_warn(const char *file, int line, const char *fmt, ...) { if (!NIL_P(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { + rb_write_warning_str(str); + } + } +} + +void +rb_enc_compile_warn(rb_encoding *enc, const char *file, int line, const char *fmt, ...) +{ + if (!NIL_P(ruby_verbose)) { + with_warn_vsprintf(enc, file, line, fmt) { rb_write_warning_str(str); } } @@ -408,7 +418,18 @@ void rb_compile_warning(const char *file, int line, const char *fmt, ...) { if (RTEST(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { + rb_write_warning_str(str); + } + } +} + +/* rb_enc_compile_warning() reports only in verbose mode */ +void +rb_enc_compile_warning(rb_encoding *enc, const char *file, int line, const char *fmt, ...) +{ + if (RTEST(ruby_verbose)) { + with_warn_vsprintf(enc, file, line, fmt) { rb_write_warning_str(str); } } @@ -418,7 +439,7 @@ void rb_category_compile_warn(rb_warning_category_t category, const char *file, int line, const char *fmt, ...) { if (!NIL_P(ruby_verbose)) { - with_warn_vsprintf(file, line, fmt) { + with_warn_vsprintf(NULL, file, line, fmt) { rb_warn_category(str, rb_warning_category_to_name(category)); } } diff --git a/ext/openssl/lib/openssl/buffering.rb b/ext/openssl/lib/openssl/buffering.rb index 68aa7bc970..216c51e3be 100644 --- a/ext/openssl/lib/openssl/buffering.rb +++ b/ext/openssl/lib/openssl/buffering.rb @@ -349,13 +349,18 @@ module OpenSSL::Buffering @wbuffer << s @wbuffer.force_encoding(Encoding::BINARY) @sync ||= false - if @sync or @wbuffer.size > BLOCK_SIZE - until @wbuffer.empty? - begin - nwrote = syswrite(@wbuffer) - rescue Errno::EAGAIN - retry + buffer_size = @wbuffer.size + if @sync or buffer_size > BLOCK_SIZE + nwrote = 0 + begin + while nwrote < buffer_size do + begin + nwrote += syswrite(@wbuffer[nwrote, buffer_size - nwrote]) + rescue Errno::EAGAIN + retry + end end + ensure @wbuffer[0, nwrote] = "" end end diff --git a/ext/openssl/lib/openssl/ssl.rb b/ext/openssl/lib/openssl/ssl.rb index 75a74a3f51..d28bf1a374 100644 --- a/ext/openssl/lib/openssl/ssl.rb +++ b/ext/openssl/lib/openssl/ssl.rb @@ -459,6 +459,32 @@ ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== nil end + # Close the stream for reading. + # This method is ignored by OpenSSL as there is no reasonable way to + # implement it, but exists for compatibility with IO. + def close_read + # Unsupported and ignored. + # Just don't read any more. + end + + # Closes the stream for writing. The behavior of this method depends on + # the version of OpenSSL and the TLS protocol in use. + # + # - Sends a 'close_notify' alert to the peer. + # - Does not wait for the peer's 'close_notify' alert in response. + # + # In TLS 1.2 and earlier: + # - On receipt of a 'close_notify' alert, responds with a 'close_notify' + # alert of its own and close down the connection immediately, + # discarding any pending writes. + # + # Therefore, on TLS 1.2, this method will cause the connection to be + # completely shut down. On TLS 1.3, the connection will remain open for + # reading only. + def close_write + stop + end + private def using_anon_cipher? diff --git a/ext/openssl/ossl_asn1.c b/ext/openssl/ossl_asn1.c index 71c452c88a..0533342077 100644 --- a/ext/openssl/ossl_asn1.c +++ b/ext/openssl/ossl_asn1.c @@ -1163,9 +1163,12 @@ ossl_asn1prim_to_der(VALUE self) rb_jump_tag(state); } p0 = p1 = (unsigned char *)RSTRING_PTR(str); - i2d_ASN1_TYPE(asn1, &p0); + if (i2d_ASN1_TYPE(asn1, &p0) < 0) { + ASN1_TYPE_free(asn1); + ossl_raise(eASN1Error, "i2d_ASN1_TYPE"); + } ASN1_TYPE_free(asn1); - assert(p0 - p1 == alllen); + ossl_str_adjust(str, p0); /* Strip header since to_der_internal() wants only the payload */ j = ASN1_get_object((const unsigned char **)&p1, &bodylen, &tag, &tc, alllen); diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c index 110610e1f9..6f74c925c0 100644 --- a/ext/openssl/ossl_cipher.c +++ b/ext/openssl/ossl_cipher.c @@ -386,11 +386,23 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self) in = (unsigned char *)RSTRING_PTR(data); in_len = RSTRING_LEN(data); GetCipher(self, ctx); - out_len = in_len+EVP_CIPHER_CTX_block_size(ctx); - if (out_len <= 0) { + + /* + * As of OpenSSL 3.2, there is no reliable way to determine the required + * output buffer size for arbitrary cipher modes. + * https://github.com/openssl/openssl/issues/22628 + * + * in_len+block_size is usually sufficient, but AES key wrap with padding + * ciphers require in_len+15 even though they have a block size of 8 bytes. + * + * Using EVP_MAX_BLOCK_LENGTH (32) as a safe upper bound for ciphers + * currently implemented in OpenSSL, but this can change in the future. + */ + if (in_len > LONG_MAX - EVP_MAX_BLOCK_LENGTH) { ossl_raise(rb_eRangeError, "data too big to make output buffer: %ld bytes", in_len); } + out_len = in_len + EVP_MAX_BLOCK_LENGTH; if (NIL_P(str)) { str = rb_str_new(0, out_len); @@ -401,7 +413,7 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self) if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len)) ossl_raise(eCipherError, NULL); - assert(out_len < RSTRING_LEN(str)); + assert(out_len <= RSTRING_LEN(str)); rb_str_set_len(str, out_len); return str; @@ -442,8 +454,8 @@ ossl_cipher_final(VALUE self) * call-seq: * cipher.name -> string * - * Returns the name of the cipher which may differ slightly from the original - * name provided. + * Returns the short name of the cipher which may differ slightly from the + * original name provided. */ static VALUE ossl_cipher_name(VALUE self) diff --git a/ext/openssl/ossl_digest.c b/ext/openssl/ossl_digest.c index 16aeeb8106..1ae26a2355 100644 --- a/ext/openssl/ossl_digest.c +++ b/ext/openssl/ossl_digest.c @@ -103,7 +103,8 @@ VALUE ossl_digest_update(VALUE, VALUE); * Digest.new(string [, data]) -> Digest * * Creates a Digest instance based on _string_, which is either the ln - * (long name) or sn (short name) of a supported digest algorithm. + * (long name) or sn (short name) of a supported digest algorithm. A list of + * supported algorithms can be obtained by calling OpenSSL::Digest.digests. * * If _data_ (a String) is given, it is used as the initial input to the * Digest instance, i.e. @@ -162,6 +163,32 @@ ossl_digest_copy(VALUE self, VALUE other) return self; } +static void +add_digest_name_to_ary(const OBJ_NAME *name, void *arg) +{ + VALUE ary = (VALUE)arg; + rb_ary_push(ary, rb_str_new2(name->name)); +} + +/* + * call-seq: + * OpenSSL::Digest.digests -> array[string...] + * + * Returns the names of all available digests in an array. + */ +static VALUE +ossl_s_digests(VALUE self) +{ + VALUE ary; + + ary = rb_ary_new(); + OBJ_NAME_do_all_sorted(OBJ_NAME_TYPE_MD_METH, + add_digest_name_to_ary, + (void*)ary); + + return ary; +} + /* * call-seq: * digest.reset -> self @@ -245,7 +272,8 @@ ossl_digest_finish(int argc, VALUE *argv, VALUE self) * call-seq: * digest.name -> string * - * Returns the sn of this Digest algorithm. + * Returns the short name of this Digest algorithm which may differ slightly + * from the original name provided. * * === Example * digest = OpenSSL::Digest.new('SHA512') @@ -412,6 +440,7 @@ Init_ossl_digest(void) rb_define_alloc_func(cDigest, ossl_digest_alloc); + rb_define_module_function(cDigest, "digests", ossl_s_digests, 0); rb_define_method(cDigest, "initialize", ossl_digest_initialize, -1); rb_define_method(cDigest, "initialize_copy", ossl_digest_copy, 1); rb_define_method(cDigest, "reset", ossl_digest_reset, 0); diff --git a/ext/openssl/ossl_pkcs7.c b/ext/openssl/ossl_pkcs7.c index 78dcbd667a..446df4c075 100644 --- a/ext/openssl/ossl_pkcs7.c +++ b/ext/openssl/ossl_pkcs7.c @@ -165,7 +165,11 @@ ossl_pkcs7_s_read_smime(VALUE klass, VALUE arg) out = NULL; pkcs7 = SMIME_read_PKCS7(in, &out); BIO_free(in); - if(!pkcs7) ossl_raise(ePKCS7Error, NULL); + if (!pkcs7) + ossl_raise(ePKCS7Error, "Could not parse the PKCS7"); + if (!pkcs7->d.ptr) + ossl_raise(ePKCS7Error, "No content in PKCS7"); + data = out ? ossl_membio2str(out) : Qnil; SetPKCS7(ret, pkcs7); ossl_pkcs7_set_data(ret, data); @@ -346,6 +350,8 @@ ossl_pkcs7_initialize(int argc, VALUE *argv, VALUE self) BIO_free(in); if (!p7) ossl_raise(rb_eArgError, "Could not parse the PKCS7"); + if (!p7->d.ptr) + ossl_raise(rb_eArgError, "No content in PKCS7"); RTYPEDDATA_DATA(self) = p7; PKCS7_free(p7_orig); @@ -842,6 +848,25 @@ ossl_pkcs7_to_der(VALUE self) } static VALUE +ossl_pkcs7_to_text(VALUE self) +{ + PKCS7 *pkcs7; + BIO *out; + VALUE str; + + GetPKCS7(self, pkcs7); + if(!(out = BIO_new(BIO_s_mem()))) + ossl_raise(ePKCS7Error, NULL); + if(!PKCS7_print_ctx(out, pkcs7, 0, NULL)) { + BIO_free(out); + ossl_raise(ePKCS7Error, NULL); + } + str = ossl_membio2str(out); + + return str; +} + +static VALUE ossl_pkcs7_to_pem(VALUE self) { PKCS7 *pkcs7; @@ -1050,6 +1075,7 @@ Init_ossl_pkcs7(void) rb_define_method(cPKCS7, "to_pem", ossl_pkcs7_to_pem, 0); rb_define_alias(cPKCS7, "to_s", "to_pem"); rb_define_method(cPKCS7, "to_der", ossl_pkcs7_to_der, 0); + rb_define_method(cPKCS7, "to_text", ossl_pkcs7_to_text, 0); cPKCS7Signer = rb_define_class_under(cPKCS7, "SignerInfo", rb_cObject); rb_define_const(cPKCS7, "Signer", cPKCS7Signer); diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c index 9f374b65ff..d68c64d5fb 100644 --- a/ext/openssl/ossl_ssl.c +++ b/ext/openssl/ossl_ssl.c @@ -1958,9 +1958,11 @@ ossl_ssl_read_internal(int argc, VALUE *argv, VALUE self, int nonblock) else rb_str_modify_expand(str, ilen - RSTRING_LEN(str)); } - rb_str_set_len(str, 0); - if (ilen == 0) - return str; + + if (ilen == 0) { + rb_str_set_len(str, 0); + return str; + } VALUE io = rb_attr_get(self, id_i_io); diff --git a/ext/openssl/ossl_ts.c b/ext/openssl/ossl_ts.c index f698bdc7ff..b18a86aad9 100644 --- a/ext/openssl/ossl_ts.c +++ b/ext/openssl/ossl_ts.c @@ -504,6 +504,25 @@ ossl_ts_req_to_der(VALUE self) } static VALUE +ossl_ts_req_to_text(VALUE self) +{ + TS_REQ *req; + BIO *out; + + GetTSRequest(self, req); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_REQ_print_bio(out, req)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + +static VALUE ossl_ts_resp_alloc(VALUE klass) { TS_RESP *resp; @@ -757,6 +776,25 @@ ossl_ts_resp_to_der(VALUE self) return asn1_to_der((void *)resp, (int (*)(void *, unsigned char **))i2d_TS_RESP); } +static VALUE +ossl_ts_resp_to_text(VALUE self) +{ + TS_RESP *resp; + BIO *out; + + GetTSResponse(self, resp); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_RESP_print_bio(out, resp)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + /* * Verifies a timestamp token by checking the signature, validating the * certificate chain implied by tsa_certificate and by checking conformance to @@ -1073,6 +1111,25 @@ ossl_ts_token_info_to_der(VALUE self) return asn1_to_der((void *)info, (int (*)(void *, unsigned char **))i2d_TS_TST_INFO); } +static VALUE +ossl_ts_token_info_to_text(VALUE self) +{ + TS_TST_INFO *info; + BIO *out; + + GetTSTokenInfo(self, info); + + out = BIO_new(BIO_s_mem()); + if (!out) ossl_raise(eTimestampError, NULL); + + if (!TS_TST_INFO_print_bio(out, info)) { + BIO_free(out); + ossl_raise(eTimestampError, NULL); + } + + return ossl_membio2str(out); +} + static ASN1_INTEGER * ossl_tsfac_serial_cb(struct TS_resp_ctx *ctx, void *data) { @@ -1356,6 +1413,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampResponse, "token_info", ossl_ts_resp_get_token_info, 0); rb_define_method(cTimestampResponse, "tsa_certificate", ossl_ts_resp_get_tsa_certificate, 0); rb_define_method(cTimestampResponse, "to_der", ossl_ts_resp_to_der, 0); + rb_define_method(cTimestampResponse, "to_text", ossl_ts_resp_to_text, 0); rb_define_method(cTimestampResponse, "verify", ossl_ts_resp_verify, -1); /* Document-class: OpenSSL::Timestamp::TokenInfo @@ -1374,6 +1432,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampTokenInfo, "ordering", ossl_ts_token_info_get_ordering, 0); rb_define_method(cTimestampTokenInfo, "nonce", ossl_ts_token_info_get_nonce, 0); rb_define_method(cTimestampTokenInfo, "to_der", ossl_ts_token_info_to_der, 0); + rb_define_method(cTimestampTokenInfo, "to_text", ossl_ts_token_info_to_text, 0); /* Document-class: OpenSSL::Timestamp::Request * Allows to create timestamp requests or parse existing ones. A Request is @@ -1399,6 +1458,7 @@ Init_ossl_ts(void) rb_define_method(cTimestampRequest, "cert_requested=", ossl_ts_req_set_cert_requested, 1); rb_define_method(cTimestampRequest, "cert_requested?", ossl_ts_req_get_cert_requested, 0); rb_define_method(cTimestampRequest, "to_der", ossl_ts_req_to_der, 0); + rb_define_method(cTimestampRequest, "to_text", ossl_ts_req_to_text, 0); /* * Indicates a successful response. Equal to +0+. diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl index 08a9c4860b..fc98c067b8 100644 --- a/ext/ripper/ripper_init.c.tmpl +++ b/ext/ripper/ripper_init.c.tmpl @@ -78,17 +78,18 @@ static const rb_data_type_t parser_data_type = { 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; -static VALUE +static rb_parser_string_t * ripper_lex_get_generic(struct parser_params *p, rb_parser_input_data input, int line_count) { VALUE src = (VALUE)input; VALUE line = rb_funcallv_public(src, id_gets, 0, 0); - if (!NIL_P(line) && !RB_TYPE_P(line, T_STRING)) { + if (NIL_P(line)) return 0; + if (!RB_TYPE_P(line, T_STRING)) { rb_raise(rb_eTypeError, "gets returned %"PRIsVALUE" (expected String or nil)", rb_obj_class(line)); } - return line; + return rb_str_to_parser_string(p, line); } void @@ -104,17 +105,19 @@ ripper_compile_error(struct parser_params *p, const char *fmt, ...) ripper_error(p); } -static VALUE +static rb_parser_string_t * ripper_lex_io_get(struct parser_params *p, rb_parser_input_data input, int line_count) { VALUE src = (VALUE)input; - return rb_io_gets(src); + VALUE line = rb_io_gets(src); + if (NIL_P(line)) return 0; + return rb_str_to_parser_string(p, line); } -static VALUE +static rb_parser_string_t * ripper_lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count) { - return rb_parser_lex_get_str((struct lex_pointer_string *)input); + return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input); } static VALUE @@ -662,8 +665,4 @@ InitVM_ripper(void) */ rb_define_global_const("SCRIPT_LINES__", Qnil); #endif - rb_ripper_none = rb_obj_alloc(rb_cObject); - rb_obj_freeze(rb_ripper_none); - rb_gc_register_mark_object(rb_ripper_none); - } diff --git a/ext/ripper/ripper_init.h b/ext/ripper/ripper_init.h index 664bb7bce3..9d228107d1 100644 --- a/ext/ripper/ripper_init.h +++ b/ext/ripper/ripper_init.h @@ -1,7 +1,6 @@ #ifndef RIPPER_INIT_H #define RIPPER_INIT_H -extern VALUE rb_ripper_none; PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3); #endif /* RIPPER_INIT_H */ diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb index 3e368813e5..38f859dd97 100644 --- a/ext/ripper/tools/dsl.rb +++ b/ext/ripper/tools/dsl.rb @@ -20,77 +20,158 @@ class DSL NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source - def initialize(code, options, lineno = nil) + def self.line?(line, lineno = nil, indent: nil) + if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line + new(code, comma_split(option), lineno, indent: indent || space) + end + end + + def self.comma_split(str) + str or return [] + str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first) + end + + using Module.new { + refine Array do + def to_s + if empty? + "rb_ary_new()" + else + "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})" + end + end + end + } + + class Var + class Table < Hash + def initialize(&block) + super() {|tbl, arg| + tbl.fetch(arg, &block) + } + end + + def fetch(arg, &block) + super { + self[arg] = Var.new(self, arg, &block) + } + end + + def add(&block) + v = new_var + self[v] = Var.new(self, v, &block) + end + + def defined?(name) + name = name.to_s + any? {|_, v| v.var == name} + end + + def new_var + "v#{size+1}" + end + end + + attr_reader :var, :value + + PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze + + def pretty_print_instance_variables + PRETTY_PRINT_INSTANCE_VARIABLES + end + + alias to_s var + + def initialize(table, arg, &block) + @var = table.new_var + @value = yield arg + @table = table + end + + # Indexing. + # + # $:1 -> v1=get_value($:1) + # $:1[0] -> rb_ary_entry(v1, 0) + # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)] + # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1) + # + # Splat needs `[range]` because `Var` does not have the length info. + def [](idx) + if ::Range === idx + idx.map {|i| self[i]} + else + @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"} + end + end + end + + def initialize(code, options, lineno = nil, indent: "\t\t\t") @lineno = lineno + @indent = indent @events = {} @error = options.include?("error") - @brace = options.include?("brace") if options.include?("final") @final = "p->result" else @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue") end - @vars = 0 - # struct parser_params *p - p = p = "p" - - @code = +"" - code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"') - @last_value = eval(code) + bind = dsl_binding + @var_table = Var::Table.new {|arg| "get_value(#{arg})"} + code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) { + if (arg = $&) == "$:$" + '"p->s_lvalue"' + elsif arg.start_with?("$:") + "(#{@var_table[arg]}=@var_table[#{arg.dump}])" + else + arg.dump + end + } + @last_value = bind.eval(code) rescue SyntaxError $stderr.puts "error on line #{@lineno}" if @lineno raise end + def dsl_binding(p = "p") + # struct parser_params *p + binding + end + attr_reader :events undef lambda undef hash - undef class + undef :class def generate - s = "#@code#@final=#@last_value;" - s = "{VALUE #{ (1..@vars).map {|v| "v#{ v }" }.join(",") };#{ s }}" if @vars > 0 + s = "#@final=#@last_value;" s << "ripper_error(p);" if @error - s = "{#{ s }}" if @brace - "\t\t\t#{s}" - end - - def new_var - "v#{ @vars += 1 }" - end - - def opt_event(event, default, addend) - add_event(event, [default, addend], true) + unless @var_table.empty? + vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ") + s = "VALUE #{ vars }; #{ s }" + end + "#{@indent}{#{s}}" end - def add_event(event, args, qundef_check = false) + def add_event(event, args) event = event.to_s.sub(/!\z/, "") @events[event] = args.size vars = [] args.each do |arg| - vars << v = new_var - if arg =~ /\A\$:#{NAME_PATTERN}\z/ - @code << "#{ v }=get_value(#{arg});" - else - @code << "#{ v }=#{ arg };" - end + arg = @var_table.add {arg} unless Var === arg + vars << arg end - v = new_var - d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })" - d = "#{ vars.last }==rb_ripper_none ? #{ vars.first } : #{ d }" if qundef_check - @code << "#{ v }=#{ d };" - v + @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"} end def method_missing(event, *args) if event.to_s =~ /!\z/ add_event(event, args) - elsif args.empty? and /\Aid[A-Z_]/ =~ event.to_s + elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event)) event else - "#{ event }(#{ args.join(", ") })" + "#{ event }(#{ args.map(&:to_s).join(", ") })" end end diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb index 27aa53bce0..92ced37f04 100644 --- a/ext/ripper/tools/generate.rb +++ b/ext/ripper/tools/generate.rb @@ -167,15 +167,13 @@ require_relative "dsl" def read_ids1_with_locations(path) h = {} File.open(path) {|f| - f.each.with_index(1) do |line, i| + f.each do |line| next if /\A\#\s*define\s+dispatch/ =~ line next if /ripper_dispatch/ =~ line line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event| (h[event] ||= []).push [f.lineno, arity.to_i] end - if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/> - gen = DSL.new($2, ($1 || "").split(","), i) - gen.generate + if gen = DSL.line?(line, f.lineno) gen.events.each do |event, arity| (h[event] ||= []).push [f.lineno, arity.to_i] end diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb index a54302fb91..5e8a6e0cb5 100644 --- a/ext/ripper/tools/preproc.rb +++ b/ext/ripper/tools/preproc.rb @@ -51,47 +51,50 @@ def process(f, out, path, template) usercode f, out, path, template end -def prelude(f, out) - @exprs = {} +require_relative 'dsl' + +def generate_line(f, out) while line = f.gets - case line - when /\A%%/ + case + when gen = DSL.line?(line, f.lineno) + out << gen.generate << "\n" + when line.start_with?("%%") out << "%%\n" - return + break else - if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line) - case line - when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\// - @exprs[$1.chomp("_bit")] = $2.strip - when /^\s*(EXPR_\w+)\s+=\s+(.+)$/ - name = $1 - val = $2.chomp(",") - @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+") - end - end + out << yield(line) end - out << line end end -require_relative "dsl" +def prelude(f, out) + @exprs = {} + generate_line(f, out) do |line| + if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line) + case line + when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\// + @exprs[$1.chomp("_bit")] = $2.strip + when /^\s*(EXPR_\w+)\s+=\s+(.+)$/ + name = $1 + val = $2.chomp(",") + @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+") + end + end + line + end +end def grammar(f, out) - while line = f.gets + generate_line(f, out) do |line| case line - when %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/> - out << DSL.new($2, ($1 || "").split(",")).generate << "\n" when %r</\*%%%\*/> - out << "#if 0\n" + "#if 0\n" when %r</\*%> - out << "#endif\n" + "#endif\n" when %r<%\*/> - out << "\n" - when /\A%%/ - out << "%%\n" - return + "\n" else - out << line + line end end end diff --git a/ext/win32/lib/win32/registry.rb b/ext/win32/lib/win32/registry.rb index 4fed03f6a9..92b95d1bf7 100644 --- a/ext/win32/lib/win32/registry.rb +++ b/ext/win32/lib/win32/registry.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true require 'fiddle/import' module Win32 @@ -254,30 +254,34 @@ For detail, see the MSDN[http://msdn.microsoft.com/library/en-us/sysinfo/base/pr /^(?:x64|x86_64)/ =~ RUBY_PLATFORM end + TEMPLATE_HANDLE = 'J<' + def packhandle(h) - win64? ? packqw(h) : packdw(h) + [h].pack(TEMPLATE_HANDLE) end def unpackhandle(h) - win64? ? unpackqw(h) : unpackdw(h) + (h + [0].pack(TEMPLATE_HANDLE)).unpack1(TEMPLATE_HANDLE) end + TEMPLATE_DWORD = 'V' + def packdw(dw) - [dw].pack('V') + [dw].pack(TEMPLATE_DWORD) end def unpackdw(dw) - dw += [0].pack('V') - dw.unpack('V')[0] + (dw + [0].pack(TEMPLATE_DWORD)).unpack1(TEMPLATE_DWORD) end + TEMPLATE_QWORD = 'Q<' + def packqw(qw) - [ qw & 0xFFFFFFFF, qw >> 32 ].pack('VV') + [qw].pack(TEMPLATE_QWORD) end def unpackqw(qw) - qw = qw.unpack('VV') - (qw[1] << 32) | qw[0] + (qw + [0].pack(TEMPLATE_QWORD)).unpack1(TEMPLATE_QWORD) end def make_wstr(str) @@ -373,8 +377,7 @@ For detail, see the MSDN[http://msdn.microsoft.com/library/en-us/sysinfo/base/pr def self.expand_environ(str) str.gsub(Regexp.compile("%([^%]+)%".encode(str.encoding))) { v = $1.encode(LOCALE) - (e = ENV[v] || ENV[v.upcase]; e.encode(str.encoding) if e) || - $& + (ENV[v] || ENV[v.upcase])&.encode(str.encoding) || $& } end @@ -384,7 +387,6 @@ For detail, see the MSDN[http://msdn.microsoft.com/library/en-us/sysinfo/base/pr REG_RESOURCE_LIST REG_FULL_RESOURCE_DESCRIPTOR REG_RESOURCE_REQUIREMENTS_LIST REG_QWORD ].inject([]) do |ary, type| - type.freeze ary[Constants.const_get(type)] = type ary end.freeze @@ -657,7 +659,7 @@ For detail, see the MSDN[http://msdn.microsoft.com/library/en-us/sysinfo/base/pr when REG_DWORD [ type, API.unpackdw(data) ] when REG_DWORD_BIG_ENDIAN - [ type, data.unpack('N')[0] ] + [ type, data.unpack1('N') ] when REG_QWORD [ type, API.unpackqw(data) ] else diff --git a/ext/zlib/zlib.c b/ext/zlib/zlib.c index fe03072576..aad9f8d28a 100644 --- a/ext/zlib/zlib.c +++ b/ext/zlib/zlib.c @@ -90,7 +90,7 @@ static void zstream_expand_buffer_into(struct zstream*, unsigned long); static int zstream_expand_buffer_non_stream(struct zstream *z); static void zstream_append_buffer(struct zstream*, const Bytef*, long); static VALUE zstream_detach_buffer(struct zstream*); -static VALUE zstream_shift_buffer(struct zstream*, long); +static VALUE zstream_shift_buffer(struct zstream*, long, VALUE); static void zstream_buffer_ungets(struct zstream*, const Bytef*, unsigned long); static void zstream_buffer_ungetbyte(struct zstream*, int); static void zstream_append_input(struct zstream*, const Bytef*, long); @@ -170,8 +170,8 @@ static void gzfile_check_footer(struct gzfile*, VALUE outbuf); static void gzfile_write(struct gzfile*, Bytef*, long); static long gzfile_read_more(struct gzfile*, VALUE outbuf); static void gzfile_calc_crc(struct gzfile*, VALUE); -static VALUE gzfile_read(struct gzfile*, long); -static VALUE gzfile_read_all(struct gzfile*); +static VALUE gzfile_read(struct gzfile*, long, VALUE); +static VALUE gzfile_read_all(struct gzfile*, VALUE); static void gzfile_ungets(struct gzfile*, const Bytef*, long); static void gzfile_ungetbyte(struct gzfile*, int); static VALUE gzfile_writer_end_run(VALUE); @@ -820,19 +820,31 @@ zstream_detach_buffer(struct zstream *z) } static VALUE -zstream_shift_buffer(struct zstream *z, long len) +zstream_shift_buffer(struct zstream *z, long len, VALUE dst) { - VALUE dst; char *bufptr; long buflen = ZSTREAM_BUF_FILLED(z); if (buflen <= len) { - return zstream_detach_buffer(z); + if (NIL_P(dst) || (!ZSTREAM_IS_FINISHED(z) && !ZSTREAM_IS_GZFILE(z) && + rb_block_given_p())) { + return zstream_detach_buffer(z); + } else { + bufptr = RSTRING_PTR(z->buf); + rb_str_resize(dst, buflen); + memcpy(RSTRING_PTR(dst), bufptr, buflen); + } + buflen = 0; + } else { + bufptr = RSTRING_PTR(z->buf); + if (NIL_P(dst)) { + dst = rb_str_new(bufptr, len); + } else { + rb_str_resize(dst, len); + memcpy(RSTRING_PTR(dst), bufptr, len); + } + buflen -= len; } - - bufptr = RSTRING_PTR(z->buf); - dst = rb_str_new(bufptr, len); - buflen -= len; memmove(bufptr, bufptr + len, buflen); rb_str_set_len(z->buf, buflen); z->stream.next_out = (Bytef*)RSTRING_END(z->buf); @@ -2874,18 +2886,18 @@ gzfile_newstr(struct gzfile *gz, VALUE str) } static long -gzfile_fill(struct gzfile *gz, long len) +gzfile_fill(struct gzfile *gz, long len, VALUE outbuf) { if (len < 0) rb_raise(rb_eArgError, "negative length %ld given", len); if (len == 0) return 0; while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) < len) { - gzfile_read_more(gz, Qnil); + gzfile_read_more(gz, outbuf); } if (GZFILE_IS_FINISHED(gz)) { if (!(gz->z.flags & GZFILE_FLAG_FOOTER_FINISHED)) { - gzfile_check_footer(gz, Qnil); + gzfile_check_footer(gz, outbuf); } return -1; } @@ -2893,14 +2905,27 @@ gzfile_fill(struct gzfile *gz, long len) } static VALUE -gzfile_read(struct gzfile *gz, long len) +gzfile_read(struct gzfile *gz, long len, VALUE outbuf) { VALUE dst; - len = gzfile_fill(gz, len); - if (len == 0) return rb_str_new(0, 0); - if (len < 0) return Qnil; - dst = zstream_shift_buffer(&gz->z, len); + len = gzfile_fill(gz, len, outbuf); + + if (len < 0) { + if (!NIL_P(outbuf)) + rb_str_resize(outbuf, 0); + return Qnil; + } + if (len == 0) { + if (NIL_P(outbuf)) + return rb_str_new(0, 0); + else { + rb_str_resize(outbuf, 0); + return outbuf; + } + } + + dst = zstream_shift_buffer(&gz->z, len, outbuf); if (!NIL_P(dst)) gzfile_calc_crc(gz, dst); return dst; } @@ -2933,29 +2958,26 @@ gzfile_readpartial(struct gzfile *gz, long len, VALUE outbuf) rb_raise(rb_eEOFError, "end of file reached"); } - dst = zstream_shift_buffer(&gz->z, len); + dst = zstream_shift_buffer(&gz->z, len, outbuf); gzfile_calc_crc(gz, dst); - if (!NIL_P(outbuf)) { - rb_str_resize(outbuf, RSTRING_LEN(dst)); - memcpy(RSTRING_PTR(outbuf), RSTRING_PTR(dst), RSTRING_LEN(dst)); - dst = outbuf; - } return dst; } static VALUE -gzfile_read_all(struct gzfile *gz) +gzfile_read_all(struct gzfile *gz, VALUE dst) { - VALUE dst; - while (!ZSTREAM_IS_FINISHED(&gz->z)) { - gzfile_read_more(gz, Qnil); + gzfile_read_more(gz, dst); } if (GZFILE_IS_FINISHED(gz)) { if (!(gz->z.flags & GZFILE_FLAG_FOOTER_FINISHED)) { - gzfile_check_footer(gz, Qnil); + gzfile_check_footer(gz, dst); } + if (!NIL_P(dst)) { + rb_str_resize(dst, 0); + return dst; + } return rb_str_new(0, 0); } @@ -2993,7 +3015,7 @@ gzfile_getc(struct gzfile *gz) de = (unsigned char *)ds + GZFILE_CBUF_CAPA; (void)rb_econv_convert(gz->ec, &sp, se, &dp, de, ECONV_PARTIAL_INPUT|ECONV_AFTER_OUTPUT); rb_econv_check_error(gz->ec); - dst = zstream_shift_buffer(&gz->z, sp - ss); + dst = zstream_shift_buffer(&gz->z, sp - ss, Qnil); gzfile_calc_crc(gz, dst); rb_str_resize(cbuf, dp - ds); return cbuf; @@ -3001,7 +3023,7 @@ gzfile_getc(struct gzfile *gz) else { buf = gz->z.buf; len = rb_enc_mbclen(RSTRING_PTR(buf), RSTRING_END(buf), gz->enc); - dst = gzfile_read(gz, len); + dst = gzfile_read(gz, len, Qnil); if (NIL_P(dst)) return dst; return gzfile_newstr(gz, dst); } @@ -3909,7 +3931,7 @@ rb_gzreader_s_zcat(int argc, VALUE *argv, VALUE klass) if (!buf) { buf = rb_str_new(0, 0); } - tmpbuf = gzfile_read_all(get_gzfile(obj)); + tmpbuf = gzfile_read_all(get_gzfile(obj), Qnil); rb_str_cat(buf, RSTRING_PTR(tmpbuf), RSTRING_LEN(tmpbuf)); } @@ -4011,19 +4033,19 @@ static VALUE rb_gzreader_read(int argc, VALUE *argv, VALUE obj) { struct gzfile *gz = get_gzfile(obj); - VALUE vlen; + VALUE vlen, outbuf; long len; - rb_scan_args(argc, argv, "01", &vlen); + rb_scan_args(argc, argv, "02", &vlen, &outbuf); if (NIL_P(vlen)) { - return gzfile_read_all(gz); + return gzfile_read_all(gz, outbuf); } len = NUM2INT(vlen); if (len < 0) { rb_raise(rb_eArgError, "negative length %ld given", len); } - return gzfile_read(gz, len); + return gzfile_read(gz, len, outbuf); } /* @@ -4096,7 +4118,7 @@ rb_gzreader_getbyte(VALUE obj) struct gzfile *gz = get_gzfile(obj); VALUE dst; - dst = gzfile_read(gz, 1); + dst = gzfile_read(gz, 1, Qnil); if (!NIL_P(dst)) { dst = INT2FIX((unsigned int)(RSTRING_PTR(dst)[0]) & 0xff); } @@ -4217,7 +4239,7 @@ gzreader_skip_linebreaks(struct gzfile *gz) } } - str = zstream_shift_buffer(&gz->z, n - 1); + str = zstream_shift_buffer(&gz->z, n - 1, Qnil); gzfile_calc_crc(gz, str); } @@ -4238,7 +4260,7 @@ gzreader_charboundary(struct gzfile *gz, long n) if (l < n) { int n_bytes = rb_enc_precise_mbclen(p, e, gz->enc); if (MBCLEN_NEEDMORE_P(n_bytes)) { - if ((l = gzfile_fill(gz, n + MBCLEN_NEEDMORE_LEN(n_bytes))) > 0) { + if ((l = gzfile_fill(gz, n + MBCLEN_NEEDMORE_LEN(n_bytes), Qnil)) > 0) { return l; } } @@ -4290,10 +4312,10 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) if (NIL_P(rs)) { if (limit < 0) { - dst = gzfile_read_all(gz); + dst = gzfile_read_all(gz, Qnil); if (RSTRING_LEN(dst) == 0) return Qnil; } - else if ((n = gzfile_fill(gz, limit)) <= 0) { + else if ((n = gzfile_fill(gz, limit, Qnil)) <= 0) { return Qnil; } else { @@ -4303,7 +4325,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) else { n = limit; } - dst = zstream_shift_buffer(&gz->z, n); + dst = zstream_shift_buffer(&gz->z, n, Qnil); if (NIL_P(dst)) return dst; gzfile_calc_crc(gz, dst); dst = gzfile_newstr(gz, dst); @@ -4330,7 +4352,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) while (ZSTREAM_BUF_FILLED(&gz->z) < rslen) { if (ZSTREAM_IS_FINISHED(&gz->z)) { if (ZSTREAM_BUF_FILLED(&gz->z) > 0) gz->lineno++; - return gzfile_read(gz, rslen); + return gzfile_read(gz, rslen, Qnil); } gzfile_read_more(gz, Qnil); } @@ -4367,7 +4389,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) } gz->lineno++; - dst = gzfile_read(gz, n); + dst = gzfile_read(gz, n, Qnil); if (NIL_P(dst)) return dst; if (rspara) { gzreader_skip_linebreaks(gz); @@ -419,7 +419,6 @@ rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val) #endif #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE || PRINT_ROOT_TICKS) -#define TICK_TYPE 1 typedef struct { size_t size_pool_init_slots[SIZE_POOL_COUNT]; @@ -690,29 +689,33 @@ typedef struct RVALUE { VALUE v3; } values; } as; +} RVALUE; - /* Start of RVALUE_OVERHEAD. - * Do not directly read these members from the RVALUE as they're located - * at the end of the slot (which may differ in size depending on the size - * pool). */ -#if RACTOR_CHECK_MODE +/* These members ae located at the end of the slot that the object is in. */ +#if RACTOR_CHECK_MODE || GC_DEBUG +struct rvalue_overhead { +# if RACTOR_CHECK_MODE uint32_t _ractor_belonging_id; -#endif -#if GC_DEBUG +# endif +# if GC_DEBUG const char *file; int line; -#endif -} RVALUE; +# endif +}; -#if RACTOR_CHECK_MODE -# define RVALUE_OVERHEAD (sizeof(RVALUE) - offsetof(RVALUE, _ractor_belonging_id)) -#elif GC_DEBUG -# define RVALUE_OVERHEAD (sizeof(RVALUE) - offsetof(RVALUE, file)) +// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE) +# define RVALUE_OVERHEAD (sizeof(struct { \ + union { \ + struct rvalue_overhead overhead; \ + VALUE value; \ + }; \ +})) +# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_obj_slot_size(obj))) #else # define RVALUE_OVERHEAD 0 #endif -STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == (SIZEOF_VALUE * 5) + RVALUE_OVERHEAD); +STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == (SIZEOF_VALUE * 5)); STATIC_ASSERT(alignof_rvalue, RUBY_ALIGNOF(RVALUE) == SIZEOF_VALUE); typedef uintptr_t bits_t; @@ -956,7 +959,7 @@ typedef struct rb_objspace { #define HEAP_PAGE_ALIGN_LOG 16 #endif -#define BASE_SLOT_SIZE sizeof(RVALUE) +#define BASE_SLOT_SIZE (sizeof(RVALUE) + RVALUE_OVERHEAD) #define CEILDIV(i, mod) roomof(i, mod) enum { @@ -1419,17 +1422,7 @@ static const char *obj_type_name(VALUE obj); static void gc_finalize_deferred(void *dmy); -/* - * 1 - TSC (H/W Time Stamp Counter) - * 2 - getrusage - */ -#ifndef TICK_TYPE -#define TICK_TYPE 1 -#endif - #if USE_TICK_T - -#if TICK_TYPE == 1 /* the following code is only for internal tuning. */ /* Source code to use RDTSC is quoted and modified from @@ -1526,28 +1519,6 @@ tick(void) return clock(); } #endif /* TSC */ - -#elif TICK_TYPE == 2 -typedef double tick_t; -#define PRItick "4.9f" - -static inline tick_t -tick(void) -{ - return getrusage_time(); -} -#else /* TICK_TYPE */ -#error "choose tick type" -#endif /* TICK_TYPE */ - -#define MEASURE_LINE(expr) do { \ - volatile tick_t start_time = tick(); \ - volatile tick_t end_time; \ - expr; \ - end_time = tick(); \ - fprintf(stderr, "0\t%"PRItick"\t%s\n", end_time - start_time, #expr); \ -} while (0) - #else /* USE_TICK_T */ #define MEASURE_LINE(expr) expr #endif /* USE_TICK_T */ @@ -2667,7 +2638,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, #endif #if GC_DEBUG - RANY(obj)->file = rb_source_location_cstr(&RANY(obj)->line); + GET_RVALUE_OVERHEAD(obj)->file = rb_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line); GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */ #endif @@ -3509,7 +3480,7 @@ obj_free(rb_objspace_t *objspace, VALUE obj) } -#define OBJ_ID_INCREMENT (sizeof(RVALUE)) +#define OBJ_ID_INCREMENT (BASE_SLOT_SIZE) #define OBJ_ID_INITIAL (OBJ_ID_INCREMENT) static int @@ -8227,7 +8198,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t * do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % sizeof(RVALUE) == 0); + GC_ASSERT(vp % BASE_SLOT_SIZE == 0); if (bitset & 1) { objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; @@ -11894,7 +11865,7 @@ static inline void * objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size) { size = objspace_malloc_size(objspace, mem, size); - objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC); + objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {} #if CALC_EXACT_MALLOC_SIZE { @@ -12373,10 +12344,12 @@ ruby_mimcalloc(size_t num, size_t size) { void *mem; #if CALC_EXACT_MALLOC_SIZE - size += sizeof(struct malloc_obj_info); -#endif - mem = calloc(num, size); -#if CALC_EXACT_MALLOC_SIZE + struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(num, size); + if (UNLIKELY(t.left)) { + return NULL; + } + size = t.right + sizeof(struct malloc_obj_info); + mem = calloc1(size); if (!mem) { return NULL; } @@ -12392,6 +12365,8 @@ ruby_mimcalloc(size_t num, size_t size) #endif mem = info + 1; } +#else + mem = calloc(num, size); #endif return mem; } @@ -13209,7 +13184,7 @@ rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj } #if GC_DEBUG - APPEND_F("@%s:%d", RANY(obj)->file, RANY(obj)->line); + APPEND_F("@%s:%d", GET_RVALUE_OVERHEAD(obj)->file, GET_RVALUE_OVERHEAD(obj)->line); #endif } end: @@ -13518,7 +13493,7 @@ rb_gcdebug_print_obj_condition(VALUE obj) { rb_objspace_t *objspace = &rb_objspace; - fprintf(stderr, "created at: %s:%d\n", RANY(obj)->file, RANY(obj)->line); + fprintf(stderr, "created at: %s:%d\n", GET_RVALUE_OVERHEAD(obj)->file, GET_RVALUE_OVERHEAD(obj)->line); if (BUILTIN_TYPE(obj) == T_MOVED) { fprintf(stderr, "moved?: true\n"); @@ -13543,7 +13518,7 @@ rb_gcdebug_print_obj_condition(VALUE obj) if (is_lazy_sweeping(objspace)) { fprintf(stderr, "lazy sweeping?: true\n"); - fprintf(stderr, "page swept?: %s\n", GET_HEAP_PAGE(ptr)->flags.before_sweep ? "false" : "true"); + fprintf(stderr, "page swept?: %s\n", GET_HEAP_PAGE(obj)->flags.before_sweep ? "false" : "true"); } else { fprintf(stderr, "lazy sweeping?: false\n"); @@ -13619,10 +13594,9 @@ rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self) * traverse all living objects with an iterator. * * ObjectSpace also provides support for object finalizers, procs that will be - * called when a specific object is about to be destroyed by garbage - * collection. See the documentation for - * <code>ObjectSpace.define_finalizer</code> for important information on - * how to use this method correctly. + * called after a specific object was destroyed by garbage collection. See + * the documentation for +ObjectSpace.define_finalizer+ for important + * information on how to use this method correctly. * * a = "A" * b = "B" @@ -13681,7 +13655,7 @@ Init_GC(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), RBOOL(GC_DEBUG)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(RVALUE))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(BASE_SLOT_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); diff --git a/gems/bundled_gems b/gems/bundled_gems index ad979c1c5c..bc141542dd 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -6,14 +6,14 @@ # - revision: revision in repository-url to test # if `revision` is not given, "v"+`version` or `version` will be used. -minitest 5.22.3 https://github.com/minitest/minitest ea9caafc0754b1d6236a490d59e624b53209734a +minitest 5.23.1 https://github.com/minitest/minitest power_assert 2.0.3 https://github.com/ruby/power_assert 84e85124c5014a139af39161d484156cfe87a9ed rake 13.2.1 https://github.com/ruby/rake test-unit 3.6.2 https://github.com/test-unit/test-unit -rexml 3.2.6 https://github.com/ruby/rexml +rexml 3.2.8 https://github.com/ruby/rexml rss 0.3.0 https://github.com/ruby/rss net-ftp 0.3.4 https://github.com/ruby/net-ftp -net-imap 0.4.10 https://github.com/ruby/net-imap +net-imap 0.4.11 https://github.com/ruby/net-imap net-pop 0.1.2 https://github.com/ruby/net-pop net-smtp 0.5.0 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix @@ -21,11 +21,11 @@ prime 0.1.2 https://github.com/ruby/prime rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd typeprof 0.21.11 https://github.com/ruby/typeprof b19a6416da3a05d57fadd6ffdadb382b6d236ca5 debug 1.9.2 https://github.com/ruby/debug -racc 1.7.3 https://github.com/ruby/racc +racc 1.8.0 https://github.com/ruby/racc mutex_m 0.2.0 https://github.com/ruby/mutex_m getoptlong 0.2.1 https://github.com/ruby/getoptlong base64 0.2.0 https://github.com/ruby/base64 -bigdecimal 3.1.7 https://github.com/ruby/bigdecimal +bigdecimal 3.1.8 https://github.com/ruby/bigdecimal observer 0.1.2 https://github.com/ruby/observer abbrev 0.1.2 https://github.com/ruby/abbrev resolv-replace 0.1.1 https://github.com/ruby/resolv-replace diff --git a/include/ruby/internal/stdckdint.h b/include/ruby/internal/stdckdint.h index d02530136e..e5b5b8b751 100644 --- a/include/ruby/internal/stdckdint.h +++ b/include/ruby/internal/stdckdint.h @@ -20,6 +20,7 @@ * @brief C23 shim for <stdckdint.h> */ #include "ruby/internal/config.h" +#include "ruby/internal/cast.h" #include "ruby/internal/has/builtin.h" #include "ruby/internal/stdbool.h" @@ -35,14 +36,21 @@ # define RBIMPL_HAVE_STDCKDINT_H #endif +#ifdef __cplusplus +# /* It seems OS/Compiler provided stdckdint.h tend not support C++ yet. +# * Situations could improve someday but in a meantime let us work around. +# */ +# undef RBIMPL_HAVE_STDCKDINT_H +#endif + #ifdef RBIMPL_HAVE_STDCKDINT_H # /* Take that. */ # include <stdckdint.h> #elif RBIMPL_HAS_BUILTIN(__builtin_add_overflow) -# define ckd_add(x, y, z) ((bool)__builtin_add_overflow((y), (z), (x))) -# define ckd_sub(x, y, z) ((bool)__builtin_sub_overflow((y), (z), (x))) -# define ckd_mul(x, y, z) ((bool)__builtin_mul_overflow((y), (z), (x))) +# define ckd_add(x, y, z) RBIMPL_CAST((bool)__builtin_add_overflow((y), (z), (x))) +# define ckd_sub(x, y, z) RBIMPL_CAST((bool)__builtin_sub_overflow((y), (z), (x))) +# define ckd_mul(x, y, z) RBIMPL_CAST((bool)__builtin_mul_overflow((y), (z), (x))) # define __STDC_VERSION_STDCKDINT_H__ 202311L #/* elif defined(__cplusplus) */ @@ -977,6 +977,9 @@ opt_newarray_send case idMax: val = vm_opt_newarray_max(ec, num, STACK_ADDR_FROM_TOP(num)); break; + case idPack: + val = rb_vm_opt_newarray_pack(ec, (long)num-1, STACK_ADDR_FROM_TOP(num), TOPN(0)); + break; default: rb_bug("unreachable"); } diff --git a/internal/basic_operators.h b/internal/basic_operators.h index a59403631e..4732a65403 100644 --- a/internal/basic_operators.h +++ b/internal/basic_operators.h @@ -37,6 +37,7 @@ enum ruby_basic_operators { BOP_OR, BOP_CMP, BOP_DEFAULT, + BOP_PACK, BOP_LAST_ }; diff --git a/internal/error.h b/internal/error.h index 7e41f134d7..7a4daca6b3 100644 --- a/internal/error.h +++ b/internal/error.h @@ -142,6 +142,8 @@ VALUE rb_syntax_error_append(VALUE, VALUE, int, int, rb_encoding*, const char*, PRINTF_ARGS(void rb_enc_warn(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_sys_enc_warning(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_syserr_enc_warning(int err, rb_encoding *enc, const char *fmt, ...), 3, 4); +PRINTF_ARGS(void rb_enc_compile_warning(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); +PRINTF_ARGS(void rb_enc_compile_warn(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); rb_warning_category_t rb_warning_category_from_name(VALUE category); bool rb_warning_category_enabled_p(rb_warning_category_t category); VALUE rb_name_err_new(VALUE mesg, VALUE recv, VALUE method); diff --git a/internal/parse.h b/internal/parse.h index f06020c73f..5f52e8a8e3 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -13,7 +13,7 @@ #include "internal/static_assert.h" #ifdef UNIVERSAL_PARSER -#define rb_encoding void +#define rb_encoding const void #endif struct rb_iseq_struct; /* in vm_core.h */ @@ -54,8 +54,8 @@ rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_str void rb_ruby_parser_set_script_lines(rb_parser_t *p); void rb_ruby_parser_error_tolerant(rb_parser_t *p); void rb_ruby_parser_keep_tokens(rb_parser_t *p); -typedef VALUE (rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int); -rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line); +typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int); +rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line); RUBY_SYMBOL_EXPORT_BEGIN diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h index 98c3461954..8e306d18de 100644 --- a/internal/ruby_parser.h +++ b/internal/ruby_parser.h @@ -25,7 +25,7 @@ VALUE rb_parser_new(void); VALUE rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line); VALUE rb_str_new_parser_string(rb_parser_string_t *str); VALUE rb_str_new_mutable_parser_string(rb_parser_string_t *str); -VALUE rb_parser_lex_get_str(struct lex_pointer_string *ptr_str); +rb_parser_string_t *rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str); VALUE rb_node_str_string_val(const NODE *); VALUE rb_node_sym_string_val(const NODE *); @@ -97,6 +97,6 @@ enum lex_state_e { }; VALUE rb_ruby_ast_new(const NODE *const root); -rb_ast_t *rb_ruby_ast_data_get(VALUE vast); +rb_ast_t *rb_ruby_ast_data_get(VALUE ast_value); #endif /* INTERNAL_RUBY_PARSE_H */ @@ -8026,37 +8026,18 @@ ruby_popen_writer(char *const *argv, rb_pid_t *pid) return NULL; } -static void -rb_scan_open_args(int argc, const VALUE *argv, - VALUE *fname_p, int *oflags_p, int *fmode_p, - struct rb_io_encoding *convconfig_p, mode_t *perm_p) +static VALUE +rb_open_file(VALUE io, VALUE fname, VALUE vmode, VALUE vperm, VALUE opt) { - VALUE opt, fname, vmode, vperm; + struct rb_io_encoding convconfig; int oflags, fmode; mode_t perm; - argc = rb_scan_args(argc, argv, "12:", &fname, &vmode, &vperm, &opt); FilePathValue(fname); - rb_io_extract_modeenc(&vmode, &vperm, opt, &oflags, &fmode, convconfig_p); - - perm = NIL_P(vperm) ? 0666 : NUM2MODET(vperm); - - *fname_p = fname; - *oflags_p = oflags; - *fmode_p = fmode; - *perm_p = perm; -} - -static VALUE -rb_open_file(int argc, const VALUE *argv, VALUE io) -{ - VALUE fname; - int oflags, fmode; - struct rb_io_encoding convconfig; - mode_t perm; + rb_io_extract_modeenc(&vmode, &vperm, opt, &oflags, &fmode, &convconfig); + perm = NIL_P(vperm) ? 0666 : NUM2MODET(vperm); - rb_scan_open_args(argc, argv, &fname, &oflags, &fmode, &convconfig, &perm); rb_file_open_generic(io, fname, oflags, fmode, &convconfig, perm); return io; @@ -9379,6 +9360,8 @@ rb_io_make_open_file(VALUE obj) return fp; } +static VALUE io_initialize(VALUE io, VALUE fnum, VALUE vmode, VALUE opt); + /* * call-seq: * IO.new(fd, mode = 'r', **opts) -> io @@ -9424,18 +9407,24 @@ static VALUE rb_io_initialize(int argc, VALUE *argv, VALUE io) { VALUE fnum, vmode; + VALUE opt; + + rb_scan_args(argc, argv, "11:", &fnum, &vmode, &opt); + return io_initialize(io, fnum, vmode, opt); +} + +static VALUE +io_initialize(VALUE io, VALUE fnum, VALUE vmode, VALUE opt) +{ rb_io_t *fp; int fd, fmode, oflags = O_RDONLY; struct rb_io_encoding convconfig; - VALUE opt; #if defined(HAVE_FCNTL) && defined(F_GETFL) int ofmode; #else struct stat st; #endif - - argc = rb_scan_args(argc, argv, "11:", &fnum, &vmode, &opt); rb_io_extract_modeenc(&vmode, 0, opt, &oflags, &fmode, &convconfig); fd = NUM2INT(fnum); @@ -9584,17 +9573,16 @@ rb_file_initialize(int argc, VALUE *argv, VALUE io) if (RFILE(io)->fptr) { rb_raise(rb_eRuntimeError, "reinitializing File"); } - if (0 < argc && argc < 3) { - VALUE fd = rb_check_to_int(argv[0]); + VALUE fname, vmode, vperm, opt; + int posargc = rb_scan_args(argc, argv, "12:", &fname, &vmode, &vperm, &opt); + if (posargc < 3) { /* perm is File only */ + VALUE fd = rb_check_to_int(fname); if (!NIL_P(fd)) { - argv[0] = fd; - return rb_io_initialize(argc, argv, io); + return io_initialize(io, fd, vmode, opt); } } - rb_open_file(argc, argv, io); - - return io; + return rb_open_file(io, fname, vmode, vperm, opt); } /* :nodoc: */ diff --git a/io_buffer.c b/io_buffer.c index 7715aa0d37..e6ca61bdae 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -260,7 +260,7 @@ io_buffer_free(struct rb_io_buffer *buffer) if (buffer->mapping) { if (RB_IO_BUFFER_DEBUG) fprintf(stderr, "io_buffer_free:CloseHandle -> %p\n", buffer->mapping); if (!CloseHandle(buffer->mapping)) { - fprintf(stderr, "io_buffer_free:GetLastError -> %d\n", GetLastError()); + fprintf(stderr, "io_buffer_free:GetLastError -> %lu\n", GetLastError()); } buffer->mapping = NULL; } @@ -2441,13 +2441,13 @@ rb_io_buffer_initialize_copy(VALUE self, VALUE source) * * #copy can be used to put buffer into strings associated with buffer: * - * string= "buffer: " - * # => "buffer: " + * string= "data: " + * # => "data: " * buffer = IO::Buffer.for(string) * buffer.copy(IO::Buffer.for("test"), 5) * # => 4 * string - * # => "buffer:test" + * # => "data:test" * * Attempt to copy into a read-only buffer will fail: * @@ -3571,30 +3571,28 @@ io_buffer_not_inplace(VALUE self) * * \Buffer from string: * - * string = 'buffer' - * buffer = IO::Buffer.for(string) - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # ... - * buffer - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # 0x00000000 64 61 74 61 buffer - * - * buffer.get_string(2) # read content starting from offset 2 - * # => "ta" - * buffer.set_string('---', 1) # write content, starting from offset 1 - * # => 3 - * buffer - * # => - * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> - * # 0x00000000 64 2d 2d 2d d--- - * string # original string changed, too - * # => "d---" + * string = 'data' + * IO::Buffer.for(string) do |buffer| + * buffer + * # => + * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> + * # 0x00000000 64 61 74 61 data + * + * buffer.get_string(2) # read content starting from offset 2 + * # => "ta" + * buffer.set_string('---', 1) # write content, starting from offset 1 + * # => 3 + * buffer + * # => + * # #<IO::Buffer 0x00007f3f02be9b18+4 SLICE> + * # 0x00000000 64 2d 2d 2d d--- + * string # original string changed, too + * # => "d---" + * end * * \Buffer from file: * - * File.write('test.txt', 'test buffer') + * File.write('test.txt', 'test data') * # => 9 * buffer = IO::Buffer.map(File.open('test.txt')) * # => @@ -3611,7 +3609,7 @@ io_buffer_not_inplace(VALUE self) * buffer.set_string('---', 1) * # => 3 -- bytes written * File.read('test.txt') - * # => "t--- buffer" + * # => "t--- data" * * <b>The class is experimental and the interface is subject to change, this * is especially true of file mappings which may be removed entirely in @@ -839,18 +839,18 @@ make_compile_option_value(rb_compile_option_t *option) } rb_iseq_t * -rb_iseq_new(const VALUE vast, VALUE name, VALUE path, VALUE realpath, +rb_iseq_new(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, enum rb_iseq_type type) { - return rb_iseq_new_with_opt(vast, name, path, realpath, 0, parent, + return rb_iseq_new_with_opt(ast_value, name, path, realpath, 0, parent, 0, type, &COMPILE_OPTION_DEFAULT, Qnil); } static int -ast_line_count(const VALUE vast) +ast_line_count(const VALUE ast_value) { - rb_ast_t *ast = rb_ruby_ast_data_get(vast); + rb_ast_t *ast = rb_ruby_ast_data_get(ast_value); return ast->body.line_count; } @@ -880,11 +880,11 @@ iseq_new_setup_coverage(VALUE path, int line_count) } rb_iseq_t * -rb_iseq_new_top(const VALUE vast, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent) +rb_iseq_new_top(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent) { - iseq_new_setup_coverage(path, ast_line_count(vast)); + iseq_new_setup_coverage(path, ast_line_count(ast_value)); - return rb_iseq_new_with_opt(vast, name, path, realpath, 0, parent, 0, + return rb_iseq_new_with_opt(ast_value, name, path, realpath, 0, parent, 0, ISEQ_TYPE_TOP, &COMPILE_OPTION_DEFAULT, Qnil); } @@ -902,11 +902,11 @@ pm_iseq_new_top(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpath, c } rb_iseq_t * -rb_iseq_new_main(const VALUE vast, VALUE path, VALUE realpath, const rb_iseq_t *parent, int opt) +rb_iseq_new_main(const VALUE ast_value, VALUE path, VALUE realpath, const rb_iseq_t *parent, int opt) { - iseq_new_setup_coverage(path, ast_line_count(vast)); + iseq_new_setup_coverage(path, ast_line_count(ast_value)); - return rb_iseq_new_with_opt(vast, rb_fstring_lit("<main>"), + return rb_iseq_new_with_opt(ast_value, rb_fstring_lit("<main>"), path, realpath, 0, parent, 0, ISEQ_TYPE_MAIN, opt ? &COMPILE_OPTION_DEFAULT : &COMPILE_OPTION_FALSE, Qnil); @@ -927,16 +927,16 @@ pm_iseq_new_main(pm_scope_node_t *node, VALUE path, VALUE realpath, const rb_ise } rb_iseq_t * -rb_iseq_new_eval(const VALUE vast, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth) +rb_iseq_new_eval(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth) { if (rb_get_coverage_mode() & COVERAGE_TARGET_EVAL) { VALUE coverages = rb_get_coverages(); if (RTEST(coverages) && RTEST(path) && !RTEST(rb_hash_has_key(coverages, path))) { - iseq_setup_coverage(coverages, path, ast_line_count(vast) + first_lineno - 1); + iseq_setup_coverage(coverages, path, ast_line_count(ast_value) + first_lineno - 1); } } - return rb_iseq_new_with_opt(vast, name, path, realpath, first_lineno, + return rb_iseq_new_with_opt(ast_value, name, path, realpath, first_lineno, parent, isolated_depth, ISEQ_TYPE_EVAL, &COMPILE_OPTION_DEFAULT, Qnil); } @@ -971,12 +971,12 @@ iseq_translate(rb_iseq_t *iseq) } rb_iseq_t * -rb_iseq_new_with_opt(const VALUE vast, VALUE name, VALUE path, VALUE realpath, +rb_iseq_new_with_opt(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth, enum rb_iseq_type type, const rb_compile_option_t *option, VALUE script_lines) { - rb_ast_t *ast = rb_ruby_ast_data_get(vast); + rb_ast_t *ast = rb_ruby_ast_data_get(ast_value); rb_ast_body_t *body = ast ? &ast->body : NULL; const NODE *node = body ? body->root : 0; /* TODO: argument check */ @@ -1029,8 +1029,13 @@ pm_iseq_new_with_opt(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpa ISEQ_BODY(iseq)->prism = true; ISEQ_BODY(iseq)->param.flags.use_block = true; // unused block warning is not supported yet + rb_compile_option_t next_option; if (!option) option = &COMPILE_OPTION_DEFAULT; + next_option = *option; + next_option.coverage_enabled = node->coverage_enabled < 0 ? 0 : node->coverage_enabled > 0; + option = &next_option; + pm_location_t *location = &node->base.location; int32_t start_line = node->parser->start_line; @@ -1217,7 +1222,7 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V #endif VALUE (*parse)(VALUE vparser, VALUE fname, VALUE file, int start); int ln; - VALUE INITIALIZED vast; + VALUE INITIALIZED ast_value; rb_ast_t *ast; VALUE name = rb_fstring_lit("<compiled>"); @@ -1239,17 +1244,17 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V rb_parser_set_context(parser, outer_scope, FALSE); if (ruby_vm_keep_script_lines) rb_parser_set_script_lines(parser); RB_GC_GUARD(outer_scope_v); - vast = (*parse)(parser, file, src, ln); + ast_value = (*parse)(parser, file, src, ln); } - ast = rb_ruby_ast_data_get(vast); + ast = rb_ruby_ast_data_get(ast_value); if (!ast || !ast->body.root) { rb_ast_dispose(ast); rb_exc_raise(GET_EC()->errinfo); } else { - iseq = rb_iseq_new_with_opt(vast, name, file, realpath, ln, + iseq = rb_iseq_new_with_opt(ast_value, name, file, realpath, ln, NULL, 0, ISEQ_TYPE_TOP, &option, Qnil); rb_ast_dispose(ast); @@ -1273,6 +1278,21 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, NUM2INT(line)); + result.node.coverage_enabled = 1; + + switch (option.frozen_string_literal) { + case ISEQ_FROZEN_STRING_LITERAL_UNSET: + break; + case ISEQ_FROZEN_STRING_LITERAL_DISABLED: + pm_options_frozen_string_literal_set(&result.options, false); + break; + case ISEQ_FROZEN_STRING_LITERAL_ENABLED: + pm_options_frozen_string_literal_set(&result.options, true); + break; + default: + rb_bug("pm_iseq_compile_with_option: invalid frozen_string_literal=%d", option.frozen_string_literal); + break; + } VALUE error; if (RB_TYPE_P(src, T_FILE)) { @@ -1610,7 +1630,7 @@ iseqw_s_compile_file(int argc, VALUE *argv, VALUE self) VALUE file, opt = Qnil; VALUE parser, f, exc = Qnil, ret; rb_ast_t *ast; - VALUE vast; + VALUE ast_value; rb_compile_option_t option; int i; @@ -1629,8 +1649,8 @@ iseqw_s_compile_file(int argc, VALUE *argv, VALUE self) parser = rb_parser_new(); rb_parser_set_context(parser, NULL, FALSE); - vast = rb_parser_load_file(parser, file); - ast = rb_ruby_ast_data_get(vast); + ast_value = rb_parser_load_file(parser, file); + ast = rb_ruby_ast_data_get(ast_value); if (!ast->body.root) exc = GET_EC()->errinfo; rb_io_close(f); @@ -1641,7 +1661,7 @@ iseqw_s_compile_file(int argc, VALUE *argv, VALUE self) make_compile_option(&option, opt); - ret = iseqw_new(rb_iseq_new_with_opt(vast, rb_fstring_lit("<main>"), + ret = iseqw_new(rb_iseq_new_with_opt(ast_value, rb_fstring_lit("<main>"), file, rb_realpath_internal(Qnil, file, 1), 1, NULL, 0, ISEQ_TYPE_TOP, &option, @@ -1694,6 +1714,7 @@ iseqw_s_compile_file_prism(int argc, VALUE *argv, VALUE self) pm_parse_result_t result = { 0 }; result.options.line = 1; + result.node.coverage_enabled = 1; VALUE error = pm_load_parse_file(&result, file); diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index ed5f940b57..c933ad0471 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -28,6 +28,7 @@ module Gem::BUNDLED_GEMS "syslog" => "3.4.0", "ostruct" => "3.5.0", "pstore" => "3.5.0", + "rdoc" => "3.5.0", }.freeze EXACT = { @@ -45,6 +46,7 @@ module Gem::BUNDLED_GEMS "syslog" => true, "ostruct" => true, "pstore" => true, + "rdoc" => true, }.freeze PREFIXED = { diff --git a/lib/bundler.rb b/lib/bundler.rb index 5033109db6..adf0a8e450 100644 --- a/lib/bundler.rb +++ b/lib/bundler.rb @@ -42,6 +42,7 @@ module Bundler autoload :Checksum, File.expand_path("bundler/checksum", __dir__) autoload :CLI, File.expand_path("bundler/cli", __dir__) autoload :CIDetector, File.expand_path("bundler/ci_detector", __dir__) + autoload :CompactIndexClient, File.expand_path("bundler/compact_index_client", __dir__) autoload :Definition, File.expand_path("bundler/definition", __dir__) autoload :Dependency, File.expand_path("bundler/dependency", __dir__) autoload :Deprecate, File.expand_path("bundler/deprecate", __dir__) diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 40f19c7fed..eb67668cd2 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -767,13 +767,10 @@ module Bundler return unless SharedHelpers.md5_available? - latest = Fetcher::CompactIndex. - new(nil, Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")), nil, nil). - send(:compact_index_client). - instance_variable_get(:@cache). - dependencies("bundler"). - map {|d| Gem::Version.new(d.first) }. - max + require_relative "vendored_uri" + remote = Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")) + cache_path = Bundler.user_cache.join("compact_index", remote.cache_slug) + latest = Bundler::CompactIndexClient.new(cache_path).latest_version("bundler") return unless latest current = Gem::Version.new(VERSION) diff --git a/lib/bundler/cli/install.rb b/lib/bundler/cli/install.rb index 6c102d537d..a233d5d2e5 100644 --- a/lib/bundler/cli/install.rb +++ b/lib/bundler/cli/install.rb @@ -14,7 +14,7 @@ module Bundler Bundler.self_manager.install_locked_bundler_and_restart_with_it_if_needed - Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Bundler::FREEBSD + Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Gem.freebsd_platform? # Disable color in deployment mode Bundler.ui.shell = Thor::Shell::Basic.new if options[:deployment] diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb index 68e0d7e0d5..2bef0a1afc 100644 --- a/lib/bundler/compact_index_client.rb +++ b/lib/bundler/compact_index_client.rb @@ -21,24 +21,17 @@ module Bundler require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" - attr_reader :directory - - def initialize(directory, fetcher) - @directory = Pathname.new(directory) - @updater = Updater.new(fetcher) - @cache = Cache.new(@directory) - @endpoints = Set.new - @info_checksums_by_name = {} - @parsed_checksums = false - @mutex = Thread::Mutex.new + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) end def execution_mode=(block) Bundler::CompactIndexClient.debug { "execution_mode=" } - @endpoints = Set.new - + @cache.reset! @execution_mode = block end @@ -60,67 +53,28 @@ module Bundler end def names - Bundler::CompactIndexClient.debug { "/names" } - update("names", @cache.names_path, @cache.names_etag_path) - @cache.names + Bundler::CompactIndexClient.debug { "names" } + @parser.names end def versions - Bundler::CompactIndexClient.debug { "/versions" } - update("versions", @cache.versions_path, @cache.versions_etag_path) - versions, @info_checksums_by_name = @cache.versions - versions + Bundler::CompactIndexClient.debug { "versions" } + @parser.versions end def dependencies(names) Bundler::CompactIndexClient.debug { "dependencies(#{names})" } - execution_mode.call(names) do |name| - update_info(name) - @cache.dependencies(name).map {|d| d.unshift(name) } - end.flatten(1) - end - - def update_and_parse_checksums! - Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } - return @info_checksums_by_name if @parsed_checksums - update("versions", @cache.versions_path, @cache.versions_etag_path) - @info_checksums_by_name = @cache.checksums - @parsed_checksums = true - end - - private - - def update(remote_path, local_path, local_etag_path) - Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } - unless synchronize { @endpoints.add?(remote_path) } - Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } - return - end - @updater.update(url(remote_path), local_path, local_etag_path) - end - - def update_info(name) - Bundler::CompactIndexClient.debug { "update_info(#{name})" } - path = @cache.info_path(name) - unless existing = @info_checksums_by_name[name] - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } - return - end - checksum = SharedHelpers.checksum_for_file(path, :MD5) - if checksum == existing - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } - return - end - Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update("info/#{name}", path, @cache.info_etag_path(name)) + execution_mode.call(names) {|name| @parser.info(name) }.flatten(1) end - def url(path) - path + def latest_version(name) + Bundler::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[1]) }.max end - def synchronize - @mutex.synchronize { yield } + def available? + Bundler::CompactIndexClient.debug { "available?" } + @parser.available? end end end diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index 55911fdecf..bedd7f8028 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -7,114 +7,89 @@ module Bundler class Cache attr_reader :directory - def initialize(directory) + def initialize(directory, fetcher = nil) @directory = Pathname.new(directory).expand_path - info_roots.each {|dir| mkdir(dir) } - mkdir(info_etag_root) + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new + + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") end def names - lines(names_path) + fetch("names", names_path, names_etag_path) end - def names_path - directory.join("names") + def versions + fetch("versions", versions_path, versions_etag_path) end - def names_etag_path - directory.join("names.etag") - end + def info(name, remote_checksum = nil) + path = info_path(name) - def versions - versions_by_name = Hash.new {|hash, key| hash[key] = [] } - info_checksums_by_name = {} - - lines(versions_path).each do |line| - name, versions_string, info_checksum = line.split(" ", 3) - info_checksums_by_name[name] = info_checksum || "" - versions_string.split(",") do |version| - delete = version.delete_prefix!("-") - version = version.split("-", 2).unshift(name) - if delete - versions_by_name[name].delete(version) - else - versions_by_name[name] << version - end - end + if remote_checksum && remote_checksum != SharedHelpers.checksum_for_file(path, :MD5) + fetch("info/#{name}", path, info_etag_path(name)) + else + Bundler::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum is nil" : "versions index checksum matches local"})" } + read(path) end - - [versions_by_name, info_checksums_by_name] - end - - def versions_path - directory.join("versions") end - def versions_etag_path - directory.join("versions.etag") + def reset! + @mutex.synchronize { @endpoints.clear } end - def checksums - checksums = {} - - lines(versions_path).each do |line| - name, _, checksum = line.split(" ", 3) - checksums[name] = checksum - end - - checksums - end + private - def dependencies(name) - lines(info_path(name)).map do |line| - parse_gem(line) - end - end + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") def info_path(name) name = name.to_s + # TODO: converge this into the info_root by hashing all filenames like info_etag_path if /[^a-z0-9_-]/.match?(name) name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}" - info_roots.last.join(name) + @special_characters_info_root.join(name) else - info_roots.first.join(name) + @info_root.join(name) end end def info_etag_path(name) name = name.to_s - info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") + @info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") end - private - - def mkdir(dir) - SharedHelpers.filesystem_access(dir) do - FileUtils.mkdir_p(dir) + def mkdir(name) + directory.join(name).tap do |dir| + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end end end - def lines(path) - return [] unless path.file? - lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines - end + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Bundler::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end - def parse_gem(line) - @dependency_parser ||= GemParser.new - @dependency_parser.parse(line) + read(path) end - def info_roots - [ - directory.join("info"), - directory.join("info-special-characters"), - ] + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } end - def info_etag_root - directory.join("info-etags") + def read(path) + return unless path.file? + SharedHelpers.filesystem_access(path, :read, &:read) end end end diff --git a/lib/bundler/compact_index_client/cache_file.rb b/lib/bundler/compact_index_client/cache_file.rb index 5988bc91b3..299d683438 100644 --- a/lib/bundler/compact_index_client/cache_file.rb +++ b/lib/bundler/compact_index_client/cache_file.rb @@ -86,11 +86,6 @@ module Bundler end end - # remove this method when we stop generating md5 digests for legacy etags - def md5 - @digests && @digests["md5"] - end - def digests? @digests&.any? end diff --git a/lib/bundler/compact_index_client/parser.rb b/lib/bundler/compact_index_client/parser.rb new file mode 100644 index 0000000000..a227bc2cfd --- /dev/null +++ b/lib/bundler/compact_index_client/parser.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Bundler + class CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + data = @compact_index.info(name, info_checksums[name]) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + def available? + return @available unless @available.nil? + @available = !info_checksums.empty? + end + + private + + def info_checksums + @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums| + parse_version_checksum(line, checksums) + end + end + + def lines(data) + return [] if data.nil? || data.empty? + lines = data.split("\n") + header = lines.index("---") + header ? lines[header + 1..-1] : lines + end + + def gem_parser + @gem_parser ||= GemParser.new + end + + # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects. + # This method gets called at least once for every gem when parsing versions. + def parse_version_checksum(line, checksums) + line.freeze # allows slicing into the string to not allocate a copy of the line + name_end = line.index(" ") + checksum_start = line.index(" ", name_end + 1) + 1 + checksum_end = line.size - checksum_start + # freeze name since it is used as a hash key + # pre-freezing means a frozen copy isn't created + name = line[0, name_end].freeze + checksum = line[checksum_start, checksum_end] + checksums[name] = checksum + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 36f6b81db8..88c7146900 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -28,7 +28,6 @@ module Bundler CacheFile.copy(local_path) do |file| etag = etag_path.read.tap(&:chomp!) if etag_path.file? - etag ||= generate_etag(etag_path, file) # Remove this after 2.5.0 has been out for a while. # Subtract a byte to ensure the range won't be empty. # Avoids 416 (Range Not Satisfiable) responses. @@ -67,16 +66,6 @@ module Bundler etag_path.read.tap(&:chomp!) if etag_path.file? end - # When first releasing this opaque etag feature, we want to generate the old MD5 etag - # based on the content of the file. After that it will always use the saved opaque etag. - # This transparently saves existing users with good caches from updating a bunch of files. - # Remove this behavior after 2.5.0 has been out for a while. - def generate_etag(etag_path, file) - etag = file.md5.hexdigest - CacheFile.write(etag_path, etag) - etag - end - def etag_from_response(response) return unless response["ETag"] etag = response["ETag"].delete_prefix("W/") diff --git a/lib/bundler/constants.rb b/lib/bundler/constants.rb index de9698b577..9564771e78 100644 --- a/lib/bundler/constants.rb +++ b/lib/bundler/constants.rb @@ -1,7 +1,14 @@ # frozen_string_literal: true +require "rbconfig" + module Bundler WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/ + deprecate_constant :WINDOWS + FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd") - NULL = File::NULL + deprecate_constant :FREEBSD + + NULL = File::NULL + deprecate_constant :NULL end diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index c8faf77b3b..6cf1f9a255 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -69,7 +69,6 @@ module Bundler @sources = sources @unlock = unlock @optional_groups = optional_groups - @remote = false @prefer_local = false @specs = nil @ruby_version = ruby_version @@ -92,11 +91,12 @@ module Bundler @platforms = @locked_platforms.dup @locked_bundler_version = @locked_gems.bundler_version @locked_ruby_version = @locked_gems.ruby_version + @originally_locked_deps = @locked_gems.dependencies @originally_locked_specs = SpecSet.new(@locked_gems.specs) @locked_checksums = @locked_gems.checksums if unlock != true - @locked_deps = @locked_gems.dependencies + @locked_deps = @originally_locked_deps @locked_specs = @originally_locked_specs @locked_sources = @locked_gems.sources else @@ -111,6 +111,7 @@ module Bundler @locked_gems = nil @locked_deps = {} @locked_specs = SpecSet.new([]) + @originally_locked_deps = {} @originally_locked_specs = @locked_specs @locked_sources = [] @locked_platforms = [] @@ -130,7 +131,7 @@ module Bundler @sources.merged_gem_lockfile_sections!(locked_gem_sources.first) end - @unlock[:sources] ||= [] + @sources_to_unlock = @unlock.delete(:sources) || [] @unlock[:ruby] ||= if @ruby_version && locked_ruby_version_object @ruby_version.diff(locked_ruby_version_object) end @@ -142,11 +143,13 @@ module Bundler @path_changes = converge_paths @source_changes = converge_sources + @explicit_unlocks = @unlock.delete(:gems) || [] + if @unlock[:conservative] - @unlock[:gems] ||= @dependencies.map(&:name) + @gems_to_unlock = @explicit_unlocks.any? ? @explicit_unlocks : @dependencies.map(&:name) else - eager_unlock = (@unlock[:gems] || []).map {|name| Dependency.new(name, ">= 0") } - @unlock[:gems] = @locked_specs.for(eager_unlock, false, platforms).map(&:name).uniq + eager_unlock = @explicit_unlocks.map {|name| Dependency.new(name, ">= 0") } + @gems_to_unlock = @locked_specs.for(eager_unlock, false, platforms).map(&:name).uniq end @dependency_changes = converge_dependencies @@ -160,37 +163,24 @@ module Bundler end def resolve_only_locally! - @remote = false sources.local_only! resolve end def resolve_with_cache! + sources.local! sources.cached! resolve end def resolve_remotely! - @remote = true + sources.cached! sources.remote! resolve end - def resolution_mode=(options) - if options["local"] - @remote = false - else - @remote = true - @prefer_local = options["prefer-local"] - end - end - - def setup_sources_for_resolve - if @remote == false - sources.cached! - else - sources.remote! - end + def prefer_local! + @prefer_local = true end # For given dependency list returns a SpecSet with Gemspec of all the required @@ -225,7 +215,6 @@ module Bundler @resolver = nil @resolution_packages = nil @specs = nil - @gem_version_promoter = nil Bundler.ui.debug "The definition is missing dependencies, failed to resolve & materialize locally (#{e})" true @@ -307,7 +296,12 @@ module Bundler end end else - Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + if lockfile_exists? + Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + else + Bundler.ui.debug "Resolving dependencies because there's no lockfile" + end + start_resolution end end @@ -480,6 +474,8 @@ module Bundler private :sources def nothing_changed? + return false unless lockfile_exists? + !@source_changes && !@dependency_changes && !@new_platform && @@ -566,8 +562,10 @@ module Bundler @resolution_packages ||= begin last_resolve = converge_locked_specs remove_invalid_platforms!(current_dependencies) - packages = Resolver::Base.new(source_requirements, expanded_dependencies, last_resolve, @platforms, locked_specs: @originally_locked_specs, unlock: @unlock[:gems], prerelease: gem_version_promoter.pre?) - additional_base_requirements_for_resolve(packages, last_resolve) + packages = Resolver::Base.new(source_requirements, expanded_dependencies, last_resolve, @platforms, locked_specs: @originally_locked_specs, unlock: @gems_to_unlock, prerelease: gem_version_promoter.pre?) + packages = additional_base_requirements_to_prevent_downgrades(packages, last_resolve) + packages = additional_base_requirements_to_force_updates(packages) + packages end end @@ -582,7 +580,7 @@ module Bundler if missing_specs.any? missing_specs.each do |s| locked_gem = @locked_specs[s.name].last - next if locked_gem.nil? || locked_gem.version != s.version || !@remote + next if locked_gem.nil? || locked_gem.version != s.version || sources.local_mode? raise GemNotFound, "Your bundle is locked to #{locked_gem} from #{locked_gem.source}, but that version can " \ "no longer be found in that source. That means the author of #{locked_gem} has removed it. " \ "You'll need to update your bundle to a version other than #{locked_gem} that hasn't been " \ @@ -601,7 +599,7 @@ module Bundler break if incomplete_specs.empty? Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies") - setup_sources_for_resolve + sources.remote! resolution_packages.delete(incomplete_specs) @resolve = start_resolution specs = resolve.materialize(dependencies) @@ -671,14 +669,18 @@ module Bundler def change_reason if unlocking? - unlock_reason = @unlock.reject {|_k, v| Array(v).empty? }.map do |k, v| - if v == true - k.to_s - else - v = Array(v) - "#{k}: (#{v.join(", ")})" - end - end.join(", ") + unlock_targets = if @gems_to_unlock.any? + ["gems", @gems_to_unlock] + elsif @sources_to_unlock.any? + ["sources", @sources_to_unlock] + end + + unlock_reason = if unlock_targets + "#{unlock_targets.first}: (#{unlock_targets.last.join(", ")})" + else + @unlock[:ruby] ? "ruby" : "" + end + return "bundler is unlocking #{unlock_reason}" end [ @@ -733,7 +735,7 @@ module Bundler spec = @dependencies.find {|s| s.name == k } source = spec&.source if source&.respond_to?(:local_override!) - source.unlock! if @unlock[:gems].include?(spec.name) + source.unlock! if @gems_to_unlock.include?(spec.name) locals << [source, source.local_override!(v)] end end @@ -741,7 +743,7 @@ module Bundler sources_with_changes = locals.select do |source, changed| changed || specs_changed?(source) end.map(&:first) - !sources_with_changes.each {|source| @unlock[:sources] << source.name }.empty? + !sources_with_changes.each {|source| @sources_to_unlock << source.name }.empty? end def check_lockfile @@ -818,7 +820,7 @@ module Bundler # gem), unlock it. For git sources, this means to unlock the revision, which # will cause the `ref` used to be the most recent for the branch (or master) if # an explicit `ref` is not used. - if source.respond_to?(:unlock!) && @unlock[:sources].include?(source.name) + if source.respond_to?(:unlock!) && @sources_to_unlock.include?(source.name) source.unlock! changes = true end @@ -835,9 +837,7 @@ module Bundler dep.source = sources.get(dep.source) end - next if unlocking? - - unless locked_dep = @locked_deps[dep.name] + unless locked_dep = @originally_locked_deps[dep.name] changes = true next end @@ -864,7 +864,7 @@ module Bundler def converge_locked_specs converged = converge_specs(@locked_specs) - resolve = SpecSet.new(converged.reject {|s| @unlock[:gems].include?(s.name) }) + resolve = SpecSet.new(converged.reject {|s| @gems_to_unlock.include?(s.name) }) diff = nil @@ -897,7 +897,7 @@ module Bundler @specs_that_changed_sources << s if gemfile_source != lockfile_source deps << dep if !dep.source || lockfile_source.include?(dep.source) - @unlock[:gems] << name if lockfile_source.include?(dep.source) && lockfile_source != gemfile_source + @gems_to_unlock << name if lockfile_source.include?(dep.source) && lockfile_source != gemfile_source # Replace the locked dependency's source with the equivalent source from the Gemfile s.source = gemfile_source @@ -906,7 +906,7 @@ module Bundler s.source = default_source unless sources.get(lockfile_source) end - next if @unlock[:sources].include?(s.source.name) + next if @sources_to_unlock.include?(s.source.name) # Path sources have special logic if s.source.instance_of?(Source::Path) || s.source.instance_of?(Source::Gemspec) @@ -928,12 +928,12 @@ module Bundler else # If the spec is no longer in the path source, unlock it. This # commonly happens if the version changed in the gemspec - @unlock[:gems] << name + @gems_to_unlock << name end end if dep.nil? && requested_dependencies.find {|d| name == d.name } - @unlock[:gems] << s.name + @gems_to_unlock << s.name else converged << s end @@ -960,7 +960,7 @@ module Bundler else { default: Source::RubygemsAggregate.new(sources, source_map) }.merge(source_map.direct_requirements) end - source_requirements.merge!(source_map.locked_requirements) unless @remote + source_requirements.merge!(source_map.locked_requirements) if nothing_changed? metadata_dependencies.each do |dep| source_requirements[dep.name] = sources.metadata_source end @@ -1010,7 +1010,7 @@ module Bundler current == proposed end - def additional_base_requirements_for_resolve(resolution_packages, last_resolve) + def additional_base_requirements_to_prevent_downgrades(resolution_packages, last_resolve) return resolution_packages unless @locked_gems && !sources.expired_sources?(@locked_gems.sources) converge_specs(@originally_locked_specs - last_resolve).each do |locked_spec| next if locked_spec.source.is_a?(Source::Path) @@ -1019,6 +1019,26 @@ module Bundler resolution_packages end + def additional_base_requirements_to_force_updates(resolution_packages) + return resolution_packages if @explicit_unlocks.empty? + full_update = dup_for_full_unlock.resolve + @explicit_unlocks.each do |name| + version = full_update[name].first&.version + resolution_packages.base_requirements[name] = Gem::Requirement.new("= #{version}") if version + end + resolution_packages + end + + def dup_for_full_unlock + unlocked_definition = self.class.new(@lockfile, @dependencies, @sources, true, @ruby_version, @optional_groups, @gemfiles) + unlocked_definition.gem_version_promoter.tap do |gvp| + gvp.level = gem_version_promoter.level + gvp.strict = gem_version_promoter.strict + gvp.pre = gem_version_promoter.pre + end + unlocked_definition + end + def remove_invalid_platforms!(dependencies) return if Bundler.frozen_bundle? diff --git a/lib/bundler/errors.rb b/lib/bundler/errors.rb index b6a11cc721..c29b1bfed8 100644 --- a/lib/bundler/errors.rb +++ b/lib/bundler/errors.rb @@ -230,4 +230,18 @@ module Bundler status_code(38) end + + class CorruptBundlerInstallError < BundlerError + def initialize(loaded_spec) + @loaded_spec = loaded_spec + end + + def message + "The running version of Bundler (#{Bundler::VERSION}) does not match the version of the specification installed for it (#{@loaded_spec.version}). " \ + "This can be caused by reinstalling Ruby without removing previous installation, leaving around an upgraded default version of Bundler. " \ + "Reinstalling Ruby from scratch should fix the problem." + end + + status_code(39) + end end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index db914839b1..b92482d888 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -4,8 +4,6 @@ require_relative "base" require_relative "../worker" module Bundler - autoload :CompactIndexClient, File.expand_path("../compact_index_client", __dir__) - class Fetcher class CompactIndex < Base def self.compact_index_request(method_name) @@ -61,7 +59,7 @@ module Bundler return nil end # Read info file checksums out of /versions, so we can know if gems are up to date - compact_index_client.update_and_parse_checksums! + compact_index_client.available? rescue CompactIndexClient::Updater::MismatchedChecksumError => e Bundler.ui.debug(e.message) nil diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb index 72e5602cc3..256f0be348 100644 --- a/lib/bundler/installer.rb +++ b/lib/bundler/installer.rb @@ -235,15 +235,15 @@ module Bundler # returns whether or not a re-resolve was needed def resolve_if_needed(options) - @definition.resolution_mode = options - - if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file? - return false if @definition.nothing_changed? && !@definition.missing_specs? + @definition.prefer_local! if options["prefer-local"] + + if options["local"] || (@definition.no_resolve_needed? && !@definition.missing_specs?) + @definition.resolve_with_cache! + false + else + @definition.resolve_remotely! + true end - - @definition.setup_sources_for_resolve - - true end def lock diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index 0b09480f88..56a3b6f85c 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-ADD" "1" "April 2024" "" +.TH "BUNDLE\-ADD" "1" "May 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index fdb86bfd29..4ec301951f 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-BINSTUBS" "1" "April 2024" "" +.TH "BUNDLE\-BINSTUBS" "1" "May 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index c5899ace1a..e2da1269e6 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CACHE" "1" "April 2024" "" +.TH "BUNDLE\-CACHE" "1" "May 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index 08d41b7881..dee1af1326 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CHECK" "1" "April 2024" "" +.TH "BUNDLE\-CHECK" "1" "May 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems .SH "SYNOPSIS" @@ -9,6 +9,8 @@ \fBcheck\fR searches the local machine for each of the gems requested in the Gemfile\. If all gems are found, Bundler prints a success message and exits with a status of 0\. .P If not, the first missing gem is listed and Bundler exits status 1\. +.P +If the lockfile needs to be updated then it will be resolved using the gems installed on the local machine, if they satisfy the requirements\. .SH "OPTIONS" .TP \fB\-\-dry\-run\fR diff --git a/lib/bundler/man/bundle-check.1.ronn b/lib/bundler/man/bundle-check.1.ronn index f2846b8ff2..eb3ff1daf9 100644 --- a/lib/bundler/man/bundle-check.1.ronn +++ b/lib/bundler/man/bundle-check.1.ronn @@ -15,6 +15,9 @@ a status of 0. If not, the first missing gem is listed and Bundler exits status 1. +If the lockfile needs to be updated then it will be resolved using the gems +installed on the local machine, if they satisfy the requirements. + ## OPTIONS * `--dry-run`: diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index ca2fb65f59..7c7f9b5c77 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CLEAN" "1" "April 2024" "" +.TH "BUNDLE\-CLEAN" "1" "May 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index a78c6b0a18..a207dbbcaa 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONFIG" "1" "April 2024" "" +.TH "BUNDLE\-CONFIG" "1" "May 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index 86be5e4185..dca18ec43d 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONSOLE" "1" "April 2024" "" +.TH "BUNDLE\-CONSOLE" "1" "May 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index 94cbff4cbd..6489cc07f7 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-DOCTOR" "1" "April 2024" "" +.TH "BUNDLE\-DOCTOR" "1" "May 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 7aa49e0096..1548d29670 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-EXEC" "1" "April 2024" "" +.TH "BUNDLE\-EXEC" "1" "May 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 89eb9f1980..5df7b0ef2f 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-GEM" "1" "April 2024" "" +.TH "BUNDLE\-GEM" "1" "May 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 441ec12735..a3e7c7770d 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-HELP" "1" "April 2024" "" +.TH "BUNDLE\-HELP" "1" "May 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 287965c06a..a3d7ff0988 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INFO" "1" "April 2024" "" +.TH "BUNDLE\-INFO" "1" "May 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index d13f8ddc3b..a0edaaa18f 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INIT" "1" "April 2024" "" +.TH "BUNDLE\-INIT" "1" "May 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index 086fc88156..7a1038206e 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INJECT" "1" "April 2024" "" +.TH "BUNDLE\-INJECT" "1" "May 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 762c99e7c0..cc46a03b7f 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INSTALL" "1" "April 2024" "" +.TH "BUNDLE\-INSTALL" "1" "May 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index 93db700091..21723608cc 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LIST" "1" "April 2024" "" +.TH "BUNDLE\-LIST" "1" "May 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index c0190f91de..8b81b7732f 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LOCK" "1" "April 2024" "" +.TH "BUNDLE\-LOCK" "1" "May 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 38bacb67dc..41a8804a09 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OPEN" "1" "April 2024" "" +.TH "BUNDLE\-OPEN" "1" "May 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index cee9d63155..838fce45cd 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OUTDATED" "1" "April 2024" "" +.TH "BUNDLE\-OUTDATED" "1" "May 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 069966f1b2..0dbce7a7a4 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLATFORM" "1" "April 2024" "" +.TH "BUNDLE\-PLATFORM" "1" "May 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index f4e043c363..1290abb3ba 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLUGIN" "1" "April 2024" "" +.TH "BUNDLE\-PLUGIN" "1" "May 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 76a0479bc6..bf4a7cd323 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PRISTINE" "1" "April 2024" "" +.TH "BUNDLE\-PRISTINE" "1" "May 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index 90a664e331..c3c96b416d 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-REMOVE" "1" "April 2024" "" +.TH "BUNDLE\-REMOVE" "1" "May 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index f9f1fd1fa3..c054875efd 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-SHOW" "1" "April 2024" "" +.TH "BUNDLE\-SHOW" "1" "May 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 340ebac687..acd80ec75c 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-UPDATE" "1" "April 2024" "" +.TH "BUNDLE\-UPDATE" "1" "May 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 00ff0153cb..44eaf92224 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VERSION" "1" "April 2024" "" +.TH "BUNDLE\-VERSION" "1" "May 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index eba3b7df25..77b902214c 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VIZ" "1" "April 2024" "" +.TH "BUNDLE\-VIZ" "1" "May 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 4bd2bcaf67..199d1ce9fd 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE" "1" "April 2024" "" +.TH "BUNDLE" "1" "May 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management .SH "SYNOPSIS" diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index c9478a953e..fa9e31f615 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "GEMFILE" "5" "April 2024" "" +.TH "GEMFILE" "5" "May 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs .SH "SYNOPSIS" diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index e0582beba2..18180a81a1 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -require "pathname" - require "rubygems" unless defined?(Gem) -require "rubygems/specification" - # We can't let `Gem::Source` be autoloaded in the `Gem::Specification#source` # redefinition below, so we need to load it upfront. The reason is that if # Bundler monkeypatches are loaded before RubyGems activates an executable (for @@ -17,10 +13,6 @@ require "rubygems/specification" # `Gem::Source` from the redefined `Gem::Specification#source`. require "rubygems/source" -require_relative "match_metadata" -require_relative "force_platform" -require_relative "match_platform" - # Cherry-pick fixes to `Gem.ruby_version` to be useful for modern Bundler # versions and ignore patchlevels # (https://github.com/rubygems/rubygems/pull/5472, @@ -31,7 +23,19 @@ unless Gem.ruby_version.to_s == RUBY_VERSION || RUBY_PATCHLEVEL == -1 end module Gem + # Can be removed once RubyGems 3.5.11 support is dropped + unless Gem.respond_to?(:freebsd_platform?) + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + end + + require "rubygems/specification" + class Specification + require_relative "match_metadata" + require_relative "match_platform" + include ::Bundler::MatchMetadata include ::Bundler::MatchPlatform @@ -48,7 +52,7 @@ module Gem def full_gem_path if source.respond_to?(:root) - Pathname.new(loaded_from).dirname.expand_path(source.root).to_s + File.expand_path(File.dirname(loaded_from), source.root) else rg_full_gem_path end @@ -146,7 +150,23 @@ module Gem end end + module BetterPermissionError + def data + super + rescue Errno::EACCES + raise Bundler::PermissionError.new(loaded_from, :read) + end + end + + require "rubygems/stub_specification" + + class StubSpecification + prepend BetterPermissionError + end + class Dependency + require_relative "force_platform" + include ::Bundler::ForcePlatform attr_accessor :source, :groups diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index bfd000b1a0..5accda4bcb 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -113,7 +113,7 @@ module Bundler end def local_specs - @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true, "allow_cached" => true).specs.select {|spec| spec.name == "bundler" } + @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true).specs.select {|spec| spec.name == "bundler" } end def remote_specs diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb index 78760e6fa4..28f0cdff19 100644 --- a/lib/bundler/shared_helpers.rb +++ b/lib/bundler/shared_helpers.rb @@ -1,15 +1,17 @@ # frozen_string_literal: true -require "pathname" -require "rbconfig" - require_relative "version" -require_relative "constants" require_relative "rubygems_integration" require_relative "current_ruby" module Bundler + autoload :WINDOWS, File.expand_path("constants", __dir__) + autoload :FREEBSD, File.expand_path("constants", __dir__) + autoload :NULL, File.expand_path("constants", __dir__) + module SharedHelpers + autoload :Pathname, "pathname" + def root gemfile = find_gemfile raise GemfileNotFound, "Could not locate Gemfile" unless gemfile diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 645851286c..2fc9c6535f 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -182,6 +182,14 @@ module Bundler if err.include?("Could not find remote branch") raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else + idx = command.index("--depth") + if idx + command.delete_at(idx) + command.delete_at(idx) + command_with_no_credentials = check_allowed(command) + + err += "Retrying without --depth argument." + end raise GitCommandError.new(command_with_no_credentials, path, err) end end diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb index 4d27761365..6b05e17727 100644 --- a/lib/bundler/source/metadata.rb +++ b/lib/bundler/source/metadata.rb @@ -11,6 +11,8 @@ module Bundler end if local_spec = Gem.loaded_specs["bundler"] + raise CorruptBundlerInstallError.new(local_spec) if local_spec.version.to_s != Bundler::VERSION + idx << local_spec else idx << Gem::Specification.new do |s| diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 2e76becb84..3ac1bd4ff8 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -17,7 +17,7 @@ module Bundler @remotes = [] @dependency_names = [] @allow_remote = false - @allow_cached = options["allow_cached"] || false + @allow_cached = false @allow_local = options["allow_local"] || false @checksum_store = Checksum::Store.new @@ -50,10 +50,11 @@ module Bundler end def cached! + return unless File.exist?(cache_path) + return if @allow_cached @specs = nil - @allow_local = true @allow_cached = true end diff --git a/lib/bundler/source_list.rb b/lib/bundler/source_list.rb index bbaac33a95..5f9dd68f17 100644 --- a/lib/bundler/source_list.rb +++ b/lib/bundler/source_list.rb @@ -9,7 +9,7 @@ module Bundler :metadata_source def global_rubygems_source - @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true, "allow_cached" => true) + @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true) end def initialize @@ -22,6 +22,7 @@ module Bundler @metadata_source = Source::Metadata.new @merged_gem_lockfile_sections = false + @local_mode = true end def merged_gem_lockfile_sections? @@ -73,6 +74,10 @@ module Bundler global_rubygems_source end + def local_mode? + @local_mode + end + def default_source global_path_source || global_rubygems_source end @@ -140,11 +145,17 @@ module Bundler all_sources.each(&:local_only!) end + def local! + all_sources.each(&:local!) + end + def cached! all_sources.each(&:cached!) end def remote! + @local_mode = false + all_sources.each(&:remote!) end @@ -178,7 +189,7 @@ module Bundler replacement_source = replace_rubygems_source(replacement_sources, global_rubygems_source) return global_rubygems_source unless replacement_source - replacement_source.cached! + replacement_source.local! replacement_source end diff --git a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt index 175b821a62..6a46aba8df 100644 --- a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt +++ b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt @@ -2,7 +2,7 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. diff --git a/lib/did_you_mean/did_you_mean.gemspec b/lib/did_you_mean/did_you_mean.gemspec index 8fe5723129..be4ac76b4b 100644 --- a/lib/did_you_mean/did_you_mean.gemspec +++ b/lib/did_you_mean/did_you_mean.gemspec @@ -22,6 +22,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = '>= 2.5.0' - - spec.add_development_dependency "rake" end diff --git a/lib/find.gemspec b/lib/find.gemspec index cb845e9409..aef24a5028 100644 --- a/lib/find.gemspec +++ b/lib/find.gemspec @@ -25,7 +25,5 @@ Gem::Specification.new do |spec| spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] end diff --git a/lib/irb.rb b/lib/irb.rb index 168595d341..b3435c257e 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -311,7 +311,9 @@ require_relative "irb/pager" # ### Input Method # # The IRB input method determines how command input is to be read; by default, -# the input method for a session is IRB::RelineInputMethod. +# the input method for a session is IRB::RelineInputMethod. Unless the +# value of the TERM environment variable is 'dumb', in which case the +# most simplistic input method is used. # # You can set the input method by: # @@ -329,7 +331,8 @@ require_relative "irb/pager" # IRB::ReadlineInputMethod. # * `--nosingleline` or `--multiline` sets the input method to # IRB::RelineInputMethod. -# +# * `--nosingleline` together with `--nomultiline` sets the +# input to IRB::StdioInputMethod. # # # Method `conf.use_multiline?` and its synonym `conf.use_reline` return: @@ -928,8 +931,11 @@ module IRB # The lexer used by this irb session attr_accessor :scanner + attr_reader :from_binding + # Creates a new irb session - def initialize(workspace = nil, input_method = nil) + def initialize(workspace = nil, input_method = nil, from_binding: false) + @from_binding = from_binding @context = Context.new(self, workspace, input_method) @context.workspace.load_helper_methods_to_main @signal_status = :IN_IRB @@ -992,6 +998,7 @@ module IRB def run(conf = IRB.conf) in_nested_session = !!conf[:MAIN_CONTEXT] conf[:IRB_RC].call(context) if conf[:IRB_RC] + prev_context = conf[:MAIN_CONTEXT] conf[:MAIN_CONTEXT] = context save_history = !in_nested_session && conf[:SAVE_HISTORY] && context.io.support_history_saving? @@ -1014,6 +1021,9 @@ module IRB eval_input end ensure + # Do not restore to nil. It will cause IRB crash when used with threads. + IRB.conf[:MAIN_CONTEXT] = prev_context if prev_context + RubyVM.keep_script_lines = keep_script_lines_backup if defined?(RubyVM.keep_script_lines) trap("SIGINT", prev_trap) conf[:AT_EXIT].each{|hook| hook.call} @@ -1238,27 +1248,33 @@ module IRB irb_bug = true else irb_bug = false - # This is mostly to make IRB work nicely with Rails console's backtrace filtering, which patches WorkSpace#filter_backtrace - # In such use case, we want to filter the exception's backtrace before its displayed through Exception#full_message - # And we clone the exception object in order to avoid mutating the original exception - # TODO: introduce better API to expose exception backtrace externally - backtrace = exc.backtrace.map { |l| @context.workspace.filter_backtrace(l) }.compact + # To support backtrace filtering while utilizing Exception#full_message, we need to clone + # the exception to avoid modifying the original exception's backtrace. exc = exc.clone - exc.set_backtrace(backtrace) - end + filtered_backtrace = exc.backtrace.map { |l| @context.workspace.filter_backtrace(l) }.compact + backtrace_filter = IRB.conf[:BACKTRACE_FILTER] - if RUBY_VERSION < '3.0.0' - if STDOUT.tty? - message = exc.full_message(order: :bottom) - order = :bottom - else - message = exc.full_message(order: :top) - order = :top + if backtrace_filter + if backtrace_filter.respond_to?(:call) + filtered_backtrace = backtrace_filter.call(filtered_backtrace) + else + warn "IRB.conf[:BACKTRACE_FILTER] #{backtrace_filter} should respond to `call` method" + end end - else # '3.0.0' <= RUBY_VERSION - message = exc.full_message(order: :top) - order = :top + + exc.set_backtrace(filtered_backtrace) end + + highlight = Color.colorable? + + order = + if RUBY_VERSION < '3.0.0' + STDOUT.tty? ? :bottom : :top + else # '3.0.0' <= RUBY_VERSION + :top + end + + message = exc.full_message(order: order, highlight: highlight) message = convert_invalid_byte_sequence(message, exc.message.encoding) message = encode_with_invalid_byte_sequence(message, IRB.conf[:LC_MESSAGES].encoding) unless message.encoding.to_s.casecmp?(IRB.conf[:LC_MESSAGES].encoding.to_s) message = message.gsub(/((?:^\t.+$\n)+)/) { |m| @@ -1583,7 +1599,7 @@ class Binding else # If we're not in a debugger session, create a new IRB instance with the current # workspace - binding_irb = IRB::Irb.new(workspace) + binding_irb = IRB::Irb.new(workspace, from_binding: true) binding_irb.context.irb_path = irb_path binding_irb.run(IRB.conf) binding_irb.debug_break diff --git a/lib/irb/color.rb b/lib/irb/color.rb index ad8670160c..fca942b28b 100644 --- a/lib/irb/color.rb +++ b/lib/irb/color.rb @@ -79,12 +79,12 @@ module IRB # :nodoc: class << self def colorable? - supported = $stdout.tty? && (/mswin|mingw/ =~ RUBY_PLATFORM || (ENV.key?('TERM') && ENV['TERM'] != 'dumb')) + supported = $stdout.tty? && (/mswin|mingw/.match?(RUBY_PLATFORM) || (ENV.key?('TERM') && ENV['TERM'] != 'dumb')) # because ruby/debug also uses irb's color module selectively, # irb won't be activated in that case. if IRB.respond_to?(:conf) - supported && IRB.conf.fetch(:USE_COLORIZE, true) + supported && !!IRB.conf.fetch(:USE_COLORIZE, true) else supported end diff --git a/lib/irb/command/base.rb b/lib/irb/command/base.rb index b078b48237..1d406630a2 100644 --- a/lib/irb/command/base.rb +++ b/lib/irb/command/base.rb @@ -18,12 +18,12 @@ module IRB class << self def category(category = nil) @category = category if category - @category + @category || "No category" end def description(description = nil) @description = description if description - @description + @description || "No description provided." end def help_message(help_message = nil) diff --git a/lib/irb/command/debug.rb b/lib/irb/command/debug.rb index f9aca0a672..8a091a49ed 100644 --- a/lib/irb/command/debug.rb +++ b/lib/irb/command/debug.rb @@ -8,11 +8,6 @@ module IRB category "Debugging" description "Start the debugger of debug.gem." - BINDING_IRB_FRAME_REGEXPS = [ - '<internal:prelude>', - binding.method(:irb).source_location.first, - ].map { |file| /\A#{Regexp.escape(file)}:\d+:in (`|'Binding#)irb'\z/ } - def execute(_arg) execute_debug_command end @@ -36,7 +31,7 @@ module IRB # 3. Insert a debug breakpoint at `Irb#debug_break` with the intended command. # 4. Exit the current Irb#run call via `throw :IRB_EXIT`. # 5. `Irb#debug_break` will be called and trigger the breakpoint, which will run the intended command. - unless binding_irb? + unless irb_context.from_binding? puts "Debugging commands are only available when IRB is started with binding.irb" return end @@ -60,16 +55,6 @@ module IRB throw :IRB_EXIT end end - - private - - def binding_irb? - caller.any? do |frame| - BINDING_IRB_FRAME_REGEXPS.any? do |regexp| - frame.match?(regexp) - end - end - end end class DebugCommand < Debug diff --git a/lib/irb/command/help.rb b/lib/irb/command/help.rb index 1ed7a7707c..c2018f9b30 100644 --- a/lib/irb/command/help.rb +++ b/lib/irb/command/help.rb @@ -28,17 +28,9 @@ module IRB commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } commands_grouped_by_categories["Helper methods"] = helper_methods_info - user_aliases = irb_context.instance_variable_get(:@user_aliases) - - commands_grouped_by_categories["Aliases"] = user_aliases.map do |alias_name, target| - { display_name: alias_name, description: "Alias for `#{target}`" } - end - if irb_context.with_debugger # Remove the original "Debugging" category commands_grouped_by_categories.delete("Debugging") - # Add an empty "Debugging (from debug.gem)" category at the end - commands_grouped_by_categories["Debugging (from debug.gem)"] = [] end longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max @@ -46,15 +38,31 @@ module IRB output = StringIO.new help_cmds = commands_grouped_by_categories.delete("Help") + no_category_cmds = commands_grouped_by_categories.delete("No category") + aliases = irb_context.instance_variable_get(:@user_aliases).map do |alias_name, target| + { display_name: alias_name, description: "Alias for `#{target}`" } + end + # Display help commands first add_category_to_output("Help", help_cmds, output, longest_cmd_name_length) + # Display the rest of the commands grouped by categories commands_grouped_by_categories.each do |category, cmds| add_category_to_output(category, cmds, output, longest_cmd_name_length) end + # Display commands without a category + if no_category_cmds + add_category_to_output("No category", no_category_cmds, output, longest_cmd_name_length) + end + + # Display aliases + add_category_to_output("Aliases", aliases, output, longest_cmd_name_length) + # Append the debugger help at the end if irb_context.with_debugger + # Add "Debugging (from debug.gem)" category as title + add_category_to_output("Debugging (from debug.gem)", [], output, longest_cmd_name_length) output.puts DEBUGGER__.help end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 22e855f1ef..aafce7aade 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -73,11 +73,12 @@ module IRB self.prompt_mode = IRB.conf[:PROMPT_MODE] - if IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) - @irb_name = IRB.conf[:IRB_NAME] - else - @irb_name = IRB.conf[:IRB_NAME]+"#"+IRB.JobManager.n_jobs.to_s + @irb_name = IRB.conf[:IRB_NAME] + + unless IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) + @irb_name = @irb_name + "#" + IRB.JobManager.n_jobs.to_s end + self.irb_path = "(" + @irb_name + ")" case input_method @@ -85,7 +86,7 @@ module IRB @io = nil case use_multiline? when nil - if STDIN.tty? && IRB.conf[:PROMPT_MODE] != :INF_RUBY && !use_singleline? + if term_interactive? && IRB.conf[:PROMPT_MODE] != :INF_RUBY && !use_singleline? # Both of multiline mode and singleline mode aren't specified. @io = RelineInputMethod.new(build_completor) else @@ -99,7 +100,7 @@ module IRB unless @io case use_singleline? when nil - if (defined?(ReadlineInputMethod) && STDIN.tty? && + if (defined?(ReadlineInputMethod) && term_interactive? && IRB.conf[:PROMPT_MODE] != :INF_RUBY) @io = ReadlineInputMethod.new else @@ -151,6 +152,11 @@ module IRB @command_aliases = @user_aliases.merge(KEYWORD_ALIASES) end + private def term_interactive? + return true if ENV['TEST_IRB_FORCE_INTERACTIVE'] + STDIN.tty? && ENV['TERM'] != 'dumb' + end + # because all input will eventually be evaluated as Ruby code, # command names that conflict with Ruby keywords need special workaround # we can remove them once we implemented a better command system for IRB @@ -602,6 +608,10 @@ module IRB nil end + def from_binding? + @irb.from_binding + end + def evaluate_expression(code, line_no) # :nodoc: result = nil if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty? diff --git a/lib/irb/helper_method/conf.rb b/lib/irb/helper_method/conf.rb index 460f5ab78a..718ed279c0 100644 --- a/lib/irb/helper_method/conf.rb +++ b/lib/irb/helper_method/conf.rb @@ -1,7 +1,7 @@ module IRB module HelperMethod class Conf < Base - description "Returns the current context." + description "Returns the current IRB context." def execute IRB.CurrentContext diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 355047519c..7dc08912ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -52,6 +52,7 @@ module IRB # :nodoc: IRB.init_error IRB.parse_opts(argv: argv) IRB.run_config + IRB.validate_config IRB.load_modules unless @CONF[:PROMPT][@CONF[:PROMPT_MODE]] @@ -427,6 +428,40 @@ module IRB # :nodoc: @irbrc_files end + def IRB.validate_config + conf[:IRB_NAME] = conf[:IRB_NAME].to_s + + irb_rc = conf[:IRB_RC] + unless irb_rc.nil? || irb_rc.respond_to?(:call) + raise_validation_error "IRB.conf[:IRB_RC] should be a callable object. Got #{irb_rc.inspect}." + end + + back_trace_limit = conf[:BACK_TRACE_LIMIT] + unless back_trace_limit.is_a?(Integer) + raise_validation_error "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got #{back_trace_limit.inspect}." + end + + prompt = conf[:PROMPT] + unless prompt.is_a?(Hash) + msg = "IRB.conf[:PROMPT] should be a Hash. Got #{prompt.inspect}." + + if prompt.is_a?(Symbol) + msg += " Did you mean to set `IRB.conf[:PROMPT_MODE]`?" + end + + raise_validation_error msg + end + + eval_history = conf[:EVAL_HISTORY] + unless eval_history.nil? || eval_history.is_a?(Integer) + raise_validation_error "IRB.conf[:EVAL_HISTORY] should be an integer. Got #{eval_history.inspect}." + end + end + + def IRB.raise_validation_error(msg) + raise TypeError, msg, @irbrc_files + end + # loading modules def IRB.load_modules for m in @CONF[:LOAD_MODULES] diff --git a/lib/irb/input-method.rb b/lib/irb/input-method.rb index e5adb350e8..684527edc4 100644 --- a/lib/irb/input-method.rb +++ b/lib/irb/input-method.rb @@ -67,6 +67,7 @@ module IRB # # See IO#gets for more information. def gets + puts if @stdout.tty? # workaround for debug compatibility test print @prompt line = @stdin.gets @line[@line_no += 1] = line diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index cfe36be83f..f6ac7f0f5f 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -219,28 +219,7 @@ module IRB :unrecoverable_error rescue SyntaxError => e case e.message - when /unterminated (?:string|regexp) meets end of file/ - # "unterminated regexp meets end of file" - # - # example: - # / - # - # "unterminated string meets end of file" - # - # example: - # ' - return :recoverable_error - when /syntax error, unexpected end-of-input/ - # "syntax error, unexpected end-of-input, expecting keyword_end" - # - # example: - # if true - # hoge - # if false - # fuga - # end - return :recoverable_error - when /syntax error, unexpected keyword_end/ + when /unexpected keyword_end/ # "syntax error, unexpected keyword_end" # # example: @@ -250,7 +229,7 @@ module IRB # example: # end return :unrecoverable_error - when /syntax error, unexpected '\.'/ + when /unexpected '\.'/ # "syntax error, unexpected '.'" # # example: @@ -262,6 +241,27 @@ module IRB # example: # method / f / return :unrecoverable_error + when /unterminated (?:string|regexp) meets end of file/ + # "unterminated regexp meets end of file" + # + # example: + # / + # + # "unterminated string meets end of file" + # + # example: + # ' + return :recoverable_error + when /unexpected end-of-input/ + # "syntax error, unexpected end-of-input, expecting keyword_end" + # + # example: + # if true + # hoge + # if false + # fuga + # end + return :recoverable_error else return :other_error end diff --git a/lib/irb/version.rb b/lib/irb/version.rb index 9a7b12766b..c41917329c 100644 --- a/lib/irb/version.rb +++ b/lib/irb/version.rb @@ -5,7 +5,7 @@ # module IRB # :nodoc: - VERSION = "1.12.0" + VERSION = "1.13.1" @RELEASE_VERSION = VERSION - @LAST_UPDATE_DATE = "2024-03-06" + @LAST_UPDATE_DATE = "2024-05-05" end diff --git a/lib/prism.rb b/lib/prism.rb index c733baf8c9..66a64e7fd0 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -13,7 +13,6 @@ module Prism autoload :BasicVisitor, "prism/visitor" autoload :Compiler, "prism/compiler" - autoload :Debug, "prism/debug" autoload :DesugarCompiler, "prism/desugar_compiler" autoload :Dispatcher, "prism/dispatcher" autoload :DotVisitor, "prism/dot_visitor" @@ -32,7 +31,6 @@ module Prism # Some of these constants are not meant to be exposed, so marking them as # private here. - private_constant :Debug private_constant :LexCompat private_constant :LexRipper @@ -67,11 +65,10 @@ module Prism end end +require_relative "prism/polyfill/byteindex" require_relative "prism/node" require_relative "prism/node_ext" require_relative "prism/parse_result" -require_relative "prism/parse_result/comments" -require_relative "prism/parse_result/newlines" # This is a Ruby implementation of the prism parser. If we're running on CRuby # and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then diff --git a/lib/prism/debug.rb b/lib/prism/debug.rb deleted file mode 100644 index 74f824faa7..0000000000 --- a/lib/prism/debug.rb +++ /dev/null @@ -1,249 +0,0 @@ -# frozen_string_literal: true - -module Prism - # This module is used for testing and debugging and is not meant to be used by - # consumers of this library. - module Debug - # A wrapper around a RubyVM::InstructionSequence that provides a more - # convenient interface for accessing parts of the iseq. - class ISeq # :nodoc: - attr_reader :parts - - def initialize(parts) - @parts = parts - end - - def type - parts[0] - end - - def local_table - parts[10] - end - - def instructions - parts[13] - end - - def each_child - instructions.each do |instruction| - # Only look at arrays. Other instructions are line numbers or - # tracepoint events. - next unless instruction.is_a?(Array) - - instruction.each do |opnd| - # Only look at arrays. Other operands are literals. - next unless opnd.is_a?(Array) - - # Only look at instruction sequences. Other operands are literals. - next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat" - - yield ISeq.new(opnd) - end - end - end - end - - private_constant :ISeq - - # :call-seq: - # Debug::cruby_locals(source) -> Array - # - # For the given source, compiles with CRuby and returns a list of all of the - # sets of local variables that were encountered. - def self.cruby_locals(source) - verbose, $VERBOSE = $VERBOSE, nil - - begin - locals = [] #: Array[Array[Symbol | Integer]] - stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)] - - while (iseq = stack.pop) - names = [*iseq.local_table] - names.map!.with_index do |name, index| - # When an anonymous local variable is present in the iseq's local - # table, it is represented as the stack offset from the top. - # However, when these are dumped to binary and read back in, they - # are replaced with the symbol :#arg_rest. To consistently handle - # this, we replace them here with their index. - if name == :"#arg_rest" - names.length - index + 1 - else - name - end - end - - locals << names - iseq.each_child { |child| stack << child } - end - - locals - ensure - $VERBOSE = verbose - end - end - - # Used to hold the place of a local that will be in the local table but - # cannot be accessed directly from the source code. For example, the - # iteration variable in a for loop or the positional parameter on a method - # definition that is destructured. - AnonymousLocal = Object.new - private_constant :AnonymousLocal - - # :call-seq: - # Debug::prism_locals(source) -> Array - # - # For the given source, parses with prism and returns a list of all of the - # sets of local variables that were encountered. - def self.prism_locals(source) - locals = [] #: Array[Array[Symbol | Integer]] - stack = [Prism.parse(source).value] #: Array[Prism::node] - - while (node = stack.pop) - case node - when BlockNode, DefNode, LambdaNode - names = node.locals - params = - if node.is_a?(DefNode) - node.parameters - elsif node.parameters.is_a?(NumberedParametersNode) - nil - else - node.parameters&.parameters - end - - # prism places parameters in the same order that they appear in the - # source. CRuby places them in the order that they need to appear - # according to their own internal calling convention. We mimic that - # order here so that we can compare properly. - if params - sorted = [ - *params.requireds.map do |required| - if required.is_a?(RequiredParameterNode) - required.name - else - AnonymousLocal - end - end, - *params.optionals.map(&:name), - *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)), - *params.posts.map do |post| - if post.is_a?(RequiredParameterNode) - post.name - else - AnonymousLocal - end - end, - *params.keywords.grep(RequiredKeywordParameterNode).map(&:name), - *params.keywords.grep(OptionalKeywordParameterNode).map(&:name), - ] - - sorted << AnonymousLocal if params.keywords.any? - - if params.keyword_rest.is_a?(ForwardingParameterNode) - sorted.push(:*, :**, :&, :"...") - elsif params.keyword_rest.is_a?(KeywordRestParameterNode) - sorted << (params.keyword_rest.name || :**) - end - - # Recurse down the parameter tree to find any destructured - # parameters and add them after the other parameters. - param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse - while (param = param_stack.pop) - case param - when MultiTargetNode - param_stack.concat(param.rights.reverse) - param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name) - param_stack.concat(param.lefts.reverse) - when RequiredParameterNode - sorted << param.name - when SplatNode - sorted << param.expression.name - end - end - - if params.block - sorted << (params.block.name || :&) - end - - names = sorted.concat(names - sorted) - end - - names.map!.with_index do |name, index| - if name == AnonymousLocal - names.length - index + 1 - else - name - end - end - - locals << names - when ClassNode, ModuleNode, ProgramNode, SingletonClassNode - locals << node.locals - when ForNode - locals << [2] - when PostExecutionNode - locals.push([], []) - when InterpolatedRegularExpressionNode - locals << [] if node.once? - end - - stack.concat(node.compact_child_nodes) - end - - locals - end - - # :call-seq: - # Debug::newlines(source) -> Array - # - # For the given source string, return the byte offsets of every newline in - # the source. - def self.newlines(source) - Prism.parse(source).source.offsets - end - - # A wrapping around prism's internal encoding data structures. This is used - # for reflection and debugging purposes. - class Encoding - # The name of the encoding, that can be passed to Encoding.find. - attr_reader :name - - # Initialize a new encoding with the given name and whether or not it is - # a multibyte encoding. - def initialize(name, multibyte) - @name = name - @multibyte = multibyte - end - - # Whether or not the encoding is a multibyte encoding. - def multibyte? - @multibyte - end - - # Returns the number of bytes of the first character in the source string, - # if it is valid for the encoding. Otherwise, returns 0. - def width(source) - Encoding._width(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphanumeric character for the encoding. - def alnum?(source) - Encoding._alnum?(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphabetic character for the encoding. - def alpha?(source) - Encoding._alpha?(name, source) - end - - # Returns true if the first character in the source string is a valid - # uppercase character for the encoding. - def upper?(source) - Encoding._upper?(name, source) - end - end - end -end diff --git a/lib/prism/desugar_compiler.rb b/lib/prism/desugar_compiler.rb index 9b62c00df3..de02445149 100644 --- a/lib/prism/desugar_compiler.rb +++ b/lib/prism/desugar_compiler.rb @@ -73,7 +73,7 @@ module Prism # Desugar `x += y` to `x = x + y` def compile - operator_loc = node.operator_loc.chop + binary_operator_loc = node.binary_operator_loc.chop write_class.new( source, @@ -84,15 +84,15 @@ module Prism 0, read_class.new(source, *arguments, node.name_loc), nil, - operator_loc.slice.to_sym, - operator_loc, + binary_operator_loc.slice.to_sym, + binary_operator_loc, nil, ArgumentsNode.new(source, 0, [node.value], node.value.location), nil, nil, node.location ), - node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.binary_operator_loc.copy(start_offset: node.binary_operator_loc.end_offset - 1, length: 1), node.location ) end diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 2014ccea31..df7e183310 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -317,7 +317,7 @@ module Prism buffer.read end - Serialize.load_tokens(Source.new(code), serialized) + Serialize.load_tokens(Source.for(code), serialized) end def parse_common(string, code, options) # :nodoc: @@ -329,7 +329,7 @@ module Prism LibRubyParser::PrismBuffer.with do |buffer| LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options)) - source = Source.new(code) + source = Source.for(code) loader = Serialize::Loader.new(source, buffer.read) loader.load_header @@ -343,7 +343,7 @@ module Prism LibRubyParser::PrismBuffer.with do |buffer| LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options)) - source = Source.new(code) + source = Source.for(code) loader = Serialize::Loader.new(source, buffer.read) tokens = loader.load_tokens @@ -394,7 +394,7 @@ module Prism template << "L" if (encoding = options[:encoding]) - name = encoding.name + name = encoding.is_a?(Encoding) ? encoding.name : encoding values.push(name.bytesize, name.b) template << "A*" else @@ -408,7 +408,7 @@ module Prism values << dump_options_command_line(options) template << "C" - values << { nil => 0, "3.3.0" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version]) + values << { nil => 0, "3.3.0" => 1, "3.3.1" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version]) template << "L" if (scopes = options[:scopes]) diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index f199af1883..4f8e443a3b 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -861,7 +861,7 @@ module Prism # We sort by location to compare against Ripper's output tokens.sort_by!(&:location) - Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source)) + Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source)) end end diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb index 8674544065..cdc4b2bf08 100644 --- a/lib/prism/node_ext.rb +++ b/lib/prism/node_ext.rb @@ -3,6 +3,17 @@ # Here we are reopening the prism module to provide methods on nodes that aren't # templated and are meant as convenience methods. module Prism + class Node + def deprecated(*replacements) # :nodoc: + suggest = replacements.map { |replacement| "#{self.class}##{replacement}" } + warn(<<~MSG, category: :deprecated) + [deprecation]: #{self.class}##{caller_locations(1, 1)[0].label} is deprecated \ + and will be removed in the next major version. Use #{suggest.join("/")} instead. + #{(caller(1, 3) || []).join("\n")} + MSG + end + end + module RegularExpressionOptions # :nodoc: # Returns a numeric value that represents the flags that were used to create # the regular expression. @@ -92,7 +103,19 @@ module Prism class RationalNode < Node # Returns the value of the node as a Ruby Rational. def value - Rational(numeric.is_a?(IntegerNode) ? numeric.value : slice.chomp("r")) + Rational(numerator, denominator) + end + + # Returns the value of the node as an IntegerNode or a FloatNode. This + # method is deprecated in favor of #value or #numerator/#denominator. + def numeric + deprecated("value", "numerator", "denominator") + + if denominator == 1 + IntegerNode.new(source, flags, numerator, location.chop) + else + FloatNode.new(source, numerator.to_f / denominator, location.chop) + end end end @@ -143,11 +166,12 @@ module Prism current = self #: node? while current.is_a?(ConstantPathNode) - child = current.child - if child.is_a?(MissingNode) + name = current.name + if name.nil? raise MissingNodesInConstantPathError, "Constant path contains missing nodes. Cannot compute full name" end - parts.unshift(child.name) + + parts.unshift(name) current = current.parent end @@ -162,6 +186,14 @@ module Prism def full_name full_name_parts.join("::") end + + # Previously, we had a child node on this class that contained either a + # constant read or a missing node. To not cause a breaking change, we + # continue to supply that API. + def child + deprecated("name", "name_loc") + name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) + end end class ConstantPathTargetNode < Node @@ -179,17 +211,25 @@ module Prism raise ConstantPathNode::DynamicPartsInConstantPathError, "Constant target path contains dynamic parts. Cannot compute full name" end - if child.is_a?(MissingNode) + if name.nil? raise ConstantPathNode::MissingNodesInConstantPathError, "Constant target path contains missing nodes. Cannot compute full name" end - parts.push(child.name) + parts.push(name) end # Returns the full name of this constant path. For example: "Foo::Bar" def full_name full_name_parts.join("::") end + + # Previously, we had a child node on this class that contained either a + # constant read or a missing node. To not cause a breaking change, we + # continue to supply that API. + def child + deprecated("name", "name_loc") + name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) + end end class ConstantTargetNode < Node @@ -257,4 +297,147 @@ module Prism names end end + + class CallNode < Node + # When a call node has the attribute_write flag set, it means that the call + # is using the attribute write syntax. This is either a method call to []= + # or a method call to a method that ends with =. Either way, the = sign is + # present in the source. + # + # Prism returns the message_loc _without_ the = sign attached, because there + # can be any amount of space between the message and the = sign. However, + # sometimes you want the location of the full message including the inner + # space and the = sign. This method provides that. + def full_message_loc + attribute_write? ? message_loc&.adjoin("=") : message_loc + end + end + + class CallOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ClassVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantPathOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class GlobalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class IndexOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class InstanceVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class LocalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end end diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb index e01aa070c2..798fde09e5 100644 --- a/lib/prism/parse_result.rb +++ b/lib/prism/parse_result.rb @@ -5,6 +5,14 @@ module Prism # conjunction with locations to allow them to resolve line numbers and source # ranges. class Source + # Create a new source object with the given source code. This method should + # be used instead of `new` and it will return either a `Source` or a + # specialized and more performant `ASCIISource` if no multibyte characters + # are present in the source code. + def self.for(source, start_line = 1, offsets = []) + source.ascii_only? ? ASCIISource.new(source, start_line, offsets): new(source, start_line, offsets) + end + # The source code that this source object represents. attr_reader :source @@ -27,6 +35,11 @@ module Prism source.encoding end + # Returns the lines of the source code as an array of strings. + def lines + source.lines + end + # Perform a byteslice on the source code using the given byte offset and # byte length. def slice(byte_offset, length) @@ -106,6 +119,39 @@ module Prism end end + # Specialized version of Prism::Source for source code that includes ASCII + # characters only. This class is used to apply performance optimizations that + # cannot be applied to sources that include multibyte characters. Sources that + # include multibyte characters are represented by the Prism::Source class. + class ASCIISource < Source + # Return the character offset for the given byte offset. + def character_offset(byte_offset) + byte_offset + end + + # Return the column number in characters for the given byte offset. + def character_column(byte_offset) + byte_offset - line_start(byte_offset) + end + + # Returns the offset from the start of the file for the given byte offset + # counting in code units for the given encoding. + # + # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the + # concept of code units that differs from the number of characters in other + # encodings, it is not captured here. + def code_units_offset(byte_offset, encoding) + byte_offset + end + + # Specialized version of `code_units_column` that does not depend on + # `code_units_offset`, which is a more expensive operation. This is + # essentialy the same as `Prism::Source#column`. + def code_units_column(byte_offset, encoding) + byte_offset - line_start(byte_offset) + end + end + # This represents a location in the source. class Location # A Source object that is used to determine more information from the given @@ -177,6 +223,11 @@ module Prism "#<Prism::Location @start_offset=#{@start_offset} @length=#{@length} start_line=#{start_line}>" end + # Returns all of the lines of the source code associated with this location. + def source_lines + source.lines + end + # The source code that this location represents. def slice source.slice(start_offset, length) @@ -296,6 +347,18 @@ module Prism Location.new(source, start_offset, other.end_offset - start_offset) end + + # Join this location with the first occurrence of the string in the source + # that occurs after this location on the same line, and return the new + # location. This will raise an error if the string does not exist. + def adjoin(string) + line_suffix = source.slice(end_offset, source.line_end(end_offset) - end_offset) + + line_suffix_index = line_suffix.byteindex(string) + raise "Could not find #{string}" if line_suffix_index.nil? + + Location.new(source, start_offset, length + line_suffix_index + string.bytesize) + end end # This represents a comment that was encountered during parsing. It is the @@ -511,6 +574,12 @@ module Prism # This is a result specific to the `parse` and `parse_file` methods. class ParseResult < Result + autoload :Comments, "prism/parse_result/comments" + autoload :Newlines, "prism/parse_result/newlines" + + private_constant :Comments + private_constant :Newlines + # The syntax tree that was parsed from the source code. attr_reader :value @@ -524,6 +593,17 @@ module Prism def deconstruct_keys(keys) super.merge!(value: value) end + + # Attach the list of comments to their respective locations in the tree. + def attach_comments! + Comments.new(self).attach! # steep:ignore + end + + # Walk the tree and mark nodes that are on a new line, loosely emulating + # the behavior of CRuby's `:line` tracepoint event. + def mark_newlines! + value.accept(Newlines.new(source.offsets.size)) # steep:ignore + end end # This is a result specific to the `lex` and `lex_file` methods. diff --git a/lib/prism/parse_result/comments.rb b/lib/prism/parse_result/comments.rb index f8f74d2503..22c4148b2c 100644 --- a/lib/prism/parse_result/comments.rb +++ b/lib/prism/parse_result/comments.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # When we've parsed the source, we have both the syntax tree and the list of # comments that we found in the source. This class is responsible for # walking the tree and finding the nearest location to attach each comment. @@ -183,12 +183,5 @@ module Prism [preceding, NodeTarget.new(node), following] end end - - private_constant :Comments - - # Attach the list of comments to their respective locations in the tree. - def attach_comments! - Comments.new(self).attach! # steep:ignore - end end end diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index 927c17fe2f..cc1343dfda 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # The :line tracepoint event gets fired whenever the Ruby VM encounters an # expression on a new line. The types of expressions that can trigger this # event are: @@ -17,21 +17,26 @@ module Prism # Note that the logic in this file should be kept in sync with the Java # MarkNewlinesVisitor, since that visitor is responsible for marking the # newlines for JRuby/TruffleRuby. + # + # This file is autoloaded only when `mark_newlines!` is called, so the + # re-opening of the various nodes in this file will only be performed in + # that case. We do that to avoid storing the extra `@newline` instance + # variable on every node if we don't need it. class Newlines < Visitor # Create a new Newlines visitor with the given newline offsets. - def initialize(newline_marked) - @newline_marked = newline_marked + def initialize(lines) + @lines = Array.new(1 + lines, false) end # Permit block/lambda nodes to mark newlines within themselves. def visit_block_node(node) - old_newline_marked = @newline_marked - @newline_marked = Array.new(old_newline_marked.size, false) + old_lines = @lines + @lines = Array.new(old_lines.size, false) begin super(node) ensure - @newline_marked = old_newline_marked + @lines = old_lines end end @@ -39,7 +44,7 @@ module Prism # Mark if/unless nodes as newlines. def visit_if_node(node) - node.set_newline_flag(@newline_marked) + node.newline!(@lines) super(node) end @@ -48,17 +53,101 @@ module Prism # Permit statements lists to mark newlines within themselves. def visit_statements_node(node) node.body.each do |child| - child.set_newline_flag(@newline_marked) + child.newline!(@lines) end super(node) end end + end + + class Node + def newline? # :nodoc: + @newline ? true : false + end + + def newline!(lines) # :nodoc: + line = location.start_line + unless lines[line] + lines[line] = true + @newline = true + end + end + end + + class BeginNode < Node + def newline!(lines) # :nodoc: + # Never mark BeginNode with a newline flag, mark children instead. + end + end + + class ParenthesesNode < Node + def newline!(lines) # :nodoc: + # Never mark ParenthesesNode with a newline flag, mark children instead. + end + end + + class IfNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UnlessNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UntilNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class WhileNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class RescueModifierNode < Node + def newline!(lines) # :nodoc: + expression.newline!(lines) + end + end + + class InterpolatedMatchLastLineNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedRegularExpressionNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - private_constant :Newlines + class InterpolatedSymbolNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - # Walk the tree and mark nodes that are on a new line. - def mark_newlines! - value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore + class InterpolatedXStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first end end end diff --git a/lib/prism/pattern.rb b/lib/prism/pattern.rb index e12cfd597f..03fec26789 100644 --- a/lib/prism/pattern.rb +++ b/lib/prism/pattern.rb @@ -149,7 +149,10 @@ module Prism parent = node.parent if parent.is_a?(ConstantReadNode) && parent.slice == "Prism" - compile_node(node.child) + name = node.name + raise CompilationError, node.inspect if name.nil? + + compile_constant_name(node, name) else compile_error(node) end @@ -158,14 +161,17 @@ module Prism # in ConstantReadNode # in String def compile_constant_read_node(node) - value = node.slice + compile_constant_name(node, node.name) + end - if Prism.const_defined?(value, false) - clazz = Prism.const_get(value) + # Compile a name associated with a constant. + def compile_constant_name(node, name) + if Prism.const_defined?(name, false) + clazz = Prism.const_get(name) ->(other) { clazz === other } - elsif Object.const_defined?(value, false) - clazz = Object.const_get(value) + elsif Object.const_defined?(name, false) + clazz = Object.const_get(name) ->(other) { clazz === other } else diff --git a/lib/prism/polyfill/byteindex.rb b/lib/prism/polyfill/byteindex.rb new file mode 100644 index 0000000000..98c6089f14 --- /dev/null +++ b/lib/prism/polyfill/byteindex.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Polyfill for String#byteindex, which didn't exist until Ruby 3.2. +if !("".respond_to?(:byteindex)) + String.include( + Module.new { + def byteindex(needle, offset = 0) + charindex = index(needle, offset) + slice(0...charindex).bytesize if charindex + end + } + ) +end diff --git a/lib/prism/polyfill/string.rb b/lib/prism/polyfill/unpack1.rb index 3fa9b5a0c5..3fa9b5a0c5 100644 --- a/lib/prism/polyfill/string.rb +++ b/lib/prism/polyfill/unpack1.rb diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index cfc10c1a07..b4504dbf4b 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "0.27.0" + spec.version = "0.29.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -70,7 +70,6 @@ Gem::Specification.new do |spec| "include/prism/version.h", "lib/prism.rb", "lib/prism/compiler.rb", - "lib/prism/debug.rb", "lib/prism/desugar_compiler.rb", "lib/prism/dispatcher.rb", "lib/prism/dot_visitor.rb", @@ -86,7 +85,8 @@ Gem::Specification.new do |spec| "lib/prism/parse_result/comments.rb", "lib/prism/parse_result/newlines.rb", "lib/prism/pattern.rb", - "lib/prism/polyfill/string.rb", + "lib/prism/polyfill/byteindex.rb", + "lib/prism/polyfill/unpack1.rb", "lib/prism/reflection.rb", "lib/prism/serialize.rb", "lib/prism/translation.rb", @@ -104,20 +104,15 @@ Gem::Specification.new do |spec| "prism.gemspec", "rbi/prism.rbi", "rbi/prism/compiler.rbi", - "rbi/prism/desugar_compiler.rbi", "rbi/prism/inspect_visitor.rbi", - "rbi/prism/mutation_compiler.rbi", "rbi/prism/node_ext.rbi", "rbi/prism/node.rbi", "rbi/prism/parse_result.rbi", "rbi/prism/reflection.rbi", "rbi/prism/translation/parser.rbi", - "rbi/prism/translation/parser/compiler.rbi", "rbi/prism/translation/parser33.rbi", "rbi/prism/translation/parser34.rbi", "rbi/prism/translation/ripper.rbi", - "rbi/prism/translation/ripper/ripper_compiler.rbi", - "rbi/prism/translation/ruby_parser.rbi", "rbi/prism/visitor.rbi", "sig/prism.rbs", "sig/prism/compiler.rbs", @@ -125,6 +120,7 @@ Gem::Specification.new do |spec| "sig/prism/dot_visitor.rbs", "sig/prism/dsl.rbs", "sig/prism/inspect_visitor.rbs", + "sig/prism/lex_compat.rbs", "sig/prism/mutation_compiler.rbs", "sig/prism/node_ext.rbs", "sig/prism/node.rbs", diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index 0d11b8f566..3748fc896e 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "parser" +begin + require "parser" +rescue LoadError + warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation @@ -46,7 +51,7 @@ module Prism source = source_buffer.source offset_cache = build_offset_cache(source) - result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) build_ast(result.value, offset_cache) ensure @@ -59,7 +64,7 @@ module Prism source = source_buffer.source offset_cache = build_offset_cache(source) - result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) [ build_ast(result.value, offset_cache), @@ -78,7 +83,7 @@ module Prism offset_cache = build_offset_cache(source) result = begin - unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) + unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) rescue ::Parser::SyntaxError raise if !recover end @@ -149,17 +154,17 @@ module Prism Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, []) when :embdoc_term Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, []) - when :incomplete_variable_class, :incomplete_variable_class_3_3_0 + when :incomplete_variable_class, :incomplete_variable_class_3_3 location = location.copy(length: location.length + 1) diagnostic_location = build_range(location, offset_cache) Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, []) - when :incomplete_variable_instance, :incomplete_variable_instance_3_3_0 + when :incomplete_variable_instance, :incomplete_variable_instance_3_3 location = location.copy(length: location.length + 1) diagnostic_location = build_range(location, offset_cache) Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, []) - when :invalid_variable_global, :invalid_variable_global_3_3_0 + when :invalid_variable_global, :invalid_variable_global_3_3 Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, []) when :module_in_method Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, []) @@ -284,7 +289,7 @@ module Prism def convert_for_prism(version) case version when 33 - "3.3.0" + "3.3.1" when 34 "3.4.0" else diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index a4aaa41d6f..08e54a7bf3 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -328,18 +328,48 @@ module Prism [], nil ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo.bar &&= baz # ^^^^^^^^^^^^^^^ - alias visit_call_and_write_node visit_call_operator_write_node + def visit_call_and_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar ||= baz # ^^^^^^^^^^^^^^^ - alias visit_call_or_write_node visit_call_operator_write_node + def visit_call_or_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar, = 1 # ^^^^^^^ @@ -419,18 +449,30 @@ module Prism def visit_class_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.cvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @@foo &&= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_and_write_node visit_class_variable_operator_write_node + def visit_class_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo ||= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_or_write_node visit_class_variable_operator_write_node + def visit_class_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo, = bar # ^^^^^ @@ -458,18 +500,30 @@ module Prism def visit_constant_operator_write_node(node) builder.op_assign( builder.assignable(builder.const([node.name, srange(node.name_loc)])), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo &&= bar # ^^^^^^^^^^^^ - alias visit_constant_and_write_node visit_constant_operator_write_node + def visit_constant_and_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo ||= bar # ^^^^^^^^^^^^ - alias visit_constant_or_write_node visit_constant_operator_write_node + def visit_constant_or_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo, = bar # ^^^ @@ -483,13 +537,13 @@ module Prism if node.parent.nil? builder.const_global( token(node.delimiter_loc), - [node.child.name, srange(node.child.location)] + [node.name, srange(node.name_loc)] ) else builder.const_fetch( visit(node.parent), token(node.delimiter_loc), - [node.child.name, srange(node.child.location)] + [node.name, srange(node.name_loc)] ) end end @@ -512,18 +566,30 @@ module Prism def visit_constant_path_operator_write_node(node) builder.op_assign( builder.assignable(visit(node.target)), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo::Bar &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_and_write_node visit_constant_path_operator_write_node + def visit_constant_path_and_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_or_write_node visit_constant_path_operator_write_node + def visit_constant_path_or_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar, = baz # ^^^^^^^^ @@ -711,18 +777,30 @@ module Prism def visit_global_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.gvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # $foo &&= bar # ^^^^^^^^^^^^ - alias visit_global_variable_and_write_node visit_global_variable_operator_write_node + def visit_global_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo ||= bar # ^^^^^^^^^^^^ - alias visit_global_variable_or_write_node visit_global_variable_operator_write_node + def visit_global_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo, = bar # ^^^^ @@ -803,7 +881,7 @@ module Prism # 1i # ^^ def visit_imaginary_node(node) - visit_numeric(node, builder.complex([imaginary_value(node), srange(node.location)])) + visit_numeric(node, builder.complex([Complex(0, node.numeric.value), srange(node.location)])) end # { foo: } @@ -857,18 +935,46 @@ module Prism visit_all(arguments), token(node.closing_loc) ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo[bar] &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_and_write_node visit_index_operator_write_node + def visit_index_and_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar] ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_or_write_node visit_index_operator_write_node + def visit_index_or_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar], = 1 # ^^^^^^^^ @@ -902,18 +1008,30 @@ module Prism def visit_instance_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ivar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @foo &&= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_and_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo ||= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_or_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo, = bar # ^^^^ @@ -1031,6 +1149,12 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + builder.ident([:it, srange(node.location)]).updated(:lvar) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) builder.args(nil, [], nil, false) @@ -1083,14 +1207,7 @@ module Prism # foo # ^^^ def visit_local_variable_read_node(node) - name = node.name - - # This is just a guess. parser doesn't have support for the implicit - # `it` variable yet, so we'll probably have to visit this once it - # does. - name = :it if name == :"0it" - - builder.ident([name, srange(node.location)]).updated(:lvar) + builder.ident([node.name, srange(node.location)]).updated(:lvar) end # foo = 1 @@ -1108,18 +1225,30 @@ module Prism def visit_local_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ident(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo &&= bar # ^^^^^^^^^^^ - alias visit_local_variable_and_write_node visit_local_variable_operator_write_node + def visit_local_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo ||= bar # ^^^^^^^^^^^ - alias visit_local_variable_or_write_node visit_local_variable_operator_write_node + def visit_local_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo, = bar # ^^^ @@ -1384,7 +1513,7 @@ module Prism # 1r # ^^ def visit_rational_node(node) - visit_numeric(node, builder.rational([rational_value(node), srange(node.location)])) + visit_numeric(node, builder.rational([node.value, srange(node.location)])) end # redo @@ -1810,12 +1939,6 @@ module Prism forwarding end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def imaginary_value(node) - Complex(0, node.numeric.is_a?(RationalNode) ? rational_value(node.numeric) : node.numeric.value) - end - # Negate the value of a numeric node. This is a special case where you # have a negative sign on one line and then a number on the next line. # In normal Ruby, this will always be a method call. The parser gem, @@ -1825,7 +1948,9 @@ module Prism case receiver.type when :integer_node, :float_node receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location)) - when :rational_node, :imaginary_node + when :rational_node + receiver.copy(numerator: -receiver.numerator, location: message_loc.join(receiver.location)) + when :imaginary_node receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location)) end end @@ -1844,16 +1969,6 @@ module Prism parameters.block.nil? end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def rational_value(node) - if node.numeric.is_a?(IntegerNode) - Rational(node.numeric.value) - else - Rational(node.slice.gsub(/\s/, "").chomp("r")) - end - end - # Locations in the parser gem AST are generated using this class. We # store a reference to its constant to make it slightly faster to look # up. diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index 2c5e4569c2..79ba0e7ab3 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -1181,8 +1181,8 @@ module Prism bounds(node.location) target = on_field(receiver, call_operator, message) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1339,8 +1339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_cvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1409,8 +1409,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_const(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1456,16 +1456,16 @@ module Prism # ^^^^^^^^ def visit_constant_path_node(node) if node.parent.nil? - bounds(node.child.location) - child = on_const(node.child.name.to_s) + bounds(node.name_loc) + child = on_const(node.name.to_s) bounds(node.location) on_top_const_ref(child) else parent = visit(node.parent) - bounds(node.child.location) - child = on_const(node.child.name.to_s) + bounds(node.name_loc) + child = on_const(node.name.to_s) bounds(node.location) on_const_path_ref(parent, child) @@ -1488,16 +1488,16 @@ module Prism # Visit a constant path that is part of a write node. private def visit_constant_path_write_node_target(node) if node.parent.nil? - bounds(node.child.location) - child = on_const(node.child.name.to_s) + bounds(node.name_loc) + child = on_const(node.name.to_s) bounds(node.location) on_top_const_field(child) else parent = visit(node.parent) - bounds(node.child.location) - child = on_const(node.child.name.to_s) + bounds(node.name_loc) + child = on_const(node.name.to_s) bounds(node.location) on_const_path_field(parent, child) @@ -1510,8 +1510,8 @@ module Prism target = visit_constant_path_write_node_target(node.target) value = visit(node.value) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1802,8 +1802,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_gvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1983,8 +1983,8 @@ module Prism bounds(node.location) target = on_aref_field(receiver, arguments) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2059,8 +2059,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ivar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2218,6 +2218,13 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + bounds(node.location) + on_vcall(on_ident(node.slice)) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) end @@ -2312,12 +2319,7 @@ module Prism # ^^^ def visit_local_variable_read_node(node) bounds(node.location) - - if node.name == :"0it" - on_vcall(on_ident(node.slice)) - else - on_var_ref(on_ident(node.slice)) - end + on_var_ref(on_ident(node.slice)) end # foo = 1 @@ -2337,8 +2339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ident(node.name_loc.slice)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -3267,7 +3269,11 @@ module Prism # Lazily initialize the parse result. def result - @result ||= Prism.parse(source) + @result ||= + begin + scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]] + Prism.parse(source, scopes: scopes) + end end ########################################################################## diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index a8692db5ea..ec458a3b63 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "ruby_parser" +begin + require "ruby_parser" +rescue LoadError + warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation @@ -271,9 +276,9 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_call_operator_write_node(node) if op_asgn?(node) - s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.operator) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator) else - s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.operator, visit_write_value(node.value)) + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value)) end end @@ -372,7 +377,7 @@ module Prism # @@foo += bar # ^^^^^^^^^^^^ def visit_class_variable_operator_write_node(node) - s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.operator, visit_write_value(node.value))) + s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value))) end # @@foo &&= bar @@ -417,7 +422,7 @@ module Prism # Foo += bar # ^^^^^^^^^^^ def visit_constant_operator_write_node(node) - s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.operator, visit_write_value(node.value))) + s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value))) end # Foo &&= bar @@ -442,9 +447,9 @@ module Prism # ^^^^^^^^ def visit_constant_path_node(node) if node.parent.nil? - s(node, :colon3, node.child.name) + s(node, :colon3, node.name) else - s(node, :colon2, visit(node.parent), node.child.name) + s(node, :colon2, visit(node.parent), node.name) end end @@ -460,7 +465,7 @@ module Prism # Foo::Bar += baz # ^^^^^^^^^^^^^^^ def visit_constant_path_operator_write_node(node) - s(node, :op_asgn, visit(node.target), node.operator, visit_write_value(node.value)) + s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value)) end # Foo::Bar &&= baz @@ -627,7 +632,7 @@ module Prism # $foo += bar # ^^^^^^^^^^^ def visit_global_variable_operator_write_node(node) - s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.operator, visit(node.value))) + s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value))) end # $foo &&= bar @@ -719,7 +724,7 @@ module Prism arglist << visit(node.block) if !node.block.nil? end - s(node, :op_asgn1, visit(node.receiver), arglist, node.operator, visit_write_value(node.value)) + s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value)) end # foo[bar] &&= baz @@ -775,7 +780,7 @@ module Prism # @foo += bar # ^^^^^^^^^^^ def visit_instance_variable_operator_write_node(node) - s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.operator, visit_write_value(node.value))) + s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value))) end # @foo &&= bar @@ -870,6 +875,8 @@ module Prism else visited << result end + elsif result[0] == :dstr + visited.concat(result[1..-1]) else visited << result end @@ -900,12 +907,23 @@ module Prism results << result state = :interpolated_content end - else - results << result + when :interpolated_content + if result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line) + results[-1][1] << result[1] + results[-1].line_max = result.line_max + else + results << result + end end end end + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + s(node, :call, nil, :it) + end + # foo(bar: baz) # ^^^^^^^^ def visit_keyword_hash_node(node) @@ -960,7 +978,7 @@ module Prism # foo += bar # ^^^^^^^^^^ def visit_local_variable_operator_write_node(node) - s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.operator, visit_write_value(node.value))) + s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value))) end # foo &&= bar @@ -1536,13 +1554,13 @@ module Prism # Parse the given source and translate it into the seattlerb/ruby_parser # gem's Sexp format. def parse(source, filepath = "(string)") - translate(Prism.parse(source, filepath: filepath), filepath) + translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath) end # Parse the given file and translate it into the seattlerb/ruby_parser # gem's Sexp format. def parse_file(filepath) - translate(Prism.parse_file(filepath), filepath) + translate(Prism.parse_file(filepath, scopes: [[]]), filepath) end class << self diff --git a/lib/rdoc/version.rb b/lib/rdoc/version.rb index 87842d9847..427d4ae232 100644 --- a/lib/rdoc/version.rb +++ b/lib/rdoc/version.rb @@ -5,6 +5,6 @@ module RDoc ## # RDoc version you are using - VERSION = '6.6.3.1' + VERSION = '6.7.0' end diff --git a/lib/reline.rb b/lib/reline.rb index f0060f5c9c..fb00b96531 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -225,17 +225,20 @@ module Reline journey_data = completion_journey_data return unless journey_data - target = journey_data.list[journey_data.pointer] + target = journey_data.list.first + completed = journey_data.list[journey_data.pointer] result = journey_data.list.drop(1) pointer = journey_data.pointer - 1 - return if target.empty? || (result == [target] && pointer < 0) + return if completed.empty? || (result == [completed] && pointer < 0) target_width = Reline::Unicode.calculate_width(target) - x = cursor_pos.x - target_width - if x < 0 - x = screen_width + x + completed_width = Reline::Unicode.calculate_width(completed) + if cursor_pos.x <= completed_width - target_width + # When target is rendered on the line above cursor position + x = screen_width - completed_width y = -1 else + x = [cursor_pos.x - completed_width, 0].max y = 0 end cursor_pos_to_render = Reline::CursorPos.new(x, y) @@ -309,6 +312,10 @@ module Reline $stderr.sync = true $stderr.puts "Reline is used by #{Process.pid}" end + unless config.test_mode or config.loaded? + config.read + io_gate.set_default_key_bindings(config) + end otio = io_gate.prep may_req_ambiguous_char_width @@ -335,12 +342,6 @@ module Reline end end - unless config.test_mode - config.read - config.reset_default_key_bindings - io_gate.set_default_key_bindings(config) - end - line_editor.print_nomultiline_prompt(prompt) line_editor.update_dialogs line_editor.rerender @@ -350,7 +351,15 @@ module Reline loop do read_io(config.keyseq_timeout) { |inputs| line_editor.set_pasting_state(io_gate.in_pasting?) - inputs.each { |key| line_editor.update(key) } + inputs.each do |key| + if key.char == :bracketed_paste_start + text = io_gate.read_bracketed_paste + line_editor.insert_pasted_text(text) + line_editor.scroll_into_view + else + line_editor.update(key) + end + end } if line_editor.finished? line_editor.render_finished diff --git a/lib/reline/ansi.rb b/lib/reline/ansi.rb index a3719f502c..fa9feb2630 100644 --- a/lib/reline/ansi.rb +++ b/lib/reline/ansi.rb @@ -45,6 +45,7 @@ class Reline::ANSI end def self.set_default_key_bindings(config, allow_terminfo: true) + set_bracketed_paste_key_bindings(config) set_default_key_bindings_ansi_cursor(config) if allow_terminfo && Reline::Terminfo.enabled? set_default_key_bindings_terminfo(config) @@ -66,6 +67,12 @@ class Reline::ANSI end end + def self.set_bracketed_paste_key_bindings(config) + [:emacs, :vi_insert, :vi_command].each do |keymap| + config.add_default_key_binding_by_keymap(keymap, START_BRACKETED_PASTE.bytes, :bracketed_paste_start) + end + end + def self.set_default_key_bindings_ansi_cursor(config) ANSI_CURSOR_KEY_BINDINGS.each do |char, (default_func, modifiers)| bindings = [["\e[#{char}", default_func]] # CSI + char @@ -178,46 +185,26 @@ class Reline::ANSI nil end - @@in_bracketed_paste_mode = false - START_BRACKETED_PASTE = String.new("\e[200~,", encoding: Encoding::ASCII_8BIT) - END_BRACKETED_PASTE = String.new("\e[200~.", encoding: Encoding::ASCII_8BIT) - def self.getc_with_bracketed_paste(timeout_second) + START_BRACKETED_PASTE = String.new("\e[200~", encoding: Encoding::ASCII_8BIT) + END_BRACKETED_PASTE = String.new("\e[201~", encoding: Encoding::ASCII_8BIT) + def self.read_bracketed_paste buffer = String.new(encoding: Encoding::ASCII_8BIT) - buffer << inner_getc(timeout_second) - while START_BRACKETED_PASTE.start_with?(buffer) or END_BRACKETED_PASTE.start_with?(buffer) do - if START_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = true - return inner_getc(timeout_second) - elsif END_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = false - ungetc(-1) - return inner_getc(timeout_second) - end - succ_c = inner_getc(Reline.core.config.keyseq_timeout) - - if succ_c - buffer << succ_c - else - break - end + until buffer.end_with?(END_BRACKETED_PASTE) + c = inner_getc(Float::INFINITY) + break unless c + buffer << c end - buffer.bytes.reverse_each do |ch| - ungetc ch - end - inner_getc(timeout_second) + string = buffer.delete_suffix(END_BRACKETED_PASTE).force_encoding(encoding) + string.valid_encoding? ? string : '' end # if the usage expects to wait indefinitely, use Float::INFINITY for timeout_second def self.getc(timeout_second) - if Reline.core.config.enable_bracketed_paste - getc_with_bracketed_paste(timeout_second) - else - inner_getc(timeout_second) - end + inner_getc(timeout_second) end def self.in_pasting? - @@in_bracketed_paste_mode or (not empty_buffer?) + not empty_buffer? end def self.empty_buffer? @@ -248,7 +235,7 @@ class Reline::ANSI s = [ENV["LINES"].to_i, ENV["COLUMNS"].to_i] return s if s[0] > 0 && s[1] > 0 [24, 80] - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV [24, 80] end @@ -284,7 +271,7 @@ class Reline::ANSI buf = @@output.pread(@@output.pos, 0) row = buf.count("\n") column = buf.rindex("\n") ? (buf.size - buf.rindex("\n")) - 1 : 0 - rescue Errno::ESPIPE + rescue Errno::ESPIPE, IOError # Just returns column 1 for ambiguous width because this I/O is not # tty and can't seek. row = 0 @@ -361,11 +348,15 @@ class Reline::ANSI end def self.prep + # Enable bracketed paste + @@output.write "\e[?2004h" if Reline.core.config.enable_bracketed_paste retrieve_keybuffer nil end def self.deprep(otio) + # Disable bracketed paste + @@output.write "\e[?2004l" if Reline.core.config.enable_bracketed_paste Signal.trap('WINCH', @@old_winch_handler) if @@old_winch_handler end end diff --git a/lib/reline/config.rb b/lib/reline/config.rb index d7564ba4b7..62c6d105b3 100644 --- a/lib/reline/config.rb +++ b/lib/reline/config.rb @@ -8,31 +8,12 @@ class Reline::Config end VARIABLE_NAMES = %w{ - bind-tty-special-chars - blink-matching-paren - byte-oriented completion-ignore-case convert-meta disable-completion - enable-keypad - expand-tilde - history-preserve-point history-size - horizontal-scroll-mode - input-meta keyseq-timeout - mark-directories - mark-modified-lines - mark-symlinked-directories - match-hidden-files - meta-flag - output-meta - page-completions - prefer-visible-bell - print-completions-horizontally show-all-if-ambiguous - show-all-if-unmodified - visible-stats show-mode-in-prompt vi-cmd-mode-string vi-ins-mode-string @@ -69,17 +50,15 @@ class Reline::Config @test_mode = false @autocompletion = false @convert_meta = true if seven_bit_encoding?(Reline::IOGate.encoding) + @loaded = false + @enable_bracketed_paste = true end def reset if editing_mode_is?(:vi_command) @editing_mode_label = :vi_insert end - @additional_key_bindings.keys.each do |key| - @additional_key_bindings[key].clear - end @oneshot_key_bindings.clear - reset_default_key_bindings end def editing_mode @@ -98,6 +77,10 @@ class Reline::Config @key_actors[@keymap_label] end + def loaded? + @loaded + end + def inputrc_path case ENV['INPUTRC'] when nil, '' @@ -129,6 +112,7 @@ class Reline::Config end def read(file = nil) + @loaded = true file ||= default_inputrc_path begin if file.respond_to?(:readlines) @@ -171,12 +155,6 @@ class Reline::Config @key_actors[@keymap_label].default_key_bindings[keystroke] = target end - def reset_default_key_bindings - @key_actors.values.each do |ka| - ka.reset_default_key_bindings - end - end - def read_lines(lines, file = nil) if not lines.empty? and lines.first.encoding != Reline.encoding_system_needs begin @@ -204,12 +182,14 @@ class Reline::Config next if if_stack.any? { |_no, skip| skip } case line - when /^set +([^ ]+) +([^ ]+)/i - var, value = $1.downcase, $2 - bind_variable(var, value) + when /^set +([^ ]+) +(.+)/i + # value ignores everything after a space, raw_value does not. + var, value, raw_value = $1.downcase, $2.partition(' ').first, $2 + bind_variable(var, value, raw_value) next when /\s*("#{KEYSEQ_PATTERN}+")\s*:\s*(.*)\s*$/o key, func_name = $1, $2 + func_name = func_name.split.first keystroke, func = bind_key(key, func_name) next unless keystroke @additional_key_bindings[@keymap_label][@keymap_prefix + keystroke] = func @@ -226,7 +206,13 @@ class Reline::Config when 'if' condition = false case args - when 'mode' + when /^mode=(vi|emacs)$/i + mode = $1.downcase + # NOTE: mode=vi means vi-insert mode + mode = 'vi_insert' if mode == 'vi' + if @editing_mode_label == mode.to_sym + condition = true + end when 'term' when 'version' else # application name @@ -249,7 +235,7 @@ class Reline::Config end end - def bind_variable(name, value) + def bind_variable(name, value, raw_value) case name when 'history-size' begin @@ -274,7 +260,7 @@ class Reline::Config when 'completion-query-items' @completion_query_items = value.to_i when 'isearch-terminators' - @isearch_terminators = retrieve_string(value) + @isearch_terminators = retrieve_string(raw_value) when 'editing-mode' case value when 'emacs' @@ -316,11 +302,11 @@ class Reline::Config @show_mode_in_prompt = false end when 'vi-cmd-mode-string' - @vi_cmd_mode_string = retrieve_string(value) + @vi_cmd_mode_string = retrieve_string(raw_value) when 'vi-ins-mode-string' - @vi_ins_mode_string = retrieve_string(value) + @vi_ins_mode_string = retrieve_string(raw_value) when 'emacs-mode-string' - @emacs_mode_string = retrieve_string(value) + @emacs_mode_string = retrieve_string(raw_value) when *VARIABLE_NAMES then variable_name = :"@#{name.tr(?-, ?_)}" instance_variable_set(variable_name, value.nil? || value == '1' || value == 'on') diff --git a/lib/reline/key_actor/base.rb b/lib/reline/key_actor/base.rb index a1cd7fb2a1..194e98938c 100644 --- a/lib/reline/key_actor/base.rb +++ b/lib/reline/key_actor/base.rb @@ -12,8 +12,4 @@ class Reline::KeyActor::Base def default_key_bindings @default_key_bindings end - - def reset_default_key_bindings - @default_key_bindings.clear - end end diff --git a/lib/reline/key_actor/emacs.rb b/lib/reline/key_actor/emacs.rb index 5d0a7fb63d..edd88289a3 100644 --- a/lib/reline/key_actor/emacs.rb +++ b/lib/reline/key_actor/emacs.rb @@ -19,7 +19,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 8 ^H :em_delete_prev_char, # 9 ^I - :ed_unassigned, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -63,7 +63,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 30 ^^ :ed_unassigned, # 31 ^_ - :ed_unassigned, + :undo, # 32 SPACE :ed_insert, # 33 ! diff --git a/lib/reline/key_actor/vi_insert.rb b/lib/reline/key_actor/vi_insert.rb index c3d7f9c12d..f8ccf468c6 100644 --- a/lib/reline/key_actor/vi_insert.rb +++ b/lib/reline/key_actor/vi_insert.rb @@ -19,7 +19,7 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 8 ^H :vi_delete_prev_char, # 9 ^I - :ed_insert, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -29,11 +29,11 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 13 ^M :ed_newline, # 14 ^N - :ed_insert, + :menu_complete, # 15 ^O :ed_insert, # 16 ^P - :ed_insert, + :menu_complete_backward, # 17 ^Q :ed_ignore, # 18 ^R diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index 81413505d7..23ece60220 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -4,7 +4,6 @@ require 'reline/unicode' require 'tempfile' class Reline::LineEditor - # TODO: undo # TODO: Use "private alias_method" idiom after drop Ruby 2.5. attr_reader :byte_pointer attr_accessor :confirm_multiline_termination_proc @@ -75,7 +74,7 @@ class Reline::LineEditor def initialize(config, encoding) @config = config @completion_append_character = '' - @screen_size = Reline::IOGate.get_screen_size + @screen_size = [0, 0] # Should be initialized with actual winsize in LineEditor#reset reset_variables(encoding: encoding) end @@ -235,7 +234,6 @@ class Reline::LineEditor @vi_waiting_operator_arg = nil @completion_journey_state = nil @completion_state = CompletionState::NORMAL - @completion_occurs = false @perfect_matched = nil @menu_info = nil @searching_prompt = nil @@ -252,6 +250,8 @@ class Reline::LineEditor @resized = false @cache = {} @rendered_screen = RenderedScreen.new(base_y: 0, lines: [], cursor_y: 0) + @past_lines = [] + @undoing = false reset_line end @@ -284,7 +284,7 @@ class Reline::LineEditor indent1 = @auto_indent_proc.(@buffer_of_lines.take(@line_index - 1).push(''), @line_index - 1, 0, true) indent2 = @auto_indent_proc.(@buffer_of_lines.take(@line_index), @line_index - 1, @buffer_of_lines[@line_index - 1].bytesize, false) indent = indent2 || indent1 - @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A */, '') + @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A\s*/, '') ) process_auto_indent @line_index, add_newline: true else @@ -387,7 +387,7 @@ class Reline::LineEditor next cached end *wrapped_prompts, code_line_prompt = split_by_width(prompt, width).first.compact - wrapped_lines = split_by_width(line, width, offset: calculate_width(code_line_prompt)).first.compact + wrapped_lines = split_by_width(line, width, offset: calculate_width(code_line_prompt, true)).first.compact wrapped_prompts.map { |p| [p, ''] } + [[code_line_prompt, wrapped_lines.first]] + wrapped_lines.drop(1).map { |c| ['', c] } end end @@ -414,8 +414,13 @@ class Reline::LineEditor @output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}" else x, w, content = new_items[level] - content = Reline::Unicode.take_range(content, base_x - x, width) unless x == base_x && w == width - Reline::IOGate.move_cursor_column base_x + cover_begin = base_x != 0 && new_levels[base_x - 1] == level + cover_end = new_levels[base_x + width] == level + pos = 0 + unless x == base_x && w == width + content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true) + end + Reline::IOGate.move_cursor_column x + pos @output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}" end base_x += width @@ -699,13 +704,6 @@ class Reline::LineEditor DIALOG_DEFAULT_HEIGHT = 20 - private def padding_space_with_escape_sequences(str, width) - padding_width = width - calculate_width(str, true) - # padding_width should be only positive value. But macOS and Alacritty returns negative value. - padding_width = 0 if padding_width < 0 - str + (' ' * padding_width) - end - private def dialog_range(dialog, dialog_y) x_range = dialog.column...dialog.column + dialog.width y_range = dialog_y + dialog.vertical_offset...dialog_y + dialog.vertical_offset + dialog.contents.size @@ -778,7 +776,7 @@ class Reline::LineEditor dialog.contents = contents.map.with_index do |item, i| line_sgr = i == pointer ? enhanced_sgr : default_sgr str_width = dialog.width - (scrollbar_pos.nil? ? 0 : @block_elem_width) - str = padding_space_with_escape_sequences(Reline::Unicode.take_range(item, 0, str_width), str_width) + str, = Reline::Unicode.take_mbchar_range(item, 0, str_width, padding: true) colored_content = "#{line_sgr}#{str}" if scrollbar_pos if scrollbar_pos <= (i * 2) and (i * 2 + 1) < (scrollbar_pos + bar_height) @@ -858,7 +856,7 @@ class Reline::LineEditor [target, preposing, completed, postposing] end - private def complete(list, just_show_list) + private def perform_completion(list, just_show_list) case @completion_state when CompletionState::NORMAL @completion_state = CompletionState::COMPLETION @@ -887,12 +885,12 @@ class Reline::LineEditor @completion_state = CompletionState::PERFECT_MATCH else @completion_state = CompletionState::MENU_WITH_PERFECT_MATCH - complete(list, true) if @config.show_all_if_ambiguous + perform_completion(list, true) if @config.show_all_if_ambiguous end @perfect_matched = completed else @completion_state = CompletionState::MENU - complete(list, true) if @config.show_all_if_ambiguous + perform_completion(list, true) if @config.show_all_if_ambiguous end if not just_show_list and target < completed @buffer_of_lines[@line_index] = (preposing + completed + completion_append_character.to_s + postposing).split("\n")[@line_index] || String.new(encoding: @encoding) @@ -951,7 +949,8 @@ class Reline::LineEditor unless @waiting_proc byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end else @@ -1012,7 +1011,8 @@ class Reline::LineEditor if @vi_waiting_operator byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end @kill_ring.process @@ -1067,10 +1067,6 @@ class Reline::LineEditor end private def normal_char(key) - if key.combined_char.is_a?(Symbol) - process_key(key.combined_char, key.combined_char) - return - end @multibyte_buffer << key.combined_char if @multibyte_buffer.size > 1 if @multibyte_buffer.dup.force_encoding(@encoding).valid_encoding? @@ -1113,6 +1109,7 @@ class Reline::LineEditor end def input_key(key) + save_old_buffer @config.reset_oneshot_key_bindings @dialogs.each do |dialog| if key.char.instance_of?(Symbol) and key.char == dialog.name @@ -1120,38 +1117,17 @@ class Reline::LineEditor end end if key.char.nil? + process_insert(force: true) if @first_char @eof = true end finish return end - old_lines = @buffer_of_lines.dup @first_char = false @completion_occurs = false - if @config.editing_mode_is?(:emacs, :vi_insert) and key.char == "\C-i".ord - if !@config.disable_completion - process_insert(force: true) - if @config.autocompletion - @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list(:down) - else - @completion_journey_state = nil - result = call_completion_proc - if result.is_a?(Array) - @completion_occurs = true - complete(result, false) - end - end - end - elsif @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) - # In vi mode, move completed list even if autocompletion is off - if not @config.disable_completion - process_insert(force: true) - @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list("\C-p".ord == key.char ? :up : :down) - end - elsif Symbol === key.char and respond_to?(key.char, true) + + if key.char.is_a?(Symbol) process_key(key.char, key.char) else normal_char(key) @@ -1161,12 +1137,15 @@ class Reline::LineEditor @completion_journey_state = nil end + push_past_lines unless @undoing + @undoing = false + if @in_pasting clear_dialogs return end - modified = old_lines != @buffer_of_lines + modified = @old_buffer_of_lines != @buffer_of_lines if !@completion_occurs && modified && !@config.disable_completion && @config.autocompletion # Auto complete starts only when edited process_insert(force: true) @@ -1175,6 +1154,26 @@ class Reline::LineEditor modified end + def save_old_buffer + @old_buffer_of_lines = @buffer_of_lines.dup + @old_byte_pointer = @byte_pointer.dup + @old_line_index = @line_index.dup + end + + def push_past_lines + if @old_buffer_of_lines != @buffer_of_lines + @past_lines.push([@old_buffer_of_lines, @old_byte_pointer, @old_line_index]) + end + trim_past_lines + end + + MAX_PAST_LINES = 100 + def trim_past_lines + if @past_lines.size > MAX_PAST_LINES + @past_lines.shift + end + end + def scroll_into_view _wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position if wrapped_cursor_y < screen_scroll_top @@ -1251,6 +1250,18 @@ class Reline::LineEditor process_auto_indent end + def set_current_lines(lines, byte_pointer = nil, line_index = 0) + cursor = current_byte_pointer_cursor + @buffer_of_lines = lines + @line_index = line_index + if byte_pointer + @byte_pointer = byte_pointer + else + calculate_nearest_cursor(cursor) + end + process_auto_indent + end + def retrieve_completion_block(set_completion_quote_character = false) if Reline.completer_word_break_characters.empty? word_break_regexp = nil @@ -1332,6 +1343,18 @@ class Reline::LineEditor @confirm_multiline_termination_proc.(temp_buffer.join("\n") + "\n") end + def insert_pasted_text(text) + save_old_buffer + pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) + post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) + lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines << '' if lines.empty? + @buffer_of_lines[@line_index, 1] = lines + @line_index += lines.size - 1 + @byte_pointer = @buffer_of_lines[@line_index].bytesize - post.bytesize + push_past_lines + end + def insert_text(text) if @buffer_of_lines[@line_index].bytesize == @byte_pointer @buffer_of_lines[@line_index] += text @@ -1430,13 +1453,42 @@ class Reline::LineEditor end end - private def completion_journey_up(key) - if not @config.disable_completion and @config.autocompletion + private def complete(_key) + return if @config.disable_completion + + process_insert(force: true) + if @config.autocompletion @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list(:up) + @completion_occurs = move_completed_list(:down) + else + @completion_journey_state = nil + result = call_completion_proc + if result.is_a?(Array) + @completion_occurs = true + perform_completion(result, false) + end end end - alias_method :menu_complete_backward, :completion_journey_up + + private def completion_journey_move(direction) + return if @config.disable_completion + + process_insert(force: true) + @completion_state = CompletionState::NORMAL + @completion_occurs = move_completed_list(direction) + end + + private def menu_complete(_key) + completion_journey_move(:down) + end + + private def menu_complete_backward(_key) + completion_journey_move(:up) + end + + private def completion_journey_up(_key) + completion_journey_move(:up) if @config.autocompletion + end # Editline:: +ed-unassigned+ This editor command always results in an error. # GNU Readline:: There is no corresponding macro. @@ -1905,7 +1957,7 @@ class Reline::LineEditor elsif !@config.autocompletion # show completed list result = call_completion_proc if result.is_a?(Array) - complete(result, true) + perform_completion(result, true) end end end @@ -2475,4 +2527,15 @@ class Reline::LineEditor private def vi_editing_mode(key) @config.editing_mode = :vi_insert end + + private def undo(_key) + return if @past_lines.empty? + + @undoing = true + + target_lines, target_cursor_x, target_cursor_y = @past_lines.last + set_current_lines(target_lines, target_cursor_x, target_cursor_y) + + @past_lines.pop + end end diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index 7f94e95287..d7460d6d4a 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -43,11 +43,13 @@ class Reline::Unicode def self.escape_for_print(str) str.chars.map! { |gr| - escaped = EscapedPairs[gr.ord] - if escaped && gr != -"\n" && gr != -"\t" - escaped - else + case gr + when -"\n" gr + when -"\t" + -' ' + else + EscapedPairs[gr.ord] || gr end }.join end @@ -179,32 +181,78 @@ class Reline::Unicode # Take a chunk of a String cut by width with escape sequences. def self.take_range(str, start_col, max_width) + take_mbchar_range(str, start_col, max_width).first + end + + def self.take_mbchar_range(str, start_col, width, cover_begin: false, cover_end: false, padding: false) chunk = String.new(encoding: str.encoding) + + end_col = start_col + width total_width = 0 rest = str.encode(Encoding::UTF_8) in_zero_width = false + chunk_start_col = nil + chunk_end_col = nil + has_csi = false rest.scan(WIDTH_SCANNER) do |non_printing_start, non_printing_end, csi, osc, gc| case when non_printing_start in_zero_width = true + chunk << NON_PRINTING_START when non_printing_end in_zero_width = false + chunk << NON_PRINTING_END when csi + has_csi = true chunk << csi when osc chunk << osc when gc if in_zero_width chunk << gc + next + end + + mbchar_width = get_mbchar_width(gc) + prev_width = total_width + total_width += mbchar_width + + if (cover_begin || padding ? total_width <= start_col : prev_width < start_col) + # Current character haven't reached start_col yet + next + elsif padding && !cover_begin && prev_width < start_col && start_col < total_width + # Add preceding padding. This padding might have background color. + chunk << ' ' + chunk_start_col ||= start_col + chunk_end_col = total_width + next + elsif (cover_end ? prev_width < end_col : total_width <= end_col) + # Current character is in the range + chunk << gc + chunk_start_col ||= prev_width + chunk_end_col = total_width + break if total_width >= end_col else - mbchar_width = get_mbchar_width(gc) - total_width += mbchar_width - break if (start_col + max_width) < total_width - chunk << gc if start_col < total_width + # Current character exceeds end_col + if padding && end_col < total_width + # Add succeeding padding. This padding might have background color. + chunk << ' ' + chunk_start_col ||= prev_width + chunk_end_col = end_col + end + break end end end - chunk + chunk_start_col ||= start_col + chunk_end_col ||= start_col + if padding && chunk_end_col < end_col + # Append padding. This padding should not include background color. + chunk << "\e[0m" if has_csi + chunk << ' ' * (end_col - chunk_end_col) + chunk_end_col = end_col + end + [chunk, chunk_start_col, chunk_end_col - chunk_start_col] end def self.get_next_mbchar_size(line, byte_pointer) diff --git a/lib/reline/version.rb b/lib/reline/version.rb index d68c7d203b..46613a5952 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.5.3' + VERSION = '0.5.7' end diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index 2346c92bd1..f9450241c9 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -5461,6 +5461,12 @@ module RubyVM::RJIT return CantCompile end + if c_method_tracing_currently_enabled? + # Don't JIT if tracing c_call or c_return + asm.incr_counter(:send_cfunc_tracing) + return CantCompile + end + off = cme.def.body.optimized.index recv_idx = argc # blockarg is not supported diff --git a/lib/rubygems.rb b/lib/rubygems.rb index ad7ab10756..ac225ca70a 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -1013,6 +1013,13 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end ## + # Is this platform FreeBSD + + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + + ## # Load +plugins+ as Ruby files def self.load_plugin_files(plugins) # :nodoc: diff --git a/lib/rubygems/commands/pristine_command.rb b/lib/rubygems/commands/pristine_command.rb index 456d897df2..b272a15b6c 100644 --- a/lib/rubygems/commands/pristine_command.rb +++ b/lib/rubygems/commands/pristine_command.rb @@ -57,7 +57,7 @@ class Gem::Commands::PristineCommand < Gem::Command end add_option("-i", "--install-dir DIR", - "Gem repository to get binstubs and plugins installed") do |value, options| + "Gem repository to get gems restored") do |value, options| options[:install_dir] = File.expand_path(value) end @@ -103,21 +103,25 @@ extensions will be restored. end def execute + install_dir = options[:install_dir] + + specification_record = install_dir ? Gem::SpecificationRecord.from_path(install_dir) : Gem::Specification.specification_record + specs = if options[:all] - Gem::Specification.map + specification_record.map # `--extensions` must be explicitly given to pristine only gems # with extensions. elsif options[:extensions_set] && options[:extensions] && options[:args].empty? - Gem::Specification.select do |spec| + specification_record.select do |spec| spec.extensions && !spec.extensions.empty? end elsif options[:only_missing_extensions] - Gem::Specification.select(&:missing_extensions?) + specification_record.select(&:missing_extensions?) else get_all_gem_names.sort.map do |gem_name| - Gem::Specification.find_all_by_name(gem_name, options[:version]).reverse + specification_record.find_all_by_name(gem_name, options[:version]).reverse end.flatten end @@ -176,7 +180,6 @@ extensions will be restored. end bin_dir = options[:bin_dir] if options[:bin_dir] - install_dir = options[:install_dir] if options[:install_dir] installer_options = { wrappers: true, diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb index 3f38074280..9c633d6ef7 100644 --- a/lib/rubygems/commands/setup_command.rb +++ b/lib/rubygems/commands/setup_command.rb @@ -585,6 +585,8 @@ abort "#{deprecation_message}" args = %w[--all --only-executables --silent] args << "--bindir=#{bindir}" + args << "--install-dir=#{default_dir}" + if options[:env_shebang] args << "--env-shebang" end diff --git a/lib/rubygems/commands/uninstall_command.rb b/lib/rubygems/commands/uninstall_command.rb index 2a77ec72cf..283bc96ce3 100644 --- a/lib/rubygems/commands/uninstall_command.rb +++ b/lib/rubygems/commands/uninstall_command.rb @@ -184,7 +184,7 @@ that is a dependency of an existing gem. You can use the rescue Gem::GemNotInHomeException => e spec = e.spec alert("In order to remove #{spec.name}, please execute:\n" \ - "\tgem uninstall #{spec.name} --install-dir=#{spec.installation_path}") + "\tgem uninstall #{spec.name} --install-dir=#{spec.base_dir}") rescue Gem::UninstallError => e spec = e.spec alert_error("Error: unable to successfully uninstall '#{spec.name}' which is " \ diff --git a/lib/rubygems/dependency.rb b/lib/rubygems/dependency.rb index d1bf074441..5ce9c5e840 100644 --- a/lib/rubygems/dependency.rb +++ b/lib/rubygems/dependency.rb @@ -271,15 +271,7 @@ class Gem::Dependency end def matching_specs(platform_only = false) - env_req = Gem.env_requirement(name) - matches = Gem::Specification.stubs_for(name).find_all do |spec| - requirement.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) - end.map(&:to_spec) - - if prioritizes_bundler? - require_relative "bundler_version_finder" - Gem::BundlerVersionFinder.prioritize!(matches) - end + matches = Gem::Specification.find_all_by_name(name, requirement) if platform_only matches.reject! do |spec| @@ -297,10 +289,6 @@ class Gem::Dependency @requirement.specific? end - def prioritizes_bundler? - name == "bundler" && !specific? - end - def to_specs matches = matching_specs true diff --git a/lib/rubygems/deprecate.rb b/lib/rubygems/deprecate.rb index 58a6c5b7dc..7d24f9cbfc 100644 --- a/lib/rubygems/deprecate.rb +++ b/lib/rubygems/deprecate.rb @@ -69,99 +69,101 @@ # end # end -module Gem::Deprecate - def self.skip # :nodoc: - @skip ||= false - end +module Gem + module Deprecate + def self.skip # :nodoc: + @skip ||= false + end - def self.skip=(v) # :nodoc: - @skip = v - end + def self.skip=(v) # :nodoc: + @skip = v + end - ## - # Temporarily turn off warnings. Intended for tests only. + ## + # Temporarily turn off warnings. Intended for tests only. - def skip_during - original = Gem::Deprecate.skip - Gem::Deprecate.skip = true - yield - ensure - Gem::Deprecate.skip = original - end + def skip_during + original = Gem::Deprecate.skip + Gem::Deprecate.skip = true + yield + ensure + Gem::Deprecate.skip = original + end - def self.next_rubygems_major_version # :nodoc: - Gem::Version.new(Gem.rubygems_version.segments.first).bump - end + def self.next_rubygems_major_version # :nodoc: + Gem::Version.new(Gem.rubygems_version.segments.first).bump + end - ## - # Simple deprecation method that deprecates +name+ by wrapping it up - # in a dummy method. It warns on each call to the dummy method - # telling the user of +repl+ (unless +repl+ is :none) and the - # year/month that it is planned to go away. + ## + # Simple deprecation method that deprecates +name+ by wrapping it up + # in a dummy method. It warns on each call to the dummy method + # telling the user of +repl+ (unless +repl+ is :none) and the + # year/month that it is planned to go away. - def deprecate(name, repl, year, month) - class_eval do - old = "_deprecated_#{name}" - alias_method old, name - define_method name do |*args, &block| - klass = is_a? Module - target = klass ? "#{self}." : "#{self.class}#" - msg = [ - "NOTE: #{target}#{name} is deprecated", - repl == :none ? " with no replacement" : "; use #{repl} instead", - format(". It will be removed on or after %4d-%02d.", year, month), - "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", - ] - warn "#{msg.join}." unless Gem::Deprecate.skip - send old, *args, &block + def deprecate(name, repl, year, month) + class_eval do + old = "_deprecated_#{name}" + alias_method old, name + define_method name do |*args, &block| + klass = is_a? Module + target = klass ? "#{self}." : "#{self.class}#" + msg = [ + "NOTE: #{target}#{name} is deprecated", + repl == :none ? " with no replacement" : "; use #{repl} instead", + format(". It will be removed on or after %4d-%02d.", year, month), + "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", + ] + warn "#{msg.join}." unless Gem::Deprecate.skip + send old, *args, &block + end + ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - end - ## - # Simple deprecation method that deprecates +name+ by wrapping it up - # in a dummy method. It warns on each call to the dummy method - # telling the user of +repl+ (unless +repl+ is :none) and the - # Rubygems version that it is planned to go away. + ## + # Simple deprecation method that deprecates +name+ by wrapping it up + # in a dummy method. It warns on each call to the dummy method + # telling the user of +repl+ (unless +repl+ is :none) and the + # Rubygems version that it is planned to go away. - def rubygems_deprecate(name, replacement=:none) - class_eval do - old = "_deprecated_#{name}" - alias_method old, name - define_method name do |*args, &block| - klass = is_a? Module - target = klass ? "#{self}." : "#{self.class}#" - msg = [ - "NOTE: #{target}#{name} is deprecated", - replacement == :none ? " with no replacement" : "; use #{replacement} instead", - ". It will be removed in Rubygems #{Gem::Deprecate.next_rubygems_major_version}", - "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", - ] - warn "#{msg.join}." unless Gem::Deprecate.skip - send old, *args, &block + def rubygems_deprecate(name, replacement=:none) + class_eval do + old = "_deprecated_#{name}" + alias_method old, name + define_method name do |*args, &block| + klass = is_a? Module + target = klass ? "#{self}." : "#{self.class}#" + msg = [ + "NOTE: #{target}#{name} is deprecated", + replacement == :none ? " with no replacement" : "; use #{replacement} instead", + ". It will be removed in Rubygems #{Gem::Deprecate.next_rubygems_major_version}", + "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", + ] + warn "#{msg.join}." unless Gem::Deprecate.skip + send old, *args, &block + end + ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - end - # Deprecation method to deprecate Rubygems commands - def rubygems_deprecate_command(version = Gem::Deprecate.next_rubygems_major_version) - class_eval do - define_method "deprecated?" do - true - end + # Deprecation method to deprecate Rubygems commands + def rubygems_deprecate_command(version = Gem::Deprecate.next_rubygems_major_version) + class_eval do + define_method "deprecated?" do + true + end - define_method "deprecation_warning" do - msg = [ - "#{command} command is deprecated", - ". It will be removed in Rubygems #{version}.\n", - ] + define_method "deprecation_warning" do + msg = [ + "#{command} command is deprecated", + ". It will be removed in Rubygems #{version}.\n", + ] - alert_warning msg.join.to_s unless Gem::Deprecate.skip + alert_warning msg.join.to_s unless Gem::Deprecate.skip + end end end - end - module_function :rubygems_deprecate, :rubygems_deprecate_command, :skip_during + module_function :rubygems_deprecate, :rubygems_deprecate_command, :skip_during + end end diff --git a/lib/rubygems/ext/cargo_builder.rb b/lib/rubygems/ext/cargo_builder.rb index 86a0e73f28..09ad1407c2 100644 --- a/lib/rubygems/ext/cargo_builder.rb +++ b/lib/rubygems/ext/cargo_builder.rb @@ -184,6 +184,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder end def cargo_dylib_path(dest_path, crate_name) + so_ext = RbConfig::CONFIG["SOEXT"] prefix = so_ext == "dll" ? "" : "lib" path_parts = [dest_path] path_parts << ENV["CARGO_BUILD_TARGET"] if ENV["CARGO_BUILD_TARGET"] @@ -312,22 +313,6 @@ EOF deffile_path end - # We have to basically reimplement <code>RbConfig::CONFIG['SOEXT']</code> here to support - # Ruby < 2.5 - # - # @see https://github.com/ruby/ruby/blob/c87c027f18c005460746a74c07cd80ee355b16e4/configure.ac#L3185 - def so_ext - return RbConfig::CONFIG["SOEXT"] if RbConfig::CONFIG.key?("SOEXT") - - if win_target? - "dll" - elsif darwin_target? - "dylib" - else - "so" - end - end - # Corresponds to $(LIBPATH) in mkmf def mkmf_libpath ["-L", "native=#{makefile_config("libdir")}"] diff --git a/lib/rubygems/gemcutter_utilities/webauthn_poller.rb b/lib/rubygems/gemcutter_utilities/webauthn_poller.rb index 0fdd1d5bf4..fe3f163a88 100644 --- a/lib/rubygems/gemcutter_utilities/webauthn_poller.rb +++ b/lib/rubygems/gemcutter_utilities/webauthn_poller.rb @@ -69,8 +69,10 @@ module Gem::GemcutterUtilities rubygems_api_request(:get, "api/v1/webauthn_verification/#{webauthn_token}/status.json") do |request| if credentials.empty? request.add_field "Authorization", api_key + elsif credentials[:identifier] && credentials[:password] + request.basic_auth credentials[:identifier], credentials[:password] else - request.basic_auth credentials[:email], credentials[:password] + raise Gem::WebauthnVerificationError, "Provided missing credentials" end end end diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index 8f6f9a5aa8..844f292ba2 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -344,7 +344,7 @@ class Gem::Installer say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? - Gem::Specification.add_spec(spec) + Gem::Specification.add_spec(spec) unless @install_dir load_plugin diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 1d5d764237..c855423ed7 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -7,7 +7,6 @@ # rubocop:enable Style/AsciiComments -require_relative "../rubygems" require_relative "security" require_relative "user_interaction" @@ -295,7 +294,6 @@ class Gem::Package Gem.load_yaml - @spec.mark_version @spec.validate true, strict_validation unless skip_validation setup_signer( @@ -528,12 +526,13 @@ EOM # Loads a Gem::Specification from the TarEntry +entry+ def load_spec(entry) # :nodoc: + limit = 10 * 1024 * 1024 case entry.full_name when "metadata" then - @spec = Gem::Specification.from_yaml entry.read + @spec = Gem::Specification.from_yaml limit_read(entry, "metadata", limit) when "metadata.gz" then Zlib::GzipReader.wrap(entry, external_encoding: Encoding::UTF_8) do |gzio| - @spec = Gem::Specification.from_yaml gzio.read + @spec = Gem::Specification.from_yaml limit_read(gzio, "metadata.gz", limit) end end end @@ -557,7 +556,7 @@ EOM @checksums = gem.seek "checksums.yaml.gz" do |entry| Zlib::GzipReader.wrap entry do |gz_io| - Gem::SafeYAML.safe_load gz_io.read + Gem::SafeYAML.safe_load limit_read(gz_io, "checksums.yaml.gz", 10 * 1024 * 1024) end end end @@ -664,7 +663,7 @@ EOM case file_name when /\.sig$/ then - @signatures[$`] = entry.read if @security_policy + @signatures[$`] = limit_read(entry, file_name, 1024 * 1024) if @security_policy return else digest entry @@ -724,6 +723,12 @@ EOM IO.copy_stream(src, dst) end end + + def limit_read(io, name, limit) + bytes = io.read(limit + 1) + raise Gem::Package::FormatError, "#{name} is too big (over #{limit} bytes)" if bytes.size > limit + bytes + end end require_relative "package/digest_io" diff --git a/lib/rubygems/package/tar_header.rb b/lib/rubygems/package/tar_header.rb index 087f13f6c9..dd5e835a1e 100644 --- a/lib/rubygems/package/tar_header.rb +++ b/lib/rubygems/package/tar_header.rb @@ -95,14 +95,14 @@ class Gem::Package::TarHeader attr_reader(*FIELDS) - EMPTY_HEADER = ("\0" * 512).freeze # :nodoc: + EMPTY_HEADER = ("\0" * 512).b.freeze # :nodoc: ## # Creates a tar header from IO +stream+ def self.from(stream) header = stream.read 512 - empty = (header == EMPTY_HEADER) + return EMPTY if header == EMPTY_HEADER fields = header.unpack UNPACK_FORMAT @@ -123,7 +123,7 @@ class Gem::Package::TarHeader devminor: strict_oct(fields.shift), prefix: fields.shift, - empty: empty + empty: false end def self.strict_oct(str) @@ -172,6 +172,22 @@ class Gem::Package::TarHeader @empty = vals[:empty] end + EMPTY = new({ # :nodoc: + checksum: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + name: "", + prefix: "", + size: 0, + uname: "", + version: 0, + + empty: true, + }).freeze + private_constant :EMPTY + ## # Is the tar entry empty? @@ -241,7 +257,7 @@ class Gem::Package::TarHeader header = header.pack PACK_FORMAT - header << ("\0" * ((512 - header.size) % 512)) + header.ljust 512, "\0" end def oct(num, len) diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb index 48b7344aee..d54ad12880 100644 --- a/lib/rubygems/platform.rb +++ b/lib/rubygems/platform.rb @@ -134,6 +134,7 @@ class Gem::Platform when /netbsdelf/ then ["netbsdelf", nil] when /openbsd(\d+\.\d+)?/ then ["openbsd", $1] when /solaris(\d+\.\d+)?/ then ["solaris", $1] + when /wasi/ then ["wasi", nil] # test when /^(\w+_platform)(\d+)?/ then [$1, $2] else ["unknown", nil] diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 29139cf725..57f9b45cf7 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -11,6 +11,7 @@ require_relative "deprecate" require_relative "basic_specification" require_relative "stub_specification" require_relative "platform" +require_relative "specification_record" require_relative "util/list" require "rbconfig" @@ -179,22 +180,12 @@ class Gem::Specification < Gem::BasicSpecification @@default_value[k].nil? end - def self.clear_specs # :nodoc: - @@all = nil - @@stubs = nil - @@stubs_by_name = {} - @@spec_with_requirable_file = {} - @@active_stub_with_requirable_file = {} - end - private_class_method :clear_specs - - clear_specs - # Sentinel object to represent "not found" stubs NOT_FOUND = Struct.new(:to_spec, :this).new # :nodoc: + deprecate_constant :NOT_FOUND # Tracking removed method calls to warn users during build time. - REMOVED_METHODS = [:rubyforge_project=].freeze # :nodoc: + REMOVED_METHODS = [:rubyforge_project=, :mark_version].freeze # :nodoc: def removed_method_calls @removed_method_calls ||= [] end @@ -770,7 +761,7 @@ class Gem::Specification < Gem::BasicSpecification attr_accessor :specification_version def self._all # :nodoc: - @@all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + specification_record.all end def self.clear_load_cache # :nodoc: @@ -788,26 +779,9 @@ class Gem::Specification < Gem::BasicSpecification end end - def self.gemspec_stubs_in(dir, pattern) + def self.gemspec_stubs_in(dir, pattern) # :nodoc: Gem::Util.glob_files_in_dir(pattern, dir).map {|path| yield path }.select(&:valid?) end - private_class_method :gemspec_stubs_in - - def self.installed_stubs(dirs, pattern) - map_stubs(dirs, pattern) do |path, base_dir, gems_dir| - Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) - end - end - private_class_method :installed_stubs - - def self.map_stubs(dirs, pattern) # :nodoc: - dirs.flat_map do |dir| - base_dir = File.dirname dir - gems_dir = File.join base_dir, "gems" - gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } - end - end - private_class_method :map_stubs def self.each_spec(dirs) # :nodoc: each_gemspec(dirs) do |path| @@ -820,13 +794,7 @@ class Gem::Specification < Gem::BasicSpecification # Returns a Gem::StubSpecification for every installed gem def self.stubs - @@stubs ||= begin - pattern = "*.gemspec" - stubs = stubs_for_pattern(pattern, false) - - @@stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) - stubs - end + specification_record.stubs end ## @@ -845,13 +813,7 @@ class Gem::Specification < Gem::BasicSpecification # only returns stubs that match Gem.platforms def self.stubs_for(name) - if @@stubs - @@stubs_by_name[name] || [] - else - @@stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| - s.name == name - end - end + specification_record.stubs_for(name) end ## @@ -859,12 +821,7 @@ class Gem::Specification < Gem::BasicSpecification # optionally filtering out specs not matching the current platform # def self.stubs_for_pattern(pattern, match_platform = true) # :nodoc: - installed_stubs = installed_stubs(Gem::Specification.dirs, pattern) - installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform - stubs = installed_stubs + default_stubs(pattern) - stubs = stubs.uniq(&:full_name) - _resort!(stubs) - stubs + specification_record.stubs_for_pattern(pattern, match_platform) end def self._resort!(specs) # :nodoc: @@ -893,23 +850,14 @@ class Gem::Specification < Gem::BasicSpecification # properly sorted. def self.add_spec(spec) - return if _all.include? spec - - _all << spec - stubs << spec - (@@stubs_by_name[spec.name] ||= []) << spec - - _resort!(@@stubs_by_name[spec.name]) - _resort!(stubs) + specification_record.add_spec(spec) end ## # Removes +spec+ from the known specs. def self.remove_spec(spec) - _all.delete spec.to_spec - stubs.delete spec - (@@stubs_by_name[spec.name] || []).delete spec + specification_record.remove_spec(spec) end ## @@ -923,27 +871,17 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the known specs to +specs+. Not guaranteed to work for you in - # the future. Use at your own risk. Caveat emptor. Doomy doom doom. - # Etc etc. - # - #-- - # Makes +specs+ the known specs - # Listen, time is a river - # Winter comes, code breaks - # - # -- wilsonb + # Sets the known specs to +specs+. def self.all=(specs) - @@stubs_by_name = specs.group_by(&:name) - @@all = @@stubs = specs + specification_record.all = specs end ## # Return full names of all specs in sorted order. def self.all_names - _all.map(&:full_name) + specification_record.all_names end ## @@ -968,9 +906,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the directories that Specification uses to find specs. def self.dirs - @@dirs ||= Gem.path.collect do |dir| - File.join dir, "specifications" - end + @@dirs ||= Gem::SpecificationRecord.dirs_from(Gem.path) end ## @@ -980,7 +916,7 @@ class Gem::Specification < Gem::BasicSpecification def self.dirs=(dirs) reset - @@dirs = Array(dirs).map {|dir| File.join dir, "specifications" } + @@dirs = Gem::SpecificationRecord.dirs_from(Array(dirs)) end extend Enumerable @@ -989,21 +925,15 @@ class Gem::Specification < Gem::BasicSpecification # Enumerate every known spec. See ::dirs= and ::add_spec to set the list of # specs. - def self.each - return enum_for(:each) unless block_given? - - _all.each do |x| - yield x - end + def self.each(&block) + specification_record.each(&block) end ## # Returns every spec that matches +name+ and optional +requirements+. def self.find_all_by_name(name, *requirements) - requirements = Gem::Requirement.default if requirements.empty? - - Gem::Dependency.new(name, *requirements).matching_specs + specification_record.find_all_by_name(name, *requirements) end ## @@ -1033,12 +963,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the best specification that contains the file matching +path+. def self.find_by_path(path) - path = path.dup.freeze - spec = @@spec_with_requirable_file[path] ||= stubs.find do |s| - s.contains_requirable_file? path - end || NOT_FOUND - - spec.to_spec + specification_record.find_by_path(path) end ## @@ -1046,19 +971,15 @@ class Gem::Specification < Gem::BasicSpecification # amongst the specs that are not activated. def self.find_inactive_by_path(path) - stub = stubs.find do |s| - next if s.activated? - s.contains_requirable_file? path - end - stub&.to_spec + specification_record.find_inactive_by_path(path) end - def self.find_active_stub_by_path(path) - stub = @@active_stub_with_requirable_file[path] ||= stubs.find do |s| - s.activated? && s.contains_requirable_file?(path) - end || NOT_FOUND + ## + # Return the best specification that contains the file matching +path+, among + # those already activated. - stub.this + def self.find_active_stub_by_path(path) + specification_record.find_active_stub_by_path(path) end ## @@ -1125,14 +1046,14 @@ class Gem::Specification < Gem::BasicSpecification # +prerelease+ is true. def self.latest_specs(prerelease = false) - _latest_specs Gem::Specification.stubs, prerelease + specification_record.latest_specs(prerelease) end ## # Return the latest installed spec for gem +name+. def self.latest_spec_for(name) - latest_specs(true).find {|installed_spec| installed_spec.name == name } + specification_record.latest_spec_for(name) end def self._latest_specs(specs, prerelease = false) # :nodoc: @@ -1270,7 +1191,7 @@ class Gem::Specification < Gem::BasicSpecification def self.reset @@dirs = nil Gem.pre_reset_hooks.each(&:call) - clear_specs + @specification_record = nil clear_load_cache unresolved = unresolved_deps unless unresolved.empty? @@ -1291,6 +1212,13 @@ class Gem::Specification < Gem::BasicSpecification Gem.post_reset_hooks.each(&:call) end + ## + # Keeps track of all currently known specifications + + def self.specification_record + @specification_record ||= Gem::SpecificationRecord.new(dirs) + end + # DOC: This method needs documented or nodoc'd def self.unresolved_deps @unresolved_deps ||= Hash.new {|h, n| h[n] = Gem::Dependency.new n } @@ -1874,8 +1802,6 @@ class Gem::Specification < Gem::BasicSpecification end def encode_with(coder) # :nodoc: - mark_version - coder.add "name", @name coder.add "version", @version platform = case @original_platform @@ -2171,13 +2097,6 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the rubygems_version to the current RubyGems version. - - def mark_version - @rubygems_version = Gem::VERSION - end - - ## # Track removed method calls to warn about during build time. # Warn about unknown attributes while loading a spec. @@ -2494,7 +2413,6 @@ class Gem::Specification < Gem::BasicSpecification # still have their default values are omitted. def to_ruby - mark_version result = [] result << "# -*- encoding: utf-8 -*-" result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}" diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index 516c26f53c..812b0f889e 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -274,7 +274,9 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: return if rubygems_version == Gem::VERSION - error "expected RubyGems version #{Gem::VERSION}, was #{rubygems_version}" + warning "expected RubyGems version #{Gem::VERSION}, was #{rubygems_version}" + + @specification.rubygems_version = Gem::VERSION end def validate_required_attributes diff --git a/lib/rubygems/specification_record.rb b/lib/rubygems/specification_record.rb new file mode 100644 index 0000000000..dd6aa7eafa --- /dev/null +++ b/lib/rubygems/specification_record.rb @@ -0,0 +1,213 @@ +# frozen_string_literal: true + +module Gem + class SpecificationRecord + def self.dirs_from(paths) + paths.map do |path| + File.join(path, "specifications") + end + end + + def self.from_path(path) + new(dirs_from([path])) + end + + def initialize(dirs) + @all = nil + @stubs = nil + @stubs_by_name = {} + @spec_with_requirable_file = {} + @active_stub_with_requirable_file = {} + + @dirs = dirs + end + + # Sentinel object to represent "not found" stubs + NOT_FOUND = Struct.new(:to_spec, :this).new + private_constant :NOT_FOUND + + ## + # Returns the list of all specifications in the record + + def all + @all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + + def stubs + @stubs ||= begin + pattern = "*.gemspec" + stubs = stubs_for_pattern(pattern, false) + + @stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) + stubs + end + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + # named +name+ only returns stubs that match Gem.platforms + + def stubs_for(name) + if @stubs + @stubs_by_name[name] || [] + else + @stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| + s.name == name + end + end + end + + ## + # Finds stub specifications matching a pattern in the record, optionally + # filtering out specs not matching the current platform + + def stubs_for_pattern(pattern, match_platform = true) + installed_stubs = installed_stubs(pattern) + installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform + stubs = installed_stubs + Gem::Specification.default_stubs(pattern) + stubs = stubs.uniq(&:full_name) + Gem::Specification._resort!(stubs) + stubs + end + + ## + # Adds +spec+ to the the record, keeping the collection properly sorted. + + def add_spec(spec) + return if all.include? spec + + all << spec + stubs << spec + (@stubs_by_name[spec.name] ||= []) << spec + + Gem::Specification._resort!(@stubs_by_name[spec.name]) + Gem::Specification._resort!(stubs) + end + + ## + # Removes +spec+ from the record. + + def remove_spec(spec) + all.delete spec.to_spec + stubs.delete spec + (@stubs_by_name[spec.name] || []).delete spec + end + + ## + # Sets the specs known by the record to +specs+. + + def all=(specs) + @stubs_by_name = specs.group_by(&:name) + @all = @stubs = specs + end + + ## + # Return full names of all specs in the record in sorted order. + + def all_names + all.map(&:full_name) + end + + include Enumerable + + ## + # Enumerate every known spec. + + def each + return enum_for(:each) unless block_given? + + all.each do |x| + yield x + end + end + + ## + # Returns every spec in the record that matches +name+ and optional +requirements+. + + def find_all_by_name(name, *requirements) + req = Gem::Requirement.create(*requirements) + env_req = Gem.env_requirement(name) + + matches = stubs_for(name).find_all do |spec| + req.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) + end.map(&:to_spec) + + if name == "bundler" && !req.specific? + require_relative "bundler_version_finder" + Gem::BundlerVersionFinder.prioritize!(matches) + end + + matches + end + + ## + # Return the best specification in the record that contains the file matching +path+. + + def find_by_path(path) + path = path.dup.freeze + spec = @spec_with_requirable_file[path] ||= stubs.find do |s| + s.contains_requirable_file? path + end || NOT_FOUND + + spec.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+ amongst the specs that are not activated. + + def find_inactive_by_path(path) + stub = stubs.find do |s| + next if s.activated? + s.contains_requirable_file? path + end + stub&.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+, among those already activated. + + def find_active_stub_by_path(path) + stub = @active_stub_with_requirable_file[path] ||= stubs.find do |s| + s.activated? && s.contains_requirable_file?(path) + end || NOT_FOUND + + stub.this + end + + ## + # Return the latest specs in the record, optionally including prerelease + # specs if +prerelease+ is true. + + def latest_specs(prerelease) + Gem::Specification._latest_specs stubs, prerelease + end + + ## + # Return the latest installed spec in the record for gem +name+. + + def latest_spec_for(name) + latest_specs(true).find {|installed_spec| installed_spec.name == name } + end + + private + + def installed_stubs(pattern) + map_stubs(pattern) do |path, base_dir, gems_dir| + Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) + end + end + + def map_stubs(pattern) + @dirs.flat_map do |dir| + base_dir = File.dirname dir + gems_dir = File.join base_dir, "gems" + Gem::Specification.gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } + end + end + end +end diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index c96df2a085..4d72f6fd0a 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -32,7 +32,7 @@ class Gem::Uninstaller attr_reader :bin_dir ## - # The gem repository the gem will be installed into + # The gem repository the gem will be uninstalled from attr_reader :gem_home @@ -49,8 +49,9 @@ class Gem::Uninstaller # TODO: document the valid options @gem = gem @version = options[:version] || Gem::Requirement.default - @gem_home = File.realpath(options[:install_dir] || Gem.dir) - @plugins_dir = Gem.plugindir(@gem_home) + @install_dir = options[:install_dir] + @gem_home = File.realpath(@install_dir || Gem.dir) + @user_dir = File.exist?(Gem.user_dir) ? File.realpath(Gem.user_dir) : Gem.user_dir @force_executables = options[:executables] @force_all = options[:all] @force_ignore = options[:ignore] @@ -70,7 +71,7 @@ class Gem::Uninstaller # only add user directory if install_dir is not set @user_install = false - @user_install = options[:user_install] unless options[:install_dir] + @user_install = options[:user_install] unless @install_dir # Optimization: populated during #uninstall @default_specs_matching_uninstall_params = [] @@ -105,7 +106,7 @@ class Gem::Uninstaller list, other_repo_specs = list.partition do |spec| @gem_home == spec.base_dir || - (@user_install && spec.base_dir == Gem.user_dir) + (@user_install && spec.base_dir == @user_dir) end list.sort! @@ -239,7 +240,7 @@ class Gem::Uninstaller def remove(spec) unless path_ok?(@gem_home, spec) || - (@user_install && path_ok?(Gem.user_dir, spec)) + (@user_install && path_ok?(@user_dir, spec)) e = Gem::GemNotInHomeException.new \ "Gem '#{spec.full_name}' is not installed in directory #{@gem_home}" e.spec = spec @@ -284,17 +285,18 @@ class Gem::Uninstaller def remove_plugins(spec) # :nodoc: return if spec.plugins.empty? - remove_plugins_for(spec, @plugins_dir) + remove_plugins_for(spec, plugin_dir_for(spec)) end ## # Regenerates plugin wrappers after removal. def regenerate_plugins - latest = Gem::Specification.latest_spec_for(@spec.name) + specification_record = @install_dir ? Gem::SpecificationRecord.from_path(@install_dir) : Gem::Specification.specification_record + latest = specification_record.latest_spec_for(@spec.name) return if latest.nil? - regenerate_plugins_for(latest, @plugins_dir) + regenerate_plugins_for(latest, plugin_dir_for(@spec)) end ## @@ -406,4 +408,8 @@ class Gem::Uninstaller say "Gem #{spec.full_name} cannot be uninstalled because it is a default gem" end end + + def plugin_dir_for(spec) + Gem.plugindir(spec.base_dir) + end end @@ -746,6 +746,7 @@ load_iseq_eval(rb_execution_context_t *ec, VALUE fname) if (*rb_ruby_prism_ptr()) { pm_parse_result_t result = { 0 }; result.options.line = 1; + result.node.coverage_enabled = 1; VALUE error = pm_load_parse_file(&result, fname); @@ -762,13 +763,13 @@ load_iseq_eval(rb_execution_context_t *ec, VALUE fname) } else { rb_ast_t *ast; - VALUE vast; + VALUE ast_value; VALUE parser = rb_parser_new(); rb_parser_set_context(parser, NULL, FALSE); - vast = rb_parser_load_file(parser, fname); - ast = rb_ruby_ast_data_get(vast); + ast_value = rb_parser_load_file(parser, fname); + ast = rb_ruby_ast_data_get(ast_value); - iseq = rb_iseq_new_top(vast, rb_fstring_lit("<top (required)>"), + iseq = rb_iseq_new_top(ast_value, rb_fstring_lit("<top (required)>"), fname, realpath_internal_cached(realpath_map, fname), NULL); rb_ast_dispose(ast); } diff --git a/man/ruby.1 b/man/ruby.1 index a0cda14641..bbad8ae203 100644 --- a/man/ruby.1 +++ b/man/ruby.1 @@ -448,18 +448,20 @@ Check syntax (same as .El .Pp Or one of the following, which are intended for debugging the interpreter: -.Bl -hang -offset indent -tag -width "parsetree_with_comment" +.Bl -hang -offset indent -tag -width "+error-tolerant" .It Sy yydebug Enable compiler debug mode (same as .Fl -yydebug). .It Sy parsetree Print a textual representation of the Ruby AST for the program. -.It Sy parsetree_with_comment -Print a textual representation of the Ruby AST for the program, but with each node annotated with the associated Ruby source code. .It Sy insns Print a list of disassembled bytecode instructions. -.It Sy insns_without_opt -Print the list of disassembled bytecode instructions before various optimizations have been applied. +.It Sy -optimize +Disable various optimizations to print a list disassembled bytecode instructions. +.It Sy +error-tolerant +Enable error-tolerant parsing, when yydebug or parsetree. +.It Sy +comment +Annotate a textual representation of the Ruby AST for the program with the associated Ruby source code. .El .Pp .It Fl -verbose @@ -537,7 +539,7 @@ malloc family of C standard library calls ( .Xr calloc 3 , and .Xr realloc 3 ) . -In this documentatation, the "heap" refers to the Ruby object heap +In this documentation, the "heap" refers to the Ruby object heap of fixed-sized slots, while "malloc" refers to auxiliary allocations commonly referred to as the "process heap". Thus there are at least two possible ways to trigger GC: @@ -1696,6 +1696,11 @@ r_copy_ivar(VALUE v, VALUE data) return v; } +#define override_ivar_error(type, str) \ + rb_raise(rb_eTypeError, \ + "can't override instance variable of "type" '%"PRIsVALUE"'", \ + (str)) + static void r_ivar(VALUE obj, int *has_encoding, struct load_arg *arg) { @@ -1703,6 +1708,12 @@ r_ivar(VALUE obj, int *has_encoding, struct load_arg *arg) len = r_long(arg); if (len > 0) { + if (RB_TYPE_P(obj, T_MODULE)) { + override_ivar_error("module", rb_mod_name(obj)); + } + else if (RB_TYPE_P(obj, T_CLASS)) { + override_ivar_error("class", rb_class_name(obj)); + } do { VALUE sym = r_symbol(arg); VALUE val = r_object(arg); @@ -1795,9 +1806,7 @@ append_extmod(VALUE obj, VALUE extmod) #define prohibit_ivar(type, str) do { \ if (!ivp || !*ivp) break; \ - rb_raise(rb_eTypeError, \ - "can't override instance variable of "type" '%"PRIsVALUE"'", \ - (str)); \ + override_ivar_error(type, str); \ } while (0) static VALUE r_object_for(struct load_arg *arg, bool partial, int *ivp, VALUE extmod, int type); diff --git a/mini_builtin.c b/mini_builtin.c index b38642e89b..810125fa2e 100644 --- a/mini_builtin.c +++ b/mini_builtin.c @@ -12,17 +12,17 @@ static struct st_table *loaded_builtin_table; #endif -VALUE rb_builtin_vast(const char *feature_name, VALUE *name_str); +VALUE rb_builtin_ast_value(const char *feature_name, VALUE *name_str); static const rb_iseq_t * builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *table) { VALUE name_str = 0; rb_ast_t *ast; - VALUE vast = rb_builtin_vast(feature_name, &name_str); + VALUE ast_value = rb_builtin_ast_value(feature_name, &name_str); rb_vm_t *vm = GET_VM(); - if (NIL_P(vast)) { + if (NIL_P(ast_value)) { rb_fatal("builtin_iseq_load: can not find %s; " "probably miniprelude.c is out of date", feature_name); @@ -40,8 +40,8 @@ builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *ta .coverage_enabled = FALSE, .debug_level = 0, }; - ast = rb_ruby_ast_data_get(vast); - const rb_iseq_t *iseq = rb_iseq_new_with_opt(vast, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil); + ast = rb_ruby_ast_data_get(ast_value); + const rb_iseq_t *iseq = rb_iseq_new_with_opt(ast_value, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil); GET_VM()->builtin_function_table = NULL; rb_ast_dispose(ast); diff --git a/misc/lldb_rb/utils.py b/misc/lldb_rb/utils.py index 86b5bdda2d..1b0e4f9f2a 100644 --- a/misc/lldb_rb/utils.py +++ b/misc/lldb_rb/utils.py @@ -60,6 +60,9 @@ class RbInspector(LLDBInterface): rbUndef = self.ruby_globals["RUBY_Qundef"] rbImmediateMask = self.ruby_globals["RUBY_IMMEDIATE_MASK"] + if self.inspect_node(val): + return + num = val.GetValueAsSigned() if num == rbFalse: print('false', file=self.result) @@ -245,227 +248,6 @@ class RbInspector(LLDBInterface): print("T_DATA:", file=self.result) self._append_expression("*(struct RData *) %0#x" % val.GetValueAsUnsigned()) - elif rval.is_type("RUBY_T_NODE"): - tRNode = self.target.FindFirstType("struct RNode").GetPointerType() - rbNodeTypeMask = self.ruby_globals["RUBY_NODE_TYPEMASK"] - rbNodeTypeShift = self.ruby_globals["RUBY_NODE_TYPESHIFT"] - - nd_type = (rval.flags & rbNodeTypeMask) >> rbNodeTypeShift - val = val.Cast(tRNode) - - self._append_expression("(node_type) %d" % nd_type) - - if nd_type == self.ruby_globals["NODE_SCOPE"]: - self._append_expression("*(struct RNode_SCOPE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BLOCK"]: - self._append_expression("*(struct RNode_BLOCK *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IF"]: - self._append_expression("*(struct RNode_IF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNLESS"]: - self._append_expression("*(struct RNode_UNLESS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE"]: - self._append_expression("*(struct RNode_CASE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE2"]: - self._append_expression("*(struct RNode_CASE2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CASE3"]: - self._append_expression("*(struct RNode_CASE3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_WHEN"]: - self._append_expression("*(struct RNode_WHEN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IN"]: - self._append_expression("*(struct RNode_IN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_WHILE"]: - self._append_expression("*(struct RNode_WHILE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNTIL"]: - self._append_expression("*(struct RNode_UNTIL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ITER"]: - self._append_expression("*(struct RNode_ITER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FOR"]: - self._append_expression("*(struct RNode_FOR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FOR_MASGN"]: - self._append_expression("*(struct RNode_FOR_MASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BREAK"]: - self._append_expression("*(struct RNode_BREAK *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NEXT"]: - self._append_expression("*(struct RNode_NEXT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_REDO"]: - self._append_expression("*(struct RNode_REDO *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RETRY"]: - self._append_expression("*(struct RNode_RETRY *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BEGIN"]: - self._append_expression("*(struct RNode_BEGIN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RESCUE"]: - self._append_expression("*(struct RNode_RESCUE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RESBODY"]: - self._append_expression("*(struct RNode_RESBODY *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ENSURE"]: - self._append_expression("*(struct RNode_ENSURE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_AND"]: - self._append_expression("*(struct RNode_AND *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OR"]: - self._append_expression("*(struct RNode_OR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MASGN"]: - self._append_expression("*(struct RNode_MASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LASGN"]: - self._append_expression("*(struct RNode_LASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DASGN"]: - self._append_expression("*(struct RNode_DASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_GASGN"]: - self._append_expression("*(struct RNode_GASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_IASGN"]: - self._append_expression("*(struct RNode_IASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CDECL"]: - self._append_expression("*(struct RNode_CDECL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CVASGN"]: - self._append_expression("*(struct RNode_CVASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN1"]: - self._append_expression("*(struct RNode_OP_ASGN1 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN2"]: - self._append_expression("*(struct RNode_OP_ASGN2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN_AND"]: - self._append_expression("*(struct RNode_OP_ASGN_AND *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_ASGN_OR"]: - self._append_expression("*(struct RNode_OP_ASGN_OR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OP_CDECL"]: - self._append_expression("*(struct RNode_OP_CDECL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CALL"]: - self._append_expression("*(struct RNode_CALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OPCALL"]: - self._append_expression("*(struct RNode_OPCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FCALL"]: - self._append_expression("*(struct RNode_FCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_VCALL"]: - self._append_expression("*(struct RNode_VCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_QCALL"]: - self._append_expression("*(struct RNode_QCALL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SUPER"]: - self._append_expression("*(struct RNode_SUPER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ZSUPER"]: - self._append_expression("*(struct RNode_ZSUPER *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LIST"]: - self._append_expression("*(struct RNode_LIST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ZLIST"]: - self._append_expression("*(struct RNode_ZLIST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_HASH"]: - self._append_expression("*(struct RNode_HASH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_RETURN"]: - self._append_expression("*(struct RNode_RETURN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_YIELD"]: - self._append_expression("*(struct RNode_YIELD *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LVAR"]: - self._append_expression("*(struct RNode_LVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DVAR"]: - self._append_expression("*(struct RNode_DVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_GVAR"]: - self._append_expression("*(struct RNode_GVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CONST"]: - self._append_expression("*(struct RNode_CONST *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CVAR"]: - self._append_expression("*(struct RNode_CVAR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NTH_REF"]: - self._append_expression("*(struct RNode_NTH_REF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_BACK_REF"]: - self._append_expression("*(struct RNode_BACK_REF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH"]: - self._append_expression("*(struct RNode_MATCH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH2"]: - self._append_expression("*(struct RNode_MATCH2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MATCH3"]: - self._append_expression("*(struct RNode_MATCH3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_STR"]: - self._append_expression("*(struct RNode_STR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DSTR"]: - self._append_expression("*(struct RNode_DSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_XSTR"]: - self._append_expression("*(struct RNode_XSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DXSTR"]: - self._append_expression("*(struct RNode_DXSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_EVSTR"]: - self._append_expression("*(struct RNode_EVSTR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_REGX"]: - self._append_expression("*(struct RNode_REGX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DREGX"]: - self._append_expression("*(struct RNode_DREGX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ONCE"]: - self._append_expression("*(struct RNode_ONCE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGS"]: - self._append_expression("*(struct RNode_ARGS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGS_AUX"]: - self._append_expression("*(struct RNode_ARGS_AUX *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_OPT_ARG"]: - self._append_expression("*(struct RNode_OPT_ARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_KW_ARG"]: - self._append_expression("*(struct RNode_KW_ARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_POSTARG"]: - self._append_expression("*(struct RNode_POSTARG *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGSCAT"]: - self._append_expression("*(struct RNode_ARGSCAT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARGSPUSH"]: - self._append_expression("*(struct RNode_ARGSPUSH *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SPLAT"]: - self._append_expression("*(struct RNode_SPLAT *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFN"]: - self._append_expression("*(struct RNode_DEFN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFS"]: - self._append_expression("*(struct RNode_DEFS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ALIAS"]: - self._append_expression("*(struct RNode_ALIAS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_VALIAS"]: - self._append_expression("*(struct RNode_VALIAS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_UNDEF"]: - self._append_expression("*(struct RNode_UNDEF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_CLASS"]: - self._append_expression("*(struct RNode_CLASS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_MODULE"]: - self._append_expression("*(struct RNode_MODULE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SCLASS"]: - self._append_expression("*(struct RNode_SCLASS *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_COLON2"]: - self._append_expression("*(struct RNode_COLON2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_COLON3"]: - self._append_expression("*(struct RNode_COLON3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DOT2"]: - self._append_expression("*(struct RNode_DOT2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DOT3"]: - self._append_expression("*(struct RNode_DOT3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FLIP2"]: - self._append_expression("*(struct RNode_FLIP2 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FLIP3"]: - self._append_expression("*(struct RNode_FLIP3 *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_SELF"]: - self._append_expression("*(struct RNode_SELF *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_NIL"]: - self._append_expression("*(struct RNode_NIL *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_TRUE"]: - self._append_expression("*(struct RNode_TRUE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FALSE"]: - self._append_expression("*(struct RNode_FALSE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ERRINFO"]: - self._append_expression("*(struct RNode_ERRINFO *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DEFINED"]: - self._append_expression("*(struct RNode_DEFINED *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_POSTEXE"]: - self._append_expression("*(struct RNode_POSTEXE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_DSYM"]: - self._append_expression("*(struct RNode_DSYM *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ATTRASGN"]: - self._append_expression("*(struct RNode_ATTRASGN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LAMBDA"]: - self._append_expression("*(struct RNode_LAMBDA *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ARYPTN"]: - self._append_expression("*(struct RNode_ARYPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_HSHPTN"]: - self._append_expression("*(struct RNode_HSHPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FNDPTN"]: - self._append_expression("*(struct RNode_FNDPTN *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_ERROR"]: - self._append_expression("*(struct RNode_ERROR *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_LINE"]: - self._append_expression("*(struct RNode_LINE *) %0#x" % val.GetValueAsUnsigned()) - elif nd_type == self.ruby_globals["NODE_FILE"]: - self._append_expression("*(struct RNode_FILE *) %0#x" % val.GetValueAsUnsigned()) - else: - self._append_expression("*(struct RNode *) %0#x" % val.GetValueAsUnsigned()) - elif rval.is_type("RUBY_T_IMEMO"): imemo_type = ((rval.flags >> self.ruby_globals["RUBY_FL_USHIFT"]) & IMEMO_MASK) @@ -492,3 +274,230 @@ class RbInspector(LLDBInterface): else: print("Not-handled type %0#x" % rval.type, file=self.result) print(val, file=self.result) + + def inspect_node(self, val): + tRNode = self.target.FindFirstType("struct RNode").GetPointerType() + + # if val.GetType() != tRNode: does not work for unknown reason + + if val.GetType().GetPointeeType().name != "NODE": + return False + + rbNodeTypeMask = self.ruby_globals["RUBY_NODE_TYPEMASK"] + rbNodeTypeShift = self.ruby_globals["RUBY_NODE_TYPESHIFT"] + flags = val.Cast(tRNode).GetChildMemberWithName("flags").GetValueAsUnsigned() + nd_type = (flags & rbNodeTypeMask) >> rbNodeTypeShift + + self._append_expression("(node_type) %d" % nd_type) + + if nd_type == self.ruby_globals["NODE_SCOPE"]: + self._append_expression("*(struct RNode_SCOPE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BLOCK"]: + self._append_expression("*(struct RNode_BLOCK *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IF"]: + self._append_expression("*(struct RNode_IF *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNLESS"]: + self._append_expression("*(struct RNode_UNLESS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE"]: + self._append_expression("*(struct RNode_CASE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE2"]: + self._append_expression("*(struct RNode_CASE2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CASE3"]: + self._append_expression("*(struct RNode_CASE3 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_WHEN"]: + self._append_expression("*(struct RNode_WHEN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IN"]: + self._append_expression("*(struct RNode_IN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_WHILE"]: + self._append_expression("*(struct RNode_WHILE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNTIL"]: + self._append_expression("*(struct RNode_UNTIL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ITER"]: + self._append_expression("*(struct RNode_ITER *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FOR"]: + self._append_expression("*(struct RNode_FOR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FOR_MASGN"]: + self._append_expression("*(struct RNode_FOR_MASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BREAK"]: + self._append_expression("*(struct RNode_BREAK *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NEXT"]: + self._append_expression("*(struct RNode_NEXT *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_REDO"]: + self._append_expression("*(struct RNode_REDO *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RETRY"]: + self._append_expression("*(struct RNode_RETRY *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BEGIN"]: + self._append_expression("*(struct RNode_BEGIN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RESCUE"]: + self._append_expression("*(struct RNode_RESCUE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RESBODY"]: + self._append_expression("*(struct RNode_RESBODY *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ENSURE"]: + self._append_expression("*(struct RNode_ENSURE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_AND"]: + self._append_expression("*(struct RNode_AND *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OR"]: + self._append_expression("*(struct RNode_OR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MASGN"]: + self._append_expression("*(struct RNode_MASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LASGN"]: + self._append_expression("*(struct RNode_LASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DASGN"]: + self._append_expression("*(struct RNode_DASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_GASGN"]: + self._append_expression("*(struct RNode_GASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_IASGN"]: + self._append_expression("*(struct RNode_IASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CDECL"]: + self._append_expression("*(struct RNode_CDECL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CVASGN"]: + self._append_expression("*(struct RNode_CVASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN1"]: + self._append_expression("*(struct RNode_OP_ASGN1 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN2"]: + self._append_expression("*(struct RNode_OP_ASGN2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN_AND"]: + self._append_expression("*(struct RNode_OP_ASGN_AND *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_ASGN_OR"]: + self._append_expression("*(struct RNode_OP_ASGN_OR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OP_CDECL"]: + self._append_expression("*(struct RNode_OP_CDECL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CALL"]: + self._append_expression("*(struct RNode_CALL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OPCALL"]: + self._append_expression("*(struct RNode_OPCALL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FCALL"]: + self._append_expression("*(struct RNode_FCALL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_VCALL"]: + self._append_expression("*(struct RNode_VCALL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_QCALL"]: + self._append_expression("*(struct RNode_QCALL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SUPER"]: + self._append_expression("*(struct RNode_SUPER *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ZSUPER"]: + self._append_expression("*(struct RNode_ZSUPER *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LIST"]: + self._append_expression("*(struct RNode_LIST *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ZLIST"]: + self._append_expression("*(struct RNode_ZLIST *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_HASH"]: + self._append_expression("*(struct RNode_HASH *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_RETURN"]: + self._append_expression("*(struct RNode_RETURN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_YIELD"]: + self._append_expression("*(struct RNode_YIELD *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LVAR"]: + self._append_expression("*(struct RNode_LVAR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DVAR"]: + self._append_expression("*(struct RNode_DVAR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_GVAR"]: + self._append_expression("*(struct RNode_GVAR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CONST"]: + self._append_expression("*(struct RNode_CONST *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CVAR"]: + self._append_expression("*(struct RNode_CVAR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NTH_REF"]: + self._append_expression("*(struct RNode_NTH_REF *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_BACK_REF"]: + self._append_expression("*(struct RNode_BACK_REF *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH"]: + self._append_expression("*(struct RNode_MATCH *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH2"]: + self._append_expression("*(struct RNode_MATCH2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MATCH3"]: + self._append_expression("*(struct RNode_MATCH3 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_STR"]: + self._append_expression("*(struct RNode_STR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DSTR"]: + self._append_expression("*(struct RNode_DSTR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_XSTR"]: + self._append_expression("*(struct RNode_XSTR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DXSTR"]: + self._append_expression("*(struct RNode_DXSTR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_EVSTR"]: + self._append_expression("*(struct RNode_EVSTR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_REGX"]: + self._append_expression("*(struct RNode_REGX *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DREGX"]: + self._append_expression("*(struct RNode_DREGX *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ONCE"]: + self._append_expression("*(struct RNode_ONCE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGS"]: + self._append_expression("*(struct RNode_ARGS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGS_AUX"]: + self._append_expression("*(struct RNode_ARGS_AUX *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_OPT_ARG"]: + self._append_expression("*(struct RNode_OPT_ARG *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_KW_ARG"]: + self._append_expression("*(struct RNode_KW_ARG *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_POSTARG"]: + self._append_expression("*(struct RNode_POSTARG *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGSCAT"]: + self._append_expression("*(struct RNode_ARGSCAT *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARGSPUSH"]: + self._append_expression("*(struct RNode_ARGSPUSH *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SPLAT"]: + self._append_expression("*(struct RNode_SPLAT *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFN"]: + self._append_expression("*(struct RNode_DEFN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFS"]: + self._append_expression("*(struct RNode_DEFS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ALIAS"]: + self._append_expression("*(struct RNode_ALIAS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_VALIAS"]: + self._append_expression("*(struct RNode_VALIAS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_UNDEF"]: + self._append_expression("*(struct RNode_UNDEF *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_CLASS"]: + self._append_expression("*(struct RNode_CLASS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_MODULE"]: + self._append_expression("*(struct RNode_MODULE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SCLASS"]: + self._append_expression("*(struct RNode_SCLASS *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_COLON2"]: + self._append_expression("*(struct RNode_COLON2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_COLON3"]: + self._append_expression("*(struct RNode_COLON3 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DOT2"]: + self._append_expression("*(struct RNode_DOT2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DOT3"]: + self._append_expression("*(struct RNode_DOT3 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FLIP2"]: + self._append_expression("*(struct RNode_FLIP2 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FLIP3"]: + self._append_expression("*(struct RNode_FLIP3 *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_SELF"]: + self._append_expression("*(struct RNode_SELF *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_NIL"]: + self._append_expression("*(struct RNode_NIL *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_TRUE"]: + self._append_expression("*(struct RNode_TRUE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FALSE"]: + self._append_expression("*(struct RNode_FALSE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ERRINFO"]: + self._append_expression("*(struct RNode_ERRINFO *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DEFINED"]: + self._append_expression("*(struct RNode_DEFINED *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_POSTEXE"]: + self._append_expression("*(struct RNode_POSTEXE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_DSYM"]: + self._append_expression("*(struct RNode_DSYM *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ATTRASGN"]: + self._append_expression("*(struct RNode_ATTRASGN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LAMBDA"]: + self._append_expression("*(struct RNode_LAMBDA *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ARYPTN"]: + self._append_expression("*(struct RNode_ARYPTN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_HSHPTN"]: + self._append_expression("*(struct RNode_HSHPTN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FNDPTN"]: + self._append_expression("*(struct RNode_FNDPTN *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_ERROR"]: + self._append_expression("*(struct RNode_ERROR *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_LINE"]: + self._append_expression("*(struct RNode_LINE *) %0#x" % val.GetValueAsUnsigned()) + elif nd_type == self.ruby_globals["NODE_FILE"]: + self._append_expression("*(struct RNode_FILE *) %0#x" % val.GetValueAsUnsigned()) + else: + self._append_expression("*(struct RNode *) %0#x" % val.GetValueAsUnsigned()) + return True diff --git a/missing/setproctitle.c b/missing/setproctitle.c index f90886671c..5b2dfa65ce 100644 --- a/missing/setproctitle.c +++ b/missing/setproctitle.c @@ -87,7 +87,30 @@ static char **argv1_addr = NULL; #endif #if ALLOCATE_ENVIRON +/* system_environ is the value of environ before we allocate a custom buffer. + * + * We use this to restore environ in ruby_free_proctitle. + */ +static char **system_environ = NULL; +/* orig_environ is the buffer we allocate for environ. + * + * We use this to free this buffer in ruby_free_proctitle. When we add new + * environment variables using setenv, the system may change environ to a + * different buffer and will not free the original buffer, so we need to hold + * onto this so we can free it in ruby_free_proctitle. + * + * We must not free any of the contents because it may change if the system + * updates existing environment variables. + */ static char **orig_environ = NULL; +/* alloc_environ is a copy of orig_environ. + * + * We use this to free all the original string copies that were in orig_environ. + * Since environ could be changed to point to strings allocated by the system + * if environment variables are updated, so we need this to point to the + * original strings. + */ +static char **alloc_environ = NULL; #endif void @@ -112,6 +135,9 @@ compat_init_setproctitle(int argc, char *argv[]) /* Fail if we can't allocate room for the new environment */ for (i = 0; envp[i] != NULL; i++); + system_environ = environ; + + alloc_environ = xcalloc(i + 1, sizeof(*environ)); orig_environ = environ = xcalloc(i + 1, sizeof(*environ)); if (environ == NULL) { environ = envp; /* put it back */ @@ -140,8 +166,8 @@ compat_init_setproctitle(int argc, char *argv[]) argv_env_len = lastenvp - argv[0]; for (i = 0; envp[i] != NULL; i++) - environ[i] = ruby_strdup(envp[i]); - environ[i] = NULL; + alloc_environ[i] = environ[i] = ruby_strdup(envp[i]); + alloc_environ[i] = environ[i] = NULL; #endif /* SPT_REUSEARGV */ } @@ -153,16 +179,13 @@ ruby_free_proctitle(void) if (!orig_environ) return; /* environ is allocated by OS */ - /* ruby_setenv could allocate a new environ, so we need to free orig_environ - * in that case. */ - if (environ != orig_environ) { - for (int i = 0; orig_environ[i] != NULL; i++) { - xfree(orig_environ[i]); - } - - xfree(orig_environ); - orig_environ = NULL; + for (int i = 0; alloc_environ[i] != NULL; i++) { + xfree(alloc_environ[i]); } + xfree(alloc_environ); + xfree(orig_environ); + + environ = system_environ; #endif } @@ -13,7 +13,6 @@ #include <stddef.h> #include "node.h" #include "rubyparser.h" -#include "internal/parse.h" #endif #include "internal/variable.h" @@ -301,8 +300,9 @@ rb_ast_t * rb_ast_new(const rb_parser_config_t *config) { node_buffer_t *nb = rb_node_buffer_new(config); - rb_ast_t *ast = config->ast_new(nb); + rb_ast_t *ast = (rb_ast_t *)config->calloc(1, sizeof(rb_ast_t)); ast->config = config; + ast->node_buffer = nb; return ast; } #else @@ -782,6 +782,12 @@ pack_pack(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) return res; } +VALUE +rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) +{ + return pack_pack(ec, ary, fmt, buffer); +} + static const char uu_table[] = "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"; static const char b64_table[] = @@ -83,11 +83,11 @@ static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, con #define compile_callback rb_suppress_tracing #endif /* !UNIVERSAL_PARSER */ -static int rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2); +#define NODE_SPECIAL_EMPTY_ARGS ((NODE *)-1) +#define NODE_EMPTY_ARGS_P(node) ((node) == NODE_SPECIAL_EMPTY_ARGS) -#ifndef RIPPER +static int rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2); static rb_parser_string_t *rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *original); -#endif static int node_integer_cmp(rb_node_integer_t *n1, rb_node_integer_t *n2) @@ -297,7 +297,6 @@ parse_isxdigit(int c) #define STRNCASECMP rb_parser_st_locale_insensitive_strncasecmp #ifdef RIPPER -VALUE rb_ripper_none; #include "ripper_init.h" #endif @@ -489,7 +488,7 @@ struct parser_params { struct { rb_strterm_t *strterm; - VALUE (*gets)(struct parser_params*,rb_parser_input_data,int); + rb_parser_lex_gets_func *gets; rb_parser_input_data input; parser_string_buffer_t string_buffer; rb_parser_string_t *lastline; @@ -520,7 +519,7 @@ struct parser_params { int line_count; int ruby_sourceline; /* current line no. */ const char *ruby_sourcefile; /* current source file */ - VALUE ruby_sourcefile_string; + rb_parser_string_t *ruby_sourcefile_string; rb_encoding *enc; token_info *token_info; st_table *case_labels; @@ -637,10 +636,10 @@ static void after_reduce(int len, struct parser_params *p) { for (int i = 0; i < len; i++) { + VALUE tos = rb_ary_pop(p->s_value_stack); if (p->debug) { - rb_parser_printf(p, "after-reduce pop: %+"PRIsVALUE"\n", rb_ary_entry(p->s_value_stack, -1)); + rb_parser_printf(p, "after-reduce pop: %+"PRIsVALUE"\n", tos); } - rb_ary_pop(p->s_value_stack); } if (p->debug) { rb_parser_printf(p, "after-reduce push: %+"PRIsVALUE"\n", p->s_lvalue); @@ -662,10 +661,10 @@ static void after_pop_stack(int len, struct parser_params *p) { for (int i = 0; i < len; i++) { + VALUE tos = rb_ary_pop(p->s_value_stack); if (p->debug) { - rb_parser_printf(p, "after-pop-stack pop: %+"PRIsVALUE"\n", rb_ary_entry(p->s_value_stack, -1)); + rb_parser_printf(p, "after-pop-stack pop: %+"PRIsVALUE"\n", tos); } - rb_ary_pop(p->s_value_stack); } } #else @@ -1161,7 +1160,7 @@ static rb_node_aryptn_t *rb_node_aryptn_new(struct parser_params *p, NODE *pre_a static rb_node_hshptn_t *rb_node_hshptn_new(struct parser_params *p, NODE *nd_pconst, NODE *nd_pkwargs, NODE *nd_pkwrestarg, const YYLTYPE *loc); static rb_node_fndptn_t *rb_node_fndptn_new(struct parser_params *p, NODE *pre_rest_arg, NODE *args, NODE *post_rest_arg, const YYLTYPE *loc); static rb_node_line_t *rb_node_line_new(struct parser_params *p, const YYLTYPE *loc); -static rb_node_file_t *rb_node_file_new(struct parser_params *p, VALUE str, const YYLTYPE *loc); +static rb_node_file_t *rb_node_file_new(struct parser_params *p, rb_parser_string_t *str, const YYLTYPE *loc); static rb_node_error_t *rb_node_error_new(struct parser_params *p, const YYLTYPE *loc); #define NEW_SCOPE(a,b,loc) (NODE *)rb_node_scope_new(p,a,b,loc) @@ -1491,14 +1490,9 @@ static NODE *heredoc_dedent(struct parser_params*,NODE*); static void check_literal_when(struct parser_params *p, NODE *args, const YYLTYPE *loc); #ifdef RIPPER -static VALUE var_field(struct parser_params *p, VALUE a); #define get_value(idx) (rb_ary_entry(p->s_value_stack, idx)) #define set_value(val) (p->s_lvalue = val) -static VALUE defs(struct parser_params *p, VALUE head, VALUE args, VALUE bodystmt); static VALUE backref_error(struct parser_params*, NODE *, VALUE); -static VALUE ripper_assignable(struct parser_params *p, ID id, VALUE lhs); -static VALUE ripper_const_decl(struct parser_params *p, VALUE path); -static VALUE ripper_heredoc_dedent(struct parser_params *p, int indent, VALUE array); static VALUE assign_error(struct parser_params *p, const char *mesg, VALUE a); static int id_is_var(struct parser_params *p, ID id); #endif @@ -1614,24 +1608,11 @@ void ripper_error(struct parser_params *p); #define yyparse ripper_yyparse -static void ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs); - -static VALUE -ripper_new_args(struct parser_params *p, VALUE pre_args, VALUE opt_args, VALUE rest_arg, VALUE post_args, VALUE tail) -{ - VALUE kw_args = rb_ary_entry(tail, 0); - VALUE kw_rest_arg = rb_ary_entry(tail, 1); - VALUE block = rb_ary_entry(tail, 2); - return dispatch7(params, pre_args, opt_args, rest_arg, post_args, kw_args, kw_rest_arg, block); -} +static VALUE ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs); static VALUE -ripper_new_array_pattern(struct parser_params *p, VALUE constant, VALUE pre_arg, VALUE aryptn) +aryptn_pre_args(struct parser_params *p, VALUE pre_arg, VALUE pre_args) { - VALUE pre_args = rb_ary_entry(aryptn, 0); - VALUE rest_arg = rb_ary_entry(aryptn, 1); - VALUE post_args = rb_ary_entry(aryptn, 2); - if (!NIL_P(pre_arg)) { if (!NIL_P(pre_args)) { rb_ary_unshift(pre_args, pre_arg); @@ -1640,50 +1621,7 @@ ripper_new_array_pattern(struct parser_params *p, VALUE constant, VALUE pre_arg, pre_args = rb_ary_new_from_args(1, pre_arg); } } - return dispatch4(aryptn, constant, pre_args, rest_arg, post_args); -} - -static VALUE -ripper_new_array_pattern_tail(struct parser_params *p, VALUE pre_args, VALUE rest_arg, VALUE post_args) -{ - return rb_ary_new_from_args(3, pre_args, rest_arg, post_args); -} - -static VALUE -ripper_new_hash_pattern(struct parser_params *p, VALUE constant, VALUE hshptn) -{ - VALUE kw_args = rb_ary_entry(hshptn, 0); - VALUE kw_rest_arg = rb_ary_entry(hshptn, 1); - - return dispatch3(hshptn, constant, kw_args, kw_rest_arg); -} - -static VALUE -ripper_new_hash_pattern_tail(struct parser_params *p, VALUE kw_args, VALUE kw_rest_arg) -{ - if (kw_rest_arg) { - kw_rest_arg = dispatch1(var_field, kw_rest_arg); - } - else { - kw_rest_arg = Qnil; - } - return rb_ary_new_from_args(2, kw_args, kw_rest_arg); -} - -static VALUE -ripper_new_find_pattern(struct parser_params *p, VALUE constant, VALUE fndptn) -{ - VALUE pre_rest_arg = rb_ary_entry(fndptn, 0); - VALUE args = rb_ary_entry(fndptn, 1); - VALUE post_rest_arg = rb_ary_entry(fndptn, 2); - - return dispatch4(fndptn, constant, pre_rest_arg, args, post_rest_arg); -} - -static VALUE -ripper_new_find_pattern_tail(struct parser_params *p, VALUE pre_rest_arg, VALUE args, VALUE post_rest_arg) -{ - return rb_ary_new_from_args(3, pre_rest_arg, args, post_rest_arg); + return pre_args; } #define ID2VAL(id) STATIC_ID2SYM(id) @@ -1772,8 +1710,10 @@ endless_method_name(struct parser_params *p, ID mid, const YYLTYPE *loc) #ifndef RIPPER # define ifndef_ripper(x) (x) +# define ifdef_ripper(r,x) (x) #else # define ifndef_ripper(x) +# define ifdef_ripper(r,x) (r) #endif # define rb_warn0(fmt) WARN_CALL(WARN_ARGS(fmt, 1)) @@ -2103,11 +2043,48 @@ rb_parser_encoding_string_new(rb_parser_t *p, const char *ptr, long len, rb_enco } #ifndef RIPPER +static bool +zero_filled(const char *s, int n) +{ + for (; n > 0; --n) { + if (*s++) return false; + } + return true; +} + +static bool +str_null_char(rb_parser_t *p, const char *s, long len, const int minlen, rb_encoding *enc) +{ + const char *e = s + len; + + for (; s + minlen <= e; s += rb_enc_mbclen(s, e, enc)) { + if (zero_filled(s, minlen)) return true; + } + return false; +} + +static bool +cstr_null_check(rb_parser_t *p, const char *s, long len, rb_encoding *enc, int *w) +{ + const int minlen = rb_enc_mbminlen(enc); + + if (minlen > 1) { + *w = 1; + return str_null_char(p, s, len, minlen, enc); + } + else { + *w = 0; + return (!s || memchr(s, 0, len)); + } +} + rb_parser_string_t * rb_str_to_parser_string(rb_parser_t *p, VALUE str) { /* Type check */ - return rb_parser_encoding_string_new(p, RSTRING_PTR(str), RSTRING_LEN(str), rb_enc_get(str)); + rb_parser_string_t *ret = rb_parser_encoding_string_new(p, RSTRING_PTR(str), RSTRING_LEN(str), rb_enc_get(str)); + RB_GC_GUARD(str); + return ret; } #endif @@ -2738,6 +2715,7 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) const struct vtable *vars; struct rb_strterm_struct *strterm; struct lex_context ctxt; + enum lex_state_e state; } %token <id> @@ -2822,22 +2800,19 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %type <node> if_tail opt_else case_body case_args cases opt_rescue exc_list exc_var opt_ensure %type <node> args arg_splat call_args opt_call_args %type <node> paren_args opt_paren_args -%type <node_args> args_tail opt_args_tail block_args_tail opt_block_args_tail +%type <node_args> args_tail block_args_tail %type <node> command_args aref_args %type <node_block_pass> opt_block_arg block_arg %type <node> var_ref var_lhs %type <node> command_rhs arg_rhs %type <node> command_asgn mrhs mrhs_arg superclass block_call block_command -%type <node_opt_arg> f_block_optarg f_block_opt %type <node_args> f_arglist f_opt_paren_args f_paren_args f_args %type <node_args_aux> f_arg f_arg_item -%type <node_opt_arg> f_optarg %type <node> f_marg f_marg_list f_rest_marg %type <node_masgn> f_margs %type <node> assoc_list assocs assoc undef_list backref string_dvar for_var %type <node_args> block_param opt_block_param block_param_def -%type <node_opt_arg> f_opt -%type <node_kw_arg> f_kwarg f_kw f_block_kwarg f_block_kw +%type <node_kw_arg> f_kw f_block_kw %type <id> bv_decls opt_bv_decl bvar %type <node> lambda lambda_body brace_body do_body %type <node_args> f_larglist @@ -2905,7 +2880,7 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %token tSTAR "*" %token tDSTAR "**arg" %token tAMPER "&" -%token tLAMBDA "->" +%token <num> tLAMBDA "->" %token tSYMBEG "symbol literal" %token tSTRING_BEG "string literal" %token tXSTRING_BEG "backtick literal" @@ -2916,7 +2891,8 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %token tQSYMBOLS_BEG "verbatim symbol list" %token tSTRING_END "terminator" %token tSTRING_DEND "'}'" -%token tSTRING_DBEG tSTRING_DVAR tLAMBEG tLABEL_END +%token <state> tSTRING_DBEG "'#{'" +%token tSTRING_DVAR tLAMBEG tLABEL_END %token tIGNORED_NL tCOMMENT tEMBDOC_BEG tEMBDOC tEMBDOC_END %token tHEREDOC_BEG tHEREDOC_END k__END__ @@ -2951,6 +2927,61 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) %token tLAST_TOKEN +/* + * parameterizing rules + */ +%rule f_opt(value) <node_opt_arg>: f_arg_asgn f_eq value + { + p->cur_arg = 0; + p->ctxt.in_argdef = 1; + $$ = NEW_OPT_ARG(assignable(p, $1, $3, &@$), &@$); + /*% ripper: [$:$, $:3] %*/ + } + ; + +%rule f_optarg(value) <node_opt_arg>: f_opt(value) + { + $$ = $1; + /*% ripper: rb_ary_new3(1, $:1) %*/ + } + | f_optarg(value) ',' f_opt(value) + { + $$ = opt_arg_append($1, $3); + /*% ripper: rb_ary_push($:1, $:3) %*/ + } + ; + +%rule f_kwarg(kw) <node_kw_arg>: kw + { + $$ = $1; + /*% ripper: rb_ary_new3(1, $:1) %*/ + } + | f_kwarg(kw) ',' kw + { + $$ = kwd_append($1, $3); + /*% ripper: rb_ary_push($:1, $:3) %*/ + } + ; + +%rule opt_args_tail(tail) <node_args>: ',' tail + { + $$ = $2; + /*% ripper: $:2 %*/ + } + | /* none */ + { + $$ = new_args_tail(p, 0, 0, 0, &@0); + /*% ripper: [Qnil, Qnil, Qnil] %*/ + } + ; + +%rule words(begin, word_list): begin ' '+ word_list tSTRING_END + { + $$ = make_list($3, &@$); + /*% ripper: array!($:3) %*/ + } + ; + %% program : { SET_LEX_STATE(EXPR_BEG); @@ -3009,7 +3040,7 @@ top_stmt : stmt | keyword_BEGIN begin_block { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -3098,7 +3129,7 @@ k_END : keyword_END lex_ctxt { $$ = $2; p->ctxt.in_rescue = before_rescue; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ }; stmt : keyword_alias fitem {SET_LEX_STATE(EXPR_FNAME|EXPR_FITEM);} fitem @@ -3269,12 +3300,8 @@ command_asgn : lhs '=' lex_ctxt command_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFN($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = dispatch3(def, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: def!($:head, $:args, $:$) %*/ local_pop(p); } | defs_head[head] f_opt_paren_args[args] '=' endless_command[bodystmt] @@ -3284,12 +3311,8 @@ command_asgn : lhs '=' lex_ctxt command_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = defs(p, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:$) %*/ local_pop(p); } | backref tOP_ASGN lex_ctxt command_rhs @@ -3298,7 +3321,7 @@ command_asgn : lhs '=' lex_ctxt command_rhs rb_backref_error(p, $1); /*% %*/ $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, RNODE($:1), assign!(var_field(p, get_value($:1)), $:4)) %*/ + /*% ripper[error]: backref_error(p, $1, opassign!(var_field!($:1), $:2, $:4)) %*/ } ; @@ -3399,7 +3422,7 @@ defn_head : k_def def_name $$ = def_head_save(p, $k_def); $$->nd_mid = $def_name; $$->nd_def = NEW_DEFN($def_name, 0, &@$); - /*% ripper: get_value($:def_name); %*/ + /*% ripper: $:def_name %*/ } ; @@ -3414,10 +3437,7 @@ defs_head : k_def singleton dot_or_colon $$ = def_head_save(p, $k_def); $$->nd_mid = $def_name; $$->nd_def = NEW_DEFS($singleton, $def_name, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:singleton), get_value($:dot_or_colon), get_value($:def_name))); - %*/ + /*% ripper: [$:singleton, $:dot_or_colon, $:def_name] %*/ } ; @@ -3435,7 +3455,7 @@ expr_value : expr expr_value_do : {COND_PUSH(1);} expr_value do {COND_POP();} { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -3455,14 +3475,14 @@ cmd_brace_block : tLBRACE_ARG brace_body '}' { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; fcall : operation { $$ = NEW_FCALL($1, 0, &@$); - /*% ripper: get_value($:1); %*/ + /*% ripper: $:1 %*/ } ; @@ -3529,14 +3549,14 @@ command : fcall command_args %prec tLOWEST { NODE *args = 0; args = ret_args(p, $2); - $<node>$ = add_block_exit(p, NEW_BREAK(args, &@$)); + $$ = add_block_exit(p, NEW_BREAK(args, &@$)); /*% ripper: break!($:2) %*/ } | keyword_next call_args { NODE *args = 0; args = ret_args(p, $2); - $<node>$ = add_block_exit(p, NEW_NEXT(args, &@$)); + $$ = add_block_exit(p, NEW_NEXT(args, &@$)); /*% ripper: next!($:2) %*/ } ; @@ -3560,7 +3580,7 @@ mlhs_inner : mlhs_basic mlhs_basic : mlhs_head { $$ = NEW_MASGN($1, 0, &@$); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | mlhs_head mlhs_item { @@ -3643,13 +3663,13 @@ mlhs_post : mlhs_item mlhs_node : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | primary_value '[' opt_call_args rbracket { @@ -3675,13 +3695,13 @@ mlhs_node : user_variable } | primary_value tCOLON2 tCONSTANT { + /*% ripper: const_path_field!($:1, $:3) %*/ $$ = const_decl(p, NEW_COLON2($1, $3, &@$), &@$); - /*% ripper: ripper_const_decl(p, const_path_field!($:1, $:3)) %*/ } | tCOLON3 tCONSTANT { + /*% ripper: top_const_field!($:2) %*/ $$ = const_decl(p, NEW_COLON3($2, &@$), &@$); - /*% ripper: ripper_const_decl(p, top_const_field!($:2)) %*/ } | backref { @@ -3689,19 +3709,19 @@ mlhs_node : user_variable rb_backref_error(p, $1); /*% %*/ $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, $1, var_field(p, get_value($:1))) %*/ + /*% ripper[error]: backref_error(p, $1, var_field!($:1)) %*/ } ; lhs : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | primary_value '[' opt_call_args rbracket { @@ -3725,13 +3745,13 @@ lhs : user_variable } | primary_value tCOLON2 tCONSTANT { + /*% ripper: const_path_field!($:1, $:3) %*/ $$ = const_decl(p, NEW_COLON2($1, $3, &@$), &@$); - /*% ripper: ripper_const_decl(p, const_path_field!($:1, $:3)) %*/ } | tCOLON3 tCONSTANT { + /*% ripper: top_const_field!($:2) %*/ $$ = const_decl(p, NEW_COLON3($2, &@$), &@$); - /*% ripper: ripper_const_decl(p, top_const_field!($:2)) %*/ } | backref { @@ -3739,7 +3759,7 @@ lhs : user_variable rb_backref_error(p, $1); /*% %*/ $$ = NEW_ERROR(&@$); - /*% ripper[error]: backref_error(p, $1, var_field(p, get_value($:1))) %*/ + /*% ripper[error]: backref_error(p, $1, var_field!($:1)) %*/ } ; @@ -3793,13 +3813,13 @@ fitem : fname undef_list : fitem { $$ = NEW_UNDEF($1, &@$); - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | undef_list ',' {SET_LEX_STATE(EXPR_FNAME|EXPR_FITEM);} fitem { NODE *undef = NEW_UNDEF($4, &@4); $$ = block_append(p, $1, undef); - /*% ripper: rb_ary_push(get_value($:1), get_value($:4)) %*/ + /*% ripper: rb_ary_push($:1, $:4) %*/ } ; @@ -3897,7 +3917,7 @@ arg : lhs '=' lex_ctxt arg_rhs /*%%%*/ $$ = NEW_ERROR(&@$); /*% %*/ - /*% ripper[error]: backref_error(p, RNODE($:1), opassign!(var_field(p, get_value($:1)), $:2, $:4)) %*/ + /*% ripper[error]: backref_error(p, $1, opassign!(var_field!($:1), $:2, $:4)) %*/ } | arg tDOT2 arg { @@ -3970,12 +3990,7 @@ arg : lhs '=' lex_ctxt arg_rhs | tUMINUS_NUM simple_numeric tPOW arg { $$ = call_uni_op(p, call_bin_op(p, $2, idPow, $4, &@2, &@$), idUMinus, &@1, &@$); - /*%%%*/ - /*% - VALUE val = dispatch3(binary, get_value($:2), ID2VAL(idPow), get_value($:4)); - val = dispatch2(unary, ID2VAL(idUMinus), val); - set_value(val); - %*/ + /*% ripper: unary!(ID2VAL(idUMinus), binary!($:2, ID2VAL(idPow), $:4)) %*/ } | tUPLUS arg { @@ -4083,12 +4098,8 @@ arg : lhs '=' lex_ctxt arg_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFN($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = dispatch3(def, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: def!($:head, $:args, $:$) %*/ local_pop(p); } | defs_head[head] f_opt_paren_args[args] '=' endless_arg[bodystmt] @@ -4098,12 +4109,8 @@ arg : lhs '=' lex_ctxt arg_rhs $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*%%%*/ - /*% - VALUE val = dispatch4(bodystmt, get_value($:bodystmt), Qnil, Qnil, Qnil); - val = defs(p, get_value($:head), get_value($:args), val); - set_value(val); - %*/ + /*% ripper: bodystmt!($:bodystmt, Qnil, Qnil, Qnil) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:$) %*/ local_pop(p); } | primary @@ -4232,13 +4239,16 @@ paren_args : '(' opt_call_args rparen opt_paren_args : none | paren_args + { + $$ = $1 ? $1 : NODE_SPECIAL_EMPTY_ARGS; + } ; opt_call_args : none | call_args | args ',' { - $$ = $1; + $$ = $1; } | args ',' assocs ',' { @@ -4313,14 +4323,14 @@ command_args : { CMDARG_POP(); if (lookahead) CMDARG_PUSH(0); $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; block_arg : tAMPER arg_value { $$ = NEW_BLOCK_PASS($2, &@$); - /*% ripper: get_value($:2) %*/ + /*% ripper: $:2 %*/ } | tAMPER { @@ -4333,7 +4343,7 @@ block_arg : tAMPER arg_value opt_block_arg : ',' block_arg { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none { @@ -4369,7 +4379,7 @@ args : arg_value arg_splat : tSTAR arg_value { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | tSTAR /* none */ { @@ -4557,7 +4567,7 @@ primary : literal k_end { if (CASE_LABELS_ENABLED_P(p->case_labels)) st_free_table(p->case_labels); - p->case_labels = $<labels>4; + p->case_labels = $4; $$ = NEW_CASE($2, $5, &@$); fixpos($$, $2); /*% ripper: case!($:2, $:5) %*/ @@ -4571,7 +4581,7 @@ primary : literal k_end { if (p->case_labels) st_free_table(p->case_labels); - p->case_labels = $<labels>3; + p->case_labels = $3; $$ = NEW_CASE2($4, &@$); /*% ripper: case!(Qnil, $:4) %*/ } @@ -4701,22 +4711,22 @@ primary : literal $bodystmt = new_scope_body(p, $args, $bodystmt, &@$); ($$ = $head->nd_def)->nd_loc = @$; RNODE_DEFS($$)->nd_defn = $bodystmt; - /*% ripper: defs(p, get_value($:head), get_value($:args), get_value($:bodystmt)) %*/ + /*% ripper: defs!(*$:head[0..2], $:args, $:bodystmt) %*/ local_pop(p); } | keyword_break { - $<node>$ = add_block_exit(p, NEW_BREAK(0, &@$)); + $$ = add_block_exit(p, NEW_BREAK(0, &@$)); /*% ripper: break!(args_new!) %*/ } | keyword_next { - $<node>$ = add_block_exit(p, NEW_NEXT(0, &@$)); + $$ = add_block_exit(p, NEW_NEXT(0, &@$)); /*% ripper: next!(args_new!) %*/ } | keyword_redo { - $<node>$ = add_block_exit(p, NEW_REDO(&@$)); + $$ = add_block_exit(p, NEW_REDO(&@$)); /*% ripper: redo! %*/ } | keyword_retry @@ -4948,7 +4958,6 @@ f_marg : f_norm_arg { $$ = assignable(p, $1, 0, &@$); mark_lvar_used(p, $$); - /*% ripper: ripper_assignable(p, $1, get_value($:1)) %*/ } | tLPAREN f_margs rparen { @@ -4972,7 +4981,7 @@ f_marg_list : f_marg f_margs : f_marg_list { $$ = NEW_MASGN($1, 0, &@$); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | f_marg_list ',' f_rest_marg { @@ -4998,9 +5007,9 @@ f_margs : f_marg_list f_rest_marg : tSTAR f_norm_arg { + /*% ripper: $:2 %*/ $$ = assignable(p, $2, 0, &@$); mark_lvar_used(p, $$); - /*% ripper: ripper_assignable(p, $2, get_value($:2)) %*/ } | tSTAR { @@ -5019,37 +5028,25 @@ f_any_kwrest : f_kwrest f_eq : {p->ctxt.in_argdef = 0;} '='; -block_args_tail : f_block_kwarg ',' f_kwrest opt_f_block_arg +block_args_tail : f_kwarg(f_block_kw) ',' f_kwrest opt_f_block_arg { $$ = new_args_tail(p, $1, $3, $4, &@3); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), get_value($:3), get_value($:4)); %*/ + /*% ripper: [$:1, $:3, $:4] %*/ } - | f_block_kwarg opt_f_block_arg + | f_kwarg(f_block_kw) opt_f_block_arg { $$ = new_args_tail(p, $1, 0, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), Qnil, get_value($:2)); %*/ + /*% ripper: [$:1, Qnil, $:2] %*/ } | f_any_kwrest opt_f_block_arg { $$ = new_args_tail(p, 0, $1, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), get_value($:2)); %*/ + /*% ripper: [Qnil, $:1, $:2] %*/ } | f_block_arg { $$ = new_args_tail(p, 0, 0, $1, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, get_value($:1)); %*/ - } - ; - -opt_block_args_tail : ',' block_args_tail - { - $$ = $2; - /*% ripper: get_value($:2); %*/ - } - | /* none */ - { - $$ = new_args_tail(p, 0, 0, 0, &@0); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, Qnil); %*/ + /*% ripper: [Qnil, Qnil, $:1] %*/ } ; @@ -5061,81 +5058,81 @@ excessed_comma : ',' } ; -block_param : f_arg ',' f_block_optarg ',' f_rest_arg opt_block_args_tail +block_param : f_arg ',' f_optarg(primary_value) ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, $5, 0, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), Qnil, get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, $:5, Qnil, *$:6[0..2]) %*/ } - | f_arg ',' f_block_optarg ',' f_rest_arg ',' f_arg opt_block_args_tail + | f_arg ',' f_optarg(primary_value) ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, $5, $7, $8, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), get_value($:7), get_value($:8)) %*/ + /*% ripper: params!($:1, $:3, $:5, $:7, *$:8[0..2]) %*/ } - | f_arg ',' f_block_optarg opt_block_args_tail + | f_arg ',' f_optarg(primary_value) opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, 0, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, $:3, Qnil, Qnil, *$:4[0..2]) %*/ } - | f_arg ',' f_block_optarg ',' f_arg opt_block_args_tail + | f_arg ',' f_optarg(primary_value) ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, $3, 0, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, Qnil, $:5, *$:6[0..2]) %*/ } - | f_arg ',' f_rest_arg opt_block_args_tail + | f_arg ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, Qnil, $:3, Qnil, *$:4[0..2]) %*/ } | f_arg excessed_comma { $$ = new_args_tail(p, 0, 0, 0, &@2); $$ = new_args(p, $1, 0, $2, 0, $$, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:2), Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!($:1, Qnil, $:2, Qnil, Qnil, Qnil, Qnil) %*/ } - | f_arg ',' f_rest_arg ',' f_arg opt_block_args_tail + | f_arg ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, Qnil, $:3, $:5, *$:6[0..2]) %*/ } - | f_arg opt_block_args_tail + | f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, $1, 0, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!($:1, Qnil, Qnil, Qnil, *$:2[0..2]) %*/ } - | f_block_optarg ',' f_rest_arg opt_block_args_tail + | f_optarg(primary_value) ',' f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, Qnil, *$:4[0..2]) %*/ } - | f_block_optarg ',' f_rest_arg ',' f_arg opt_block_args_tail + | f_optarg(primary_value) ',' f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, $:5, *$:6[0..2]) %*/ } - | f_block_optarg opt_block_args_tail + | f_optarg(primary_value) opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, Qnil, *$:2[0..2]) %*/ } - | f_block_optarg ',' f_arg opt_block_args_tail + | f_optarg(primary_value) ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, $1, 0, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, $:3, *$:4[0..2]) %*/ } - | f_rest_arg opt_block_args_tail + | f_rest_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, 0, $1, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, Qnil, *$:2[0..2]) %*/ } - | f_rest_arg ',' f_arg opt_block_args_tail + | f_rest_arg ',' f_arg opt_args_tail(block_args_tail) { $$ = new_args(p, 0, 0, $1, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, $:3, *$:4[0..2]) %*/ } | block_args_tail { $$ = new_args(p, 0, 0, 0, 0, $1, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, get_value($:1)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, *$:1[0..2]) %*/ } ; @@ -5152,12 +5149,7 @@ block_param_def : '|' opt_bv_decl '|' p->max_numparam = ORDINAL_PARAM; p->ctxt.in_argdef = 0; $$ = 0; - /*%%%*/ - /*% - VALUE val = dispatch7(params, Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil); - val = dispatch2(block_var, val, get_value($:2)); - set_value(val); - %*/ + /*% ripper: block_var!(params!(Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil), $:2) %*/ } | '|' block_param opt_bv_decl '|' { @@ -5178,20 +5170,20 @@ opt_bv_decl : '\n'? | '\n'? ';' bv_decls '\n'? { $$ = 0; - /*% ripper: get_value($:3) %*/ + /*% ripper: $:3 %*/ } ; bv_decls : bvar - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | bv_decls ',' bvar - /*% ripper[brace]: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper[brace]: rb_ary_push($:1, $:3) %*/ ; bvar : tIDENTIFIER { new_bv(p, $1); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | f_bad_arg { @@ -5216,13 +5208,12 @@ it_id : { } ; -lambda : tLAMBDA[dyna] +lambda : tLAMBDA[lpar] { token_info_push(p, "->", &@1); - $<vars>dyna = dyna_push(p); - $<num>$ = p->lex.lpar_beg; + $$ = dyna_push(p); p->lex.lpar_beg = p->lex.paren_nest; - }[lpar] + }[dyna]<vars> max_numparam numparam it_id allow_exits f_larglist[args] { @@ -5232,7 +5223,7 @@ lambda : tLAMBDA[dyna] { int max_numparam = p->max_numparam; ID it_id = p->it_id; - p->lex.lpar_beg = $<num>lpar; + p->lex.lpar_beg = $lpar; p->max_numparam = $max_numparam; p->it_id = $it_id; restore_block_exit(p, $allow_exits); @@ -5247,7 +5238,7 @@ lambda : tLAMBDA[dyna] } /*% ripper: lambda!($:args, $:body) %*/ numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; @@ -5271,7 +5262,7 @@ lambda_body : tLAMBEG compstmt '}' { token_info_pop(p, "}", &@3); $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | keyword_do_LAMBDA { @@ -5280,7 +5271,7 @@ lambda_body : tLAMBEG compstmt '}' bodystmt k_end { $$ = $3; - /*% ripper: get_value($:3); %*/ + /*% ripper: $:3 %*/ } ; @@ -5288,7 +5279,7 @@ do_block : k_do_block do_body k_end { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -5306,13 +5297,23 @@ block_call : command do_block } | block_call call_op2 operation2 opt_paren_args { + bool has_args = $4 != 0; + if (NODE_EMPTY_ARGS_P($4)) $4 = 0; $$ = new_qcall(p, $2, $1, $3, $4, &@3, &@$); - /*% ripper: opt_event(:method_add_arg!, call!($:1, $:2, $:3), $:4) %*/ + /*% ripper: call!($:1, $:2, $:3) %*/ + if (has_args) { + /*% ripper: method_add_arg!($:$, $:4) %*/ + } } | block_call call_op2 operation2 opt_paren_args brace_block { + bool has_args = $5 != 0; + if (NODE_EMPTY_ARGS_P($5)) $5 = 0; $$ = new_command_qcall(p, $2, $1, $3, $4, $5, &@3, &@$); - /*% ripper: opt_event(:method_add_block!, command_call!($:1, $:2, $:3, $:4), $:5) %*/ + /*% ripper: command_call!($:1, $:2, $:3, $:4) %*/ + if (has_args) { + /*% ripper: method_add_block!($:$, $:5) %*/ + } } | block_call call_op2 operation2 command_args do_block { @@ -5330,9 +5331,14 @@ method_call : fcall paren_args } | primary_value call_op operation2 opt_paren_args { + bool has_args = $4 != 0; + if (NODE_EMPTY_ARGS_P($4)) $4 = 0; $$ = new_qcall(p, $2, $1, $3, $4, &@3, &@$); nd_set_line($$, @3.end_pos.lineno); - /*% ripper: opt_event(:method_add_arg!, call!($:1, $:2, $:3), $:4) %*/ + /*% ripper: call!($:1, $:2, $:3) %*/ + if (has_args) { + /*% ripper: method_add_arg!($:$, $:4) %*/ + } } | primary_value tCOLON2 operation2 paren_args { @@ -5379,17 +5385,17 @@ brace_block : '{' brace_body '}' { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | k_do do_body k_end { $$ = $2; set_embraced_location($$, &@1, &@3); - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; -brace_body : {$<vars>$ = dyna_push(p);}[dyna] +brace_body : {$$ = dyna_push(p);}[dyna]<vars> max_numparam numparam it_id allow_exits opt_block_param[args] compstmt { @@ -5402,28 +5408,28 @@ brace_body : {$<vars>$ = dyna_push(p);}[dyna] /*% ripper: brace_block!($:args, $:compstmt) %*/ restore_block_exit(p, $allow_exits); numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; do_body : { - $<vars>$ = dyna_push(p); + $$ = dyna_push(p); CMDARG_PUSH(0); - }[dyna] + }[dyna]<vars> max_numparam numparam it_id allow_exits opt_block_param[args] bodystmt { int max_numparam = p->max_numparam; ID it_id = p->it_id; p->max_numparam = $max_numparam; - p->it_id = $<id>it_id; + p->it_id = $it_id; $args = args_with_numbered(p, $args, max_numparam, it_id); $$ = NEW_ITER($args, $bodystmt, &@$); /*% ripper: do_block!($:args, $:bodystmt) %*/ CMDARG_POP(); restore_block_exit(p, $allow_exits); numparam_pop(p, $numparam); - dyna_pop(p, $<vars>dyna); + dyna_pop(p, $dyna); } ; @@ -5516,28 +5522,28 @@ p_top_expr_body : p_expr { $$ = new_array_pattern_tail(p, 0, 1, 0, 0, &@$); $$ = new_array_pattern(p, 0, $1, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, get_value($:1), rb_ary_new()); %*/ + /*% ripper: aryptn!(Qnil, [$:1], Qnil, Qnil) %*/ } | p_expr ',' p_args { $$ = new_array_pattern(p, 0, $1, $3, &@$); nd_set_first_loc($$, @1.beg_pos); - /*% ripper: ripper_new_array_pattern(p, Qnil, get_value($:1), get_value($:3)); %*/ + /*% ripper: aryptn!(Qnil, aryptn_pre_args(p, $:1, $:3[0]), *$:3[1..2]) %*/ } | p_find { $$ = new_find_pattern(p, 0, $1, &@$); - /*% ripper: ripper_new_find_pattern(p, Qnil, get_value($:1)); %*/ + /*% ripper: fndptn!(Qnil, *$:1[0..2]) %*/ } | p_args_tail { $$ = new_array_pattern(p, 0, 0, $1, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, get_value($:1)); %*/ + /*% ripper: aryptn!(Qnil, *$:1[0..2]) %*/ } | p_kwargs { $$ = new_hash_pattern(p, 0, $1, &@$); - /*% ripper: ripper_new_hash_pattern(p, Qnil, get_value($:1)); %*/ + /*% ripper: hshptn!(Qnil, *$:1[0..1]) %*/ } ; @@ -5565,14 +5571,14 @@ p_alt : p_alt '|' p_expr_basic p_lparen : '(' p_pktbl { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; p_lbracket : '[' p_pktbl { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; @@ -5583,70 +5589,70 @@ p_expr_basic : p_value pop_pktbl(p, $p_pktbl); $$ = new_array_pattern(p, $p_const, 0, $p_args, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!($:p_const, *$:p_args[0..2]) %*/ } | p_const p_lparen[p_pktbl] p_find rparen { pop_pktbl(p, $p_pktbl); $$ = new_find_pattern(p, $p_const, $p_find, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_find_pattern(p, get_value($:p_const), get_value($:p_find)); %*/ + /*% ripper: fndptn!($:p_const, *$:p_find[0..2]) %*/ } | p_const p_lparen[p_pktbl] p_kwargs rparen { pop_pktbl(p, $p_pktbl); $$ = new_hash_pattern(p, $p_const, $p_kwargs, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_hash_pattern(p, get_value($:p_const), get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!($:p_const, *$:p_kwargs[0..1]) %*/ } | p_const '(' rparen { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, $p_const, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!($:p_const, Qnil, Qnil, Qnil) %*/ } | p_const p_lbracket[p_pktbl] p_args rbracket { pop_pktbl(p, $p_pktbl); $$ = new_array_pattern(p, $p_const, 0, $p_args, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_array_pattern(p, get_value($:p_const), Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!($:p_const, *$:p_args[0..2]) %*/ } | p_const p_lbracket[p_pktbl] p_find rbracket { pop_pktbl(p, $p_pktbl); $$ = new_find_pattern(p, $p_const, $p_find, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_find_pattern(p, get_value($:p_const), get_value($:p_find)); %*/ + /*% ripper: fndptn!($:p_const, *$:p_find[0..2]) %*/ } | p_const p_lbracket[p_pktbl] p_kwargs rbracket { pop_pktbl(p, $p_pktbl); $$ = new_hash_pattern(p, $p_const, $p_kwargs, &@$); nd_set_first_loc($$, @p_const.beg_pos); - /*% ripper: ripper_new_hash_pattern(p, get_value($:p_const), get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!($:p_const, *$:p_kwargs[0..1]) %*/ } | p_const '[' rbracket { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, $1, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, get_value($:1), Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!($:1, Qnil, Qnil, Qnil) %*/ } | tLBRACK p_args rbracket { $$ = new_array_pattern(p, 0, 0, $p_args, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, get_value($:p_args)); %*/ + /*% ripper: aryptn!(Qnil, *$:p_args[0..2]) %*/ } | tLBRACK p_find rbracket { $$ = new_find_pattern(p, 0, $p_find, &@$); - /*% ripper: ripper_new_find_pattern(p, Qnil, get_value($:p_find)); %*/ + /*% ripper: fndptn!(Qnil, *$:p_find[0..2]) %*/ } | tLBRACK rbracket { $$ = new_array_pattern_tail(p, 0, 0, 0, 0, &@$); $$ = new_array_pattern(p, 0, 0, $$, &@$); - /*% ripper: ripper_new_array_pattern(p, Qnil, Qnil, rb_ary_new()); %*/ + /*% ripper: aryptn!(Qnil, Qnil, Qnil, Qnil) %*/ } | tLBRACE p_pktbl lex_ctxt[ctxt] { @@ -5657,24 +5663,19 @@ p_expr_basic : p_value pop_pktbl(p, $p_pktbl); p->ctxt.in_kwarg = $ctxt.in_kwarg; $$ = new_hash_pattern(p, 0, $p_kwargs, &@$); - /*% ripper: ripper_new_hash_pattern(p, Qnil, get_value($:p_kwargs)); %*/ + /*% ripper: hshptn!(Qnil, *$:p_kwargs[0..1]) %*/ } | tLBRACE rbrace { $$ = new_hash_pattern_tail(p, 0, 0, &@$); $$ = new_hash_pattern(p, 0, $$, &@$); - /*%%%*/ - /*% - VALUE val = ripper_new_hash_pattern_tail(p, Qnil, 0); - val = ripper_new_hash_pattern(p, Qnil, val); - set_value(val); - %*/ + /*% ripper: hshptn!(Qnil, Qnil, Qnil) %*/ } | tLPAREN p_pktbl p_expr rparen { pop_pktbl(p, $p_pktbl); $$ = $p_expr; - /*% ripper: get_value($:p_expr); %*/ + /*% ripper: $:p_expr %*/ } ; @@ -5682,44 +5683,27 @@ p_args : p_expr { NODE *pre_args = NEW_LIST($1, &@$); $$ = new_array_pattern_tail(p, pre_args, 0, 0, 0, &@$); - /*%%%*/ - /*% - VALUE ary = rb_ary_new_from_args(1, get_value($:1)); - set_value(rb_ary_new_from_args(3, ary, Qnil, Qnil)); - %*/ + /*% ripper: [[$:1], Qnil, Qnil] %*/ } | p_args_head { $$ = new_array_pattern_tail(p, $1, 1, 0, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), Qnil, Qnil)); - %*/ + /*% ripper: [$:1, Qnil, Qnil] %*/ } | p_args_head p_arg { $$ = new_array_pattern_tail(p, list_concat($1, $2), 0, 0, 0, &@$); - /*%%%*/ - /*% - VALUE pre_args = rb_ary_concat(get_value($:1), get_value($:2)); - set_value(rb_ary_new_from_args(3, pre_args, Qnil, Qnil)); - %*/ + /*% ripper: [rb_ary_concat($:1, $:2), Qnil, Qnil] %*/ } | p_args_head p_rest { $$ = new_array_pattern_tail(p, $1, 1, $2, 0, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), get_value($:2), Qnil)); - %*/ + /*% ripper: [$:1, $:2, Qnil] %*/ } | p_args_head p_rest ',' p_args_post { $$ = new_array_pattern_tail(p, $1, 1, $2, $4, &@$); - /*%%%*/ - /*% - set_value(rb_ary_new_from_args(3, get_value($:1), get_value($:2), get_value($:4))); - %*/ + /*% ripper: [$:1, $:2, $:4] %*/ } | p_args_tail ; @@ -5731,26 +5715,26 @@ p_args_head : p_arg ',' | p_args_head p_arg ',' { $$ = list_concat($1, $2); - /*% ripper: rb_ary_concat(get_value($:1), get_value($:2)) %*/ + /*% ripper: rb_ary_concat($:1, $:2) %*/ } ; p_args_tail : p_rest { $$ = new_array_pattern_tail(p, 0, 1, $1, 0, &@$); - /*% ripper: ripper_new_array_pattern_tail(p, Qnil, get_value($:1), Qnil); %*/ + /*% ripper: [Qnil, $:1, Qnil] %*/ } | p_rest ',' p_args_post { $$ = new_array_pattern_tail(p, 0, 1, $1, $3, &@$); - /*% ripper: ripper_new_array_pattern_tail(p, Qnil, get_value($:1), get_value($:3)); %*/ + /*% ripper: [Qnil, $:1, $:3] %*/ } ; p_find : p_rest ',' p_args_post ',' p_rest { $$ = new_find_pattern_tail(p, $1, $3, $5, &@$); - /*% ripper: ripper_new_find_pattern_tail(p, get_value($:1), get_value($:3), get_value($:5)) %*/ + /*% ripper: [$:1, $:3, $:5] %*/ } ; @@ -5758,13 +5742,13 @@ p_find : p_rest ',' p_args_post ',' p_rest p_rest : tSTAR tIDENTIFIER { error_duplicate_pattern_variable(p, $2, &@2); + /*% ripper: var_field!($:2) %*/ $$ = assignable(p, $2, 0, &@$); - /*% ripper: ripper_assignable(p, $2, var_field(p, get_value($:2))) %*/ } | tSTAR { $$ = 0; - /*% ripper: var_field(p, Qnil) %*/ + /*% ripper: var_field!(Qnil) %*/ } ; @@ -5772,45 +5756,45 @@ p_args_post : p_arg | p_args_post ',' p_arg { $$ = list_concat($1, $3); - /*% ripper: rb_ary_concat(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_concat($:1, $:3) %*/ } ; p_arg : p_expr { $$ = NEW_LIST($1, &@$); - /*% ripper: rb_ary_new_from_args(1, get_value($:1)) %*/ + /*% ripper: [$:1] %*/ } ; p_kwargs : p_kwarg ',' p_any_kwrest { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), $3, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), get_value($:3)) %*/ + /*% ripper: [$:1, $:3] %*/ } | p_kwarg { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), 0, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), 0) %*/ + /*% ripper: [$:1, Qnil] %*/ } | p_kwarg ',' { $$ = new_hash_pattern_tail(p, new_unique_key_hash(p, $1, &@$), 0, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, get_value($:1), 0) %*/ + /*% ripper: [$:1, Qnil] %*/ } | p_any_kwrest { $$ = new_hash_pattern_tail(p, new_hash(p, 0, &@$), $1, &@$); - /*% ripper: ripper_new_hash_pattern_tail(p, rb_ary_new(), get_value($:1)) %*/ + /*% ripper: [[], $:1] %*/ } ; p_kwarg : p_kw - /*% ripper[brace]: rb_ary_new_from_args(1, get_value($:1)) %*/ + /*% ripper[brace]: [$:1] %*/ | p_kwarg ',' p_kw { $$ = list_concat($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; @@ -5818,7 +5802,7 @@ p_kw : p_kw_label p_expr { error_duplicate_pattern_key(p, $1, &@1); $$ = list_append(p, NEW_LIST(NEW_SYM(rb_id2str($1), &@1), &@$), $2); - /*% ripper: rb_ary_new_from_args(2, get_value($:1), get_value($:2)) %*/ + /*% ripper: [$:1, $:2] %*/ } | p_kw_label { @@ -5828,7 +5812,7 @@ p_kw : p_kw_label p_expr } error_duplicate_pattern_variable(p, $1, &@1); $$ = list_append(p, NEW_LIST(NEW_SYM(rb_id2str($1), &@$), &@$), assignable(p, $1, 0, &@$)); - /*% ripper: rb_ary_new_from_args(2, ripper_assignable(p, $1, get_value($:1)), Qnil) %*/ + /*% ripper: [$:1, Qnil] %*/ } ; @@ -5844,19 +5828,19 @@ p_kw_label : tLABEL yyerror1(&loc, "symbol literal with interpolation is not allowed"); $$ = rb_intern_str(STR_NEW0()); } - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; p_kwrest : kwrest_mark tIDENTIFIER { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: var_field!($:2) %*/ } | kwrest_mark { $$ = 0; - /*% ripper: 0; %*/ + /*% ripper: Qnil %*/ } ; @@ -5870,7 +5854,7 @@ p_any_kwrest : p_kwrest | p_kwnorest { $$ = idNil; - /*% ripper: ID2VAL(idNil) %*/ + /*% ripper: var_field!(ID2VAL(idNil)) %*/ } ; @@ -5937,8 +5921,8 @@ p_primitive : literal p_variable : tIDENTIFIER { error_duplicate_pattern_variable(p, $1, &@1); + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } ; @@ -6013,7 +5997,7 @@ opt_rescue : k_rescue exc_list exc_var then exc_list : arg_value { $$ = NEW_LIST($1, &@$); - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper: rb_ary_new3(1, $:1) %*/ } | mrhs { @@ -6025,7 +6009,7 @@ exc_list : arg_value exc_var : tASSOC lhs { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none ; @@ -6053,7 +6037,7 @@ strings : string node = evstr2dstr(p, node); } $$ = node; - /*% ripper: get_value($:1); %*/ + /*% ripper: $:1 %*/ } ; @@ -6068,32 +6052,26 @@ string : tCHAR string1 : tSTRING_BEG string_contents tSTRING_END { - /*%%%*/ - /*% - int indent = p->heredoc_indent; - %*/ $$ = heredoc_dedent(p, $2); if ($$) nd_set_loc($$, &@$); - /*%%%*/ - /*% - VALUE val = dispatch1(string_literal, ripper_heredoc_dedent(p, indent, get_value($:2))); - set_value(val); - %*/ + /*% ripper: $:2 %*/ + if (p->heredoc_indent > 0) { + /*% ripper: heredoc_dedent!($:$, INT2NUM(%{p->heredoc_indent})) %*/ + p->heredoc_indent = 0; + } + /*% ripper: string_literal!($:$) %*/ } ; xstring : tXSTRING_BEG xstring_contents tSTRING_END { - /*%%%*/ - /*% - int indent = p->heredoc_indent; - %*/ $$ = new_xstring(p, heredoc_dedent(p, $2), &@$); - /*%%%*/ - /*% - VALUE val = dispatch1(xstring_literal, ripper_heredoc_dedent(p, indent, get_value($:2))); - set_value(val); - %*/ + /*% ripper: $:2 %*/ + if (p->heredoc_indent > 0) { + /*% ripper: heredoc_dedent!($:$, INT2NUM(%{p->heredoc_indent})) %*/ + p->heredoc_indent = 0; + } + /*% ripper: xstring_literal!($:$) %*/ } ; @@ -6104,11 +6082,7 @@ regexp : tREGEXP_BEG regexp_contents tREGEXP_END } ; -words : tWORDS_BEG ' '+ word_list tSTRING_END - { - $$ = make_list($3, &@$); - /*% ripper: array!($:3) %*/ - } +words : words(tWORDS_BEG, word_list) <node> ; word_list : /* none */ @@ -6132,11 +6106,7 @@ word : string_content } ; -symbols : tSYMBOLS_BEG ' '+ symbol_list tSTRING_END - { - $$ = make_list($3, &@$); - /*% ripper: array!($:3) %*/ - } +symbols : words(tSYMBOLS_BEG, symbol_list) <node> ; symbol_list : /* none */ @@ -6151,18 +6121,10 @@ symbol_list : /* none */ } ; -qwords : tQWORDS_BEG ' '+ qword_list tSTRING_END - { - $$ = make_list($3, &@$); - /*% ripper: array!($:3) %*/ - } +qwords : words(tQWORDS_BEG, qword_list) <node> ; -qsymbols : tQSYMBOLS_BEG ' '+ qsym_list tSTRING_END - { - $$ = make_list($3, &@$); - /*% ripper: array!($:3) %*/ - } +qsymbols : words(tQSYMBOLS_BEG, qsym_list) <node> ; qword_list : /* none */ @@ -6247,47 +6209,46 @@ regexp_contents: /* none */ ; string_content : tSTRING_CONTENT - /*% ripper[brace]: get_value($:1); %*/ + /*% ripper[brace]: $:1 %*/ | tSTRING_DVAR { /* need to backup p->lex.strterm so that a string literal `%&foo,#$&,bar&` can be parsed */ - $<strterm>$ = p->lex.strterm; + $$ = p->lex.strterm; p->lex.strterm = 0; SET_LEX_STATE(EXPR_BEG); - } + }<strterm> string_dvar { - p->lex.strterm = $<strterm>2; + p->lex.strterm = $2; $$ = NEW_EVSTR($3, &@$); nd_set_line($$, @3.end_pos.lineno); /*% ripper: string_dvar!($:3) %*/ } - | tSTRING_DBEG[term] + | tSTRING_DBEG[state] { CMDARG_PUSH(0); COND_PUSH(0); /* need to backup p->lex.strterm so that a string literal `%!foo,#{ !0 },bar!` can be parsed */ - $<strterm>term = p->lex.strterm; + $$ = p->lex.strterm; p->lex.strterm = 0; - $<num>$ = p->lex.state; SET_LEX_STATE(EXPR_BEG); - }[state] + }[term]<strterm> { - $<num>$ = p->lex.brace_nest; + $$ = p->lex.brace_nest; p->lex.brace_nest = 0; - }[brace] + }[brace]<num> { - $<num>$ = p->heredoc_indent; + $$ = p->heredoc_indent; p->heredoc_indent = 0; - }[indent] + }[indent]<num> compstmt string_dend { COND_POP(); CMDARG_POP(); - p->lex.strterm = $<strterm>term; - SET_LEX_STATE($<num>state); - p->lex.brace_nest = $<num>brace; - p->heredoc_indent = $<num>indent; + p->lex.strterm = $term; + SET_LEX_STATE($state); + p->lex.brace_nest = $brace; + p->heredoc_indent = $indent; p->heredoc_line_indent = -1; if ($compstmt) nd_unset_fl_newline($compstmt); $$ = new_evstr(p, $compstmt, &@$); @@ -6376,17 +6337,12 @@ keyword_variable: keyword_nil {$$ = KWD2EID(nil, $1);} var_ref : user_variable { if (!($$ = gettable(p, $1, &@$))) $$ = NEW_ERROR(&@$); - /*%%%*/ - /*% - if (id_is_var(p, $1)) { - VALUE val = dispatch1(var_ref, get_value($:1)); - set_value(val); + if (ifdef_ripper(id_is_var(p, $1), false)) { + /*% ripper: var_ref!($:1) %*/ } else { - VALUE val = dispatch1(vcall, get_value($:1)); - set_value(val); + /*% ripper: vcall!($:1) %*/ } - %*/ } | keyword_variable { @@ -6397,13 +6353,13 @@ var_ref : user_variable var_lhs : user_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } | keyword_variable { + /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); - /*% ripper: ripper_assignable(p, $1, var_field(p, get_value($:1))) %*/ } ; @@ -6419,7 +6375,7 @@ superclass : '<' expr_value term { $$ = $3; - /*% ripper: get_value($:3); %*/ + /*% ripper: $:3 %*/ } | /* none */ { @@ -6434,7 +6390,7 @@ f_opt_paren_args: f_paren_args p->ctxt.in_argdef = 0; $$ = new_args_tail(p, 0, 0, 0, &@0); $$ = new_args(p, 0, 0, 0, 0, $$, &@0); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, Qnil, Qnil, Qnil) %*/ } ; @@ -6450,138 +6406,126 @@ f_paren_args : '(' f_args rparen f_arglist : f_paren_args | { - $<ctxt>$ = p->ctxt; + $$ = p->ctxt; p->ctxt.in_kwarg = 1; p->ctxt.in_argdef = 1; SET_LEX_STATE(p->lex.state|EXPR_LABEL); /* force for args */ - } + }<ctxt> f_args term { - p->ctxt.in_kwarg = $<ctxt>1.in_kwarg; + p->ctxt.in_kwarg = $1.in_kwarg; p->ctxt.in_argdef = 0; $$ = $2; SET_LEX_STATE(EXPR_BEG); p->command_start = TRUE; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } ; -args_tail : f_kwarg ',' f_kwrest opt_f_block_arg +args_tail : f_kwarg(f_kw) ',' f_kwrest opt_f_block_arg { $$ = new_args_tail(p, $1, $3, $4, &@3); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), get_value($:3), get_value($:4)); %*/ + /*% ripper: [$:1, $:3, $:4] %*/ } - | f_kwarg opt_f_block_arg + | f_kwarg(f_kw) opt_f_block_arg { $$ = new_args_tail(p, $1, 0, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, get_value($:1), Qnil, get_value($:2)); %*/ + /*% ripper: [$:1, Qnil, $:2] %*/ } | f_any_kwrest opt_f_block_arg { $$ = new_args_tail(p, 0, $1, $2, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), get_value($:2)); %*/ + /*% ripper: [Qnil, $:1, $:2] %*/ } | f_block_arg { $$ = new_args_tail(p, 0, 0, $1, &@1); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, get_value($:1)); %*/ + /*% ripper: [Qnil, Qnil, $:1] %*/ } | args_forward { add_forwarding_args(p); $$ = new_args_tail(p, 0, $1, arg_FWD_BLOCK, &@1); $$->nd_ainfo.forwarding = 1; - /*% ripper: rb_ary_new_from_args(3, Qnil, get_value($:1), Qnil); %*/ + /*% ripper: [Qnil, $:1, Qnil] %*/ } ; -opt_args_tail : ',' args_tail - { - $$ = $2; - /*% ripper: get_value($:2); %*/ - } - | /* none */ - { - $$ = new_args_tail(p, 0, 0, 0, &@0); - /*% ripper: rb_ary_new_from_args(3, Qnil, Qnil, Qnil); %*/ - } - ; - -f_args : f_arg ',' f_optarg ',' f_rest_arg opt_args_tail +f_args : f_arg ',' f_optarg(arg_value) ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, $5, 0, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), Qnil, get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, $:5, Qnil, *$:6[0..2]) %*/ } - | f_arg ',' f_optarg ',' f_rest_arg ',' f_arg opt_args_tail + | f_arg ',' f_optarg(arg_value) ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, $5, $7, $8, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), get_value($:5), get_value($:7), get_value($:8)) %*/ + /*% ripper: params!($:1, $:3, $:5, $:7, *$:8[0..2]) %*/ } - | f_arg ',' f_optarg opt_args_tail + | f_arg ',' f_optarg(arg_value) opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, 0, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, $:3, Qnil, Qnil, *$:4[0..2]) %*/ } - | f_arg ',' f_optarg ',' f_arg opt_args_tail + | f_arg ',' f_optarg(arg_value) ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, $3, 0, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), get_value($:3), Qnil, get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, $:3, Qnil, $:5, *$:6[0..2]) %*/ } - | f_arg ',' f_rest_arg opt_args_tail + | f_arg ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!($:1, Qnil, $:3, Qnil, *$:4[0..2]) %*/ } - | f_arg ',' f_rest_arg ',' f_arg opt_args_tail + | f_arg ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!($:1, Qnil, $:3, $:5, *$:6[0..2]) %*/ } - | f_arg opt_args_tail + | f_arg opt_args_tail(args_tail) { $$ = new_args(p, $1, 0, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, get_value($:1), Qnil, Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!($:1, Qnil, Qnil, Qnil, *$:2[0..2]) %*/ } - | f_optarg ',' f_rest_arg opt_args_tail + | f_optarg(arg_value) ',' f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, $3, 0, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), Qnil, get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, Qnil, *$:4[0..2]) %*/ } - | f_optarg ',' f_rest_arg ',' f_arg opt_args_tail + | f_optarg(arg_value) ',' f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, $3, $5, $6, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), get_value($:3), get_value($:5), get_value($:6)) %*/ + /*% ripper: params!(Qnil, $:1, $:3, $:5, *$:6[0..2]) %*/ } - | f_optarg opt_args_tail + | f_optarg(arg_value) opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, 0, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, Qnil, *$:2[0..2]) %*/ } - | f_optarg ',' f_arg opt_args_tail + | f_optarg(arg_value) ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, $1, 0, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, get_value($:1), Qnil, get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, $:1, Qnil, $:3, *$:4[0..2]) %*/ } - | f_rest_arg opt_args_tail + | f_rest_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, 0, $1, 0, $2, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), Qnil, get_value($:2)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, Qnil, *$:2[0..2]) %*/ } - | f_rest_arg ',' f_arg opt_args_tail + | f_rest_arg ',' f_arg opt_args_tail(args_tail) { $$ = new_args(p, 0, 0, $1, $3, $4, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, get_value($:1), get_value($:3), get_value($:4)) %*/ + /*% ripper: params!(Qnil, Qnil, $:1, $:3, *$:4[0..2]) %*/ } | args_tail { $$ = new_args(p, 0, 0, 0, 0, $1, &@$); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, get_value($:1)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, *$:1[0..2]) %*/ } | /* none */ { $$ = new_args_tail(p, 0, 0, 0, &@0); $$ = new_args(p, 0, 0, 0, 0, $$, &@0); - /*% ripper: ripper_new_args(p, Qnil, Qnil, Qnil, Qnil, rb_ary_new_from_args(3, Qnil, Qnil, Qnil)) %*/ + /*% ripper: params!(Qnil, Qnil, Qnil, Qnil, Qnil, Qnil, Qnil) %*/ } ; @@ -6640,10 +6584,7 @@ f_norm_arg : f_bad_arg formal_argument(p, $1); p->max_numparam = ORDINAL_PARAM; $$ = $1; - /*%%%*/ - /*% - ripper_formal_argument(p, $1, get_value($:1)); - %*/ + /*% ripper: ripper_formal_argument(p, $1, $:1) %*/ } ; @@ -6660,7 +6601,7 @@ f_arg_item : f_arg_asgn { p->cur_arg = 0; $$ = NEW_ARGS_AUX($1, 1, &NULL_LOC); - /*% ripper: get_value($:1) %*/ + /*% ripper: $:1 %*/ } | tLPAREN f_margs rparen { @@ -6682,14 +6623,14 @@ f_arg_item : f_arg_asgn ; f_arg : f_arg_item - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | f_arg ',' f_arg_item { $$ = $1; $$->nd_plen++; $$->nd_next = block_append(p, $$->nd_next, $3->nd_next); rb_discard_node(p, (NODE *)$3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; @@ -6701,10 +6642,7 @@ f_label : tLABEL p->max_numparam = ORDINAL_PARAM; p->ctxt.in_argdef = 0; $$ = $1; - /*%%%*/ - /*% - ripper_formal_argument(p, $1, get_value($:1)); - %*/ + /*% ripper: ripper_formal_argument(p, $1, $:1) %*/ } ; @@ -6713,14 +6651,14 @@ f_kw : f_label arg_value p->cur_arg = 0; p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, $2, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:2)) %*/ + /*% ripper: [$:$, $:2] %*/ } | f_label { p->cur_arg = 0; p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, NODE_SPECIAL_REQUIRED_KEYWORD, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), 0) %*/ + /*% ripper: [$:$, 0] %*/ } ; @@ -6728,38 +6666,13 @@ f_block_kw : f_label primary_value { p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, $2, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:2)) %*/ + /*% ripper: [$:$, $:2] %*/ } | f_label { p->ctxt.in_argdef = 1; $$ = new_kw_arg(p, assignable(p, $1, NODE_SPECIAL_REQUIRED_KEYWORD, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), 0) %*/ - } - ; - -f_block_kwarg : f_block_kw - { - $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ - } - | f_block_kwarg ',' f_block_kw - { - $$ = kwd_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ - } - ; - - -f_kwarg : f_kw - { - $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ - } - | f_kwarg ',' f_kw - { - $$ = kwd_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: [$:$, 0] %*/ } ; @@ -6787,48 +6700,6 @@ f_kwrest : kwrest_mark tIDENTIFIER } ; -f_opt : f_arg_asgn f_eq arg_value - { - p->cur_arg = 0; - p->ctxt.in_argdef = 1; - $$ = NEW_OPT_ARG(assignable(p, $1, $3, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:3)) %*/ - } - ; - -f_block_opt : f_arg_asgn f_eq primary_value - { - p->cur_arg = 0; - p->ctxt.in_argdef = 1; - $$ = NEW_OPT_ARG(assignable(p, $1, $3, &@$), &@$); - /*% ripper: rb_assoc_new(ripper_assignable(p, $1, get_value($:1)), get_value($:3)) %*/ - } - ; - -f_block_optarg : f_block_opt - { - $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ - } - | f_block_optarg ',' f_block_opt - { - $$ = opt_arg_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ - } - ; - -f_optarg : f_opt - { - $$ = $1; - /*% ripper: rb_ary_new3(1, get_value($:1)) %*/ - } - | f_optarg ',' f_opt - { - $$ = opt_arg_append($1, $3); - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ - } - ; - restarg_mark : '*' | tSTAR ; @@ -6868,12 +6739,12 @@ f_block_arg : blkarg_mark tIDENTIFIER opt_f_block_arg : ',' f_block_arg { $$ = $2; - /*% ripper: get_value($:2); %*/ + /*% ripper: $:2 %*/ } | none { $$ = 0; - /*% ripper: Qnil; %*/ + /*% ripper: Qnil %*/ } ; @@ -6923,7 +6794,7 @@ assoc_list : none ; assocs : assoc - /*% ripper[brace]: rb_ary_new3(1, get_value($:1)) %*/ + /*% ripper[brace]: rb_ary_new3(1, $:1) %*/ | assocs ',' assoc { NODE *assocs = $1; @@ -6945,7 +6816,7 @@ assocs : assoc } } $$ = assocs; - /*% ripper: rb_ary_push(get_value($:1), get_value($:3)) %*/ + /*% ripper: rb_ary_push($:1, $:3) %*/ } ; @@ -7040,10 +6911,6 @@ terms : term none : /* none */ { $$ = 0; - /*%%%*/ - /*% - set_value(rb_ripper_none); - %*/ } ; %% @@ -7701,7 +7568,7 @@ yycompile0(VALUE arg) struct parser_params *p = (struct parser_params *)arg; int cov = FALSE; - if (!compile_for_eval && !NIL_P(p->ruby_sourcefile_string) && !e_option_supplied(p)) { + if (!compile_for_eval && p->ruby_sourcefile_string && !e_option_supplied(p)) { cov = TRUE; } @@ -7755,17 +7622,39 @@ yycompile0(VALUE arg) return TRUE; } +static void +set_arg_error(struct parser_params *p, const char *err) +{ + VALUE excargs[3]; + + excargs[0] = rb_eArgError; + excargs[1] = rb_str_new_cstr(err); + excargs[2] = rb_make_backtrace(); + rb_set_errinfo(rb_make_exception(3, excargs)); +} + static rb_ast_t * -yycompile(struct parser_params *p, VALUE fname, int line) +yycompile(struct parser_params *p, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, int line) { rb_ast_t *ast; - if (NIL_P(fname)) { - p->ruby_sourcefile_string = Qnil; + if (!fname_ptr) { + p->ruby_sourcefile_string = NULL; p->ruby_sourcefile = "(none)"; } else { - p->ruby_sourcefile_string = rb_str_to_interned_str(fname); - p->ruby_sourcefile = StringValueCStr(fname); + int w; + if (cstr_null_check(p, fname_ptr, fname_len, fname_enc, &w)) { + if (w) { + set_arg_error(p, "string contains null char"); + } + else { + set_arg_error(p, "string contains null byte"); + } + return rb_ast_new(); + } + + p->ruby_sourcefile_string = rb_parser_encoding_string_new(p, fname_ptr, fname_len, fname_enc); + p->ruby_sourcefile = fname_ptr; } p->ruby_sourceline = line - 1; @@ -7784,9 +7673,9 @@ yycompile(struct parser_params *p, VALUE fname, int line) #endif /* !RIPPER */ static rb_encoding * -must_be_ascii_compatible(struct parser_params *p, VALUE s) +must_be_ascii_compatible(struct parser_params *p, rb_parser_string_t *s) { - rb_encoding *enc = rb_enc_get(s); + rb_encoding *enc = rb_parser_str_get_encoding(s); if (!rb_enc_asciicompat(enc)) { rb_raise(rb_eArgError, "invalid source encoding"); } @@ -7796,25 +7685,24 @@ must_be_ascii_compatible(struct parser_params *p, VALUE s) static rb_parser_string_t * lex_getline(struct parser_params *p) { - rb_parser_string_t *str; - VALUE line = (*p->lex.gets)(p, p->lex.input, p->line_count); - if (NIL_P(line)) return 0; - must_be_ascii_compatible(p, line); + rb_parser_string_t *line = (*p->lex.gets)(p, p->lex.input, p->line_count); + if (!line) return 0; p->line_count++; - str = rb_str_to_parser_string(p, line); - string_buffer_append(p, str); - return str; + string_buffer_append(p, line); + must_be_ascii_compatible(p, line); + return line; } #ifndef RIPPER rb_ast_t* -rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line) +rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, + const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line) { p->lex.gets = gets; p->lex.input = input; p->lex.pbeg = p->lex.pcur = p->lex.pend = 0; - return yycompile(p, fname, line); + return yycompile(p, fname_ptr, fname_len, fname_enc, line); } #endif /* !RIPPER */ @@ -8580,6 +8468,10 @@ parser_update_heredoc_indent(struct parser_params *p, int c) } p->heredoc_line_indent = -1; } + else { + /* Whitespace only line has no indentation */ + p->heredoc_line_indent = 0; + } } return FALSE; } @@ -8835,6 +8727,7 @@ parser_peek_variable_name(struct parser_params *p) case '{': p->lex.pcur = ptr; p->command_start = TRUE; + yylval.state = p->lex.state; return tSTRING_DBEG; default: return 0; @@ -9113,7 +9006,6 @@ heredoc_dedent(struct parser_params *p, NODE *root) rb_parser_string_t *prev_lit = 0; if (indent <= 0) return root; - p->heredoc_indent = 0; if (!root) return root; prev_node = node = str_node = root; @@ -9157,17 +9049,6 @@ heredoc_dedent(struct parser_params *p, NODE *root) return root; } -#ifdef RIPPER -static VALUE -ripper_heredoc_dedent(struct parser_params *p, int indent, VALUE array) -{ - if (indent <= 0) return array; - p->heredoc_indent = 0; - dispatch2(heredoc_dedent, array, INT2NUM(indent)); - return array; -} -#endif - static int whole_match_p(struct parser_params *p, const char *eos, long len, int indent) { @@ -9533,7 +9414,7 @@ formal_argument(struct parser_params *p, ID id) } #ifdef RIPPER -static void +static VALUE ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs) { switch (id_type(id)) { @@ -9542,21 +9423,22 @@ ripper_formal_argument(struct parser_params *p, ID id, VALUE lhs) #define ERR(mesg) (dispatch2(param_error, WARN_S(mesg), lhs), ripper_error(p)) case ID_CONST: ERR("formal argument cannot be a constant"); - return; + break; case ID_INSTANCE: ERR("formal argument cannot be an instance variable"); - return; + break; case ID_GLOBAL: ERR("formal argument cannot be a global variable"); - return; + break; case ID_CLASS: ERR("formal argument cannot be a class variable"); - return; + break; default: ERR("formal argument must be local variable"); - return; + break; #undef ERR } + return lhs; } #endif @@ -9609,7 +9491,7 @@ parser_set_encode(struct parser_params *p, const char *name) error: excargs[0] = rb_eArgError; excargs[2] = rb_make_backtrace(); - rb_ary_unshift(excargs[2], rb_sprintf("%"PRIsVALUE":%d", p->ruby_sourcefile_string, p->ruby_sourceline)); + rb_ary_unshift(excargs[2], rb_sprintf("%"PRIsVALUE":%d", rb_str_new_mutable_parser_string(p->ruby_sourcefile_string), p->ruby_sourceline)); VALUE exc = rb_make_exception(3, excargs); ruby_show_error_line(p, exc, &(YYLTYPE)RUBY_INIT_YYLLOC(), p->ruby_sourceline, p->lex.lastline); rb_exc_raise(exc); @@ -10487,7 +10369,7 @@ parse_gvar(struct parser_params *p, const enum lex_state_e last_state) case '.': /* $.: last read line number */ case '=': /* $=: ignorecase */ case ':': /* $:: load path */ - case '<': /* $<: reading filename */ + case '<': /* $<: default input handle */ case '>': /* $>: default output handle */ case '\"': /* $": already loaded files */ tokadd(p, '$'); @@ -11217,6 +11099,7 @@ parser_yylex(struct parser_params *p) } if (c == '>') { SET_LEX_STATE(EXPR_ENDFN); + yylval.num = p->lex.lpar_beg; return tLAMBDA; } if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous(p, '-'))) { @@ -12586,10 +12469,10 @@ rb_node_line_new(struct parser_params *p, const YYLTYPE *loc) } static rb_node_file_t * -rb_node_file_new(struct parser_params *p, VALUE str, const YYLTYPE *loc) +rb_node_file_new(struct parser_params *p, rb_parser_string_t *str, const YYLTYPE *loc) { rb_node_file_t *n = NODE_NEWNODE(NODE_FILE, rb_node_file_t, loc); - n->path = rb_str_to_parser_string(p, str); + n->path = str; return n; } @@ -12838,7 +12721,6 @@ string_literal_head(struct parser_params *p, enum node_type htype, NODE *head) return lit; } -#ifndef RIPPER static rb_parser_string_t * rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *orig) { @@ -12849,7 +12731,6 @@ rb_parser_string_deep_copy(struct parser_params *p, const rb_parser_string_t *or copy->enc = orig->enc; return copy; } -#endif /* concat two string literals */ static NODE * @@ -13183,9 +13064,13 @@ gettable(struct parser_params *p, ID id, const YYLTYPE *loc) return NEW_FALSE(loc); case keyword__FILE__: { - VALUE file = p->ruby_sourcefile_string; - if (NIL_P(file)) - file = rb_str_new(0, 0); + rb_parser_string_t *file; + if (p->ruby_sourcefile_string) { + file = rb_parser_string_deep_copy(p, p->ruby_sourcefile_string); + } + else { + file = STRING_NEW0(); + } node = NEW_FILE(file, loc); } return node; @@ -13449,8 +13334,8 @@ check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) else { st_data_t line; if (st_lookup(p->case_labels, (st_data_t)arg, &line)) { - rb_warning1("duplicated 'when' clause with line %d is ignored", - WARN_I((int)line)); + rb_warning2("'when' clause on line %d duplicates 'when' clause on line %d and is ignored", + WARN_I((int)nd_line(arg)), WARN_I((int)line)); return; } } @@ -13743,19 +13628,11 @@ assignable(struct parser_params *p, ID id, NODE *val, const YYLTYPE *loc) /* TODO: FIXME */ #ifndef RIPPER if (err) yyerror1(loc, err); +#else + if (err) set_value(assign_error(p, err, p->s_lvalue)); #endif return NEW_ERROR(loc); } -#ifdef RIPPER -static VALUE -ripper_assignable(struct parser_params *p, ID id, VALUE lhs) -{ - const char *err = 0; - assignable0(p, id, &err); - if (err) lhs = assign_error(p, err, lhs); - return lhs; -} -#endif static int is_private_local_id(struct parser_params *p, ID name) @@ -13882,17 +13759,6 @@ rb_backref_error(struct parser_params *p, NODE *node) #ifdef RIPPER static VALUE -defs(struct parser_params *p, VALUE head, VALUE args, VALUE bodystmt) -{ - return dispatch5(defs, - rb_ary_entry(head, 0), /* nd_recv */ - rb_ary_entry(head, 1), /* dot_or_colon */ - rb_ary_entry(head, 2), /* nd_mid */ - args, - bodystmt); -} - -static VALUE backref_error(struct parser_params *p, NODE *node, VALUE expr) { VALUE mesg = rb_str_new_cstr("Can't set variable "); @@ -15047,20 +14913,16 @@ static NODE * const_decl(struct parser_params *p, NODE *path, const YYLTYPE *loc) { if (p->ctxt.in_def) { +#ifndef RIPPER yyerror1(loc, "dynamic constant assignment"); +#else + set_value(assign_error(p, "dynamic constant assignment", p->s_lvalue)); +#endif } return NEW_CDECL(0, 0, (path), p->ctxt.shareable_constant_value, loc); } -#ifdef RIPPER -static VALUE -ripper_const_decl(struct parser_params *p, VALUE path) -{ - if (p->ctxt.in_def) { - path = assign_error(p, "dynamic constant assignment", path); - } - return path; -} +#ifdef RIPPER static VALUE assign_error(struct parser_params *p, const char *mesg, VALUE a) { @@ -15068,12 +14930,6 @@ assign_error(struct parser_params *p, const char *mesg, VALUE a) ripper_error(p); return a; } - -static VALUE -var_field(struct parser_params *p, VALUE a) -{ - return dispatch1(var_field, a); -} #endif static NODE * @@ -15087,9 +14943,6 @@ new_bodystmt(struct parser_params *p, NODE *head, NODE *rescue, NODE *rescue_els result = NEW_RESCUE(head, rescue, rescue_else, &rescue_loc); nd_set_line(result, rescue->nd_loc.beg_pos.lineno); } - else if (rescue_else) { - result = block_append(p, result, rescue_else); - } if (ensure) { result = NEW_ENSURE(result, ensure, loc); } @@ -15768,7 +15621,7 @@ parser_initialize(struct parser_params *p) { /* note: we rely on TypedData_Make_Struct to set most fields to 0 */ p->command_start = TRUE; - p->ruby_sourcefile_string = Qnil; + p->ruby_sourcefile_string = NULL; p->lex.lpar_beg = -1; /* make lambda_beginning_p() == FALSE at first */ string_buffer_init(p); p->node_id = 0; @@ -15803,7 +15656,6 @@ rb_ruby_parser_mark(void *ptr) { struct parser_params *p = (struct parser_params*)ptr; - rb_gc_mark(p->ruby_sourcefile_string); #ifndef RIPPER rb_gc_mark(p->error_buffer); #else @@ -15834,6 +15686,10 @@ rb_ruby_parser_free(void *ptr) ruby_sized_xfree(p->tokenbuf, p->toksiz); } + if (p->ruby_sourcefile_string) { + rb_parser_string_free(p, p->ruby_sourcefile_string); + } + for (local = p->lvtbl; local; local = prev) { prev = local->prev; local_free(p, local); @@ -16016,7 +15872,7 @@ rb_ruby_parser_ripper_initialize(rb_parser_t *p, rb_parser_lex_gets_func *gets, p->lex.gets = gets; p->lex.input = input; p->eofp = 0; - p->ruby_sourcefile_string = sourcefile_string; + p->ruby_sourcefile_string = rb_str_to_parser_string(p, sourcefile_string); p->ruby_sourcefile = sourcefile; p->ruby_sourceline = sourceline; } @@ -16036,7 +15892,7 @@ rb_ruby_parser_enc(rb_parser_t *p) VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p) { - return p->ruby_sourcefile_string; + return rb_str_new_parser_string(p->ruby_sourcefile_string); } int @@ -16057,7 +15913,7 @@ rb_ruby_ripper_parse0(rb_parser_t *p) parser_prepare(p); p->ast = rb_ast_new(); ripper_yyparse((void*)p); - rb_ast_dispose(p->ast); + rb_ast_free(p->ast); p->ast = 0; p->eval_tree = 0; p->eval_tree_begin = 0; @@ -16173,7 +16029,7 @@ parser_compile_error(struct parser_params *p, const rb_code_location_t *loc, con va_start(ap, fmt); p->error_buffer = rb_syntax_error_append(p->error_buffer, - p->ruby_sourcefile_string, + p->ruby_sourcefile_string ? rb_str_new_parser_string(p->ruby_sourcefile_string) : Qnil, lineno, column, p->enc, fmt, ap); va_end(ap); @@ -16257,7 +16113,7 @@ rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr) #endif #ifdef RIPPER -#define validate(x) ((x) = (x) == rb_ripper_none ? Qnil : x) +#define validate(x) (void)(x) static VALUE ripper_dispatch0(struct parser_params *p, ID mid) diff --git a/prism/config.yml b/prism/config.yml index 56c95e4e55..35daacf614 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -7,13 +7,16 @@ errors: - ARGUMENT_BARE_HASH - ARGUMENT_BLOCK_FORWARDING - ARGUMENT_BLOCK_MULTI + - ARGUMENT_CONFLICT_AMPERSAND + - ARGUMENT_CONFLICT_STAR + - ARGUMENT_CONFLICT_STAR_STAR - ARGUMENT_FORMAL_CLASS - ARGUMENT_FORMAL_CONSTANT - ARGUMENT_FORMAL_GLOBAL - ARGUMENT_FORMAL_IVAR - ARGUMENT_FORWARDING_UNBOUND - ARGUMENT_IN - - ARGUMENT_NO_FORWARDING_AMP + - ARGUMENT_NO_FORWARDING_AMPERSAND - ARGUMENT_NO_FORWARDING_ELLIPSES - ARGUMENT_NO_FORWARDING_STAR - ARGUMENT_NO_FORWARDING_STAR_STAR @@ -95,6 +98,7 @@ errors: - EXPECT_EXPRESSION_AFTER_SPLAT_HASH - EXPECT_EXPRESSION_AFTER_STAR - EXPECT_IDENT_REQ_PARAMETER + - EXPECT_IN_DELIMITER - EXPECT_LPAREN_REQ_PARAMETER - EXPECT_MESSAGE - EXPECT_RBRACKET @@ -110,6 +114,7 @@ errors: - EXPRESSION_NOT_WRITABLE_FILE - EXPRESSION_NOT_WRITABLE_LINE - EXPRESSION_NOT_WRITABLE_NIL + - EXPRESSION_NOT_WRITABLE_NUMBERED - EXPRESSION_NOT_WRITABLE_SELF - EXPRESSION_NOT_WRITABLE_TRUE - FLOAT_PARSE @@ -123,16 +128,18 @@ errors: - HASH_ROCKET - HASH_TERM - HASH_VALUE + - HEREDOC_IDENTIFIER - HEREDOC_TERM - INCOMPLETE_QUESTION_MARK - INCOMPLETE_VARIABLE_CLASS - - INCOMPLETE_VARIABLE_CLASS_3_3_0 + - INCOMPLETE_VARIABLE_CLASS_3_3 - INCOMPLETE_VARIABLE_INSTANCE - - INCOMPLETE_VARIABLE_INSTANCE_3_3_0 + - INCOMPLETE_VARIABLE_INSTANCE_3_3 - INSTANCE_VARIABLE_BARE - INVALID_BLOCK_EXIT - INVALID_CHARACTER - INVALID_ENCODING_MAGIC_COMMENT + - INVALID_ESCAPE_CHARACTER - INVALID_FLOAT_EXPONENT - INVALID_LOCAL_VARIABLE_READ - INVALID_LOCAL_VARIABLE_WRITE @@ -141,16 +148,20 @@ errors: - INVALID_MULTIBYTE_ESCAPE - INVALID_NUMBER_BINARY - INVALID_NUMBER_DECIMAL + - INVALID_NUMBER_FRACTION - INVALID_NUMBER_HEXADECIMAL - INVALID_NUMBER_OCTAL - - INVALID_NUMBER_UNDERSCORE + - INVALID_NUMBER_UNDERSCORE_INNER + - INVALID_NUMBER_UNDERSCORE_TRAILING - INVALID_PERCENT + - INVALID_PERCENT_EOF - INVALID_PRINTABLE_CHARACTER - INVALID_RETRY_AFTER_ELSE - INVALID_RETRY_AFTER_ENSURE - INVALID_RETRY_WITHOUT_RESCUE + - INVALID_SYMBOL - INVALID_VARIABLE_GLOBAL - - INVALID_VARIABLE_GLOBAL_3_3_0 + - INVALID_VARIABLE_GLOBAL_3_3 - INVALID_YIELD - IT_NOT_ALLOWED_NUMBERED - IT_NOT_ALLOWED_ORDINARY @@ -175,15 +186,17 @@ errors: - NO_LOCAL_VARIABLE - NOT_EXPRESSION - NUMBER_LITERAL_UNDERSCORE + - NUMBERED_PARAMETER_INNER_BLOCK - NUMBERED_PARAMETER_IT - NUMBERED_PARAMETER_ORDINARY - - NUMBERED_PARAMETER_OUTER_SCOPE + - NUMBERED_PARAMETER_OUTER_BLOCK - OPERATOR_MULTI_ASSIGN - OPERATOR_WRITE_ARGUMENTS - OPERATOR_WRITE_BLOCK - PARAMETER_ASSOC_SPLAT_MULTI - PARAMETER_BLOCK_MULTI - PARAMETER_CIRCULAR + - PARAMETER_FORWARDING_AFTER_REST - PARAMETER_METHOD_NAME - PARAMETER_NAME_DUPLICATED - PARAMETER_NO_DEFAULT @@ -194,6 +207,7 @@ errors: - PARAMETER_STAR - PARAMETER_UNEXPECTED_FWD - PARAMETER_WILD_LOOSE_COMMA + - PARAMETER_UNEXPECTED_NO_KW - PATTERN_CAPTURE_DUPLICATE - PATTERN_EXPRESSION_AFTER_BRACKET - PATTERN_EXPRESSION_AFTER_COMMA @@ -205,8 +219,10 @@ errors: - PATTERN_EXPRESSION_AFTER_PIPE - PATTERN_EXPRESSION_AFTER_RANGE - PATTERN_EXPRESSION_AFTER_REST + - PATTERN_HASH_IMPLICIT - PATTERN_HASH_KEY - PATTERN_HASH_KEY_DUPLICATE + - PATTERN_HASH_KEY_INTERPOLATED - PATTERN_HASH_KEY_LABEL - PATTERN_HASH_KEY_LOCALS - PATTERN_IDENT_AFTER_HROCKET @@ -247,6 +263,9 @@ errors: - UNARY_RECEIVER - UNDEF_ARGUMENT - UNEXPECTED_BLOCK_ARGUMENT + - UNEXPECTED_INDEX_BLOCK + - UNEXPECTED_INDEX_KEYWORDS + - UNEXPECTED_SAFE_NAVIGATION - UNEXPECTED_TOKEN_CLOSE_CONTEXT - UNEXPECTED_TOKEN_IGNORE - UNTIL_TERM @@ -257,6 +276,7 @@ errors: - WRITE_TARGET_UNEXPECTED - XSTRING_TERM warnings: + - AMBIGUOUS_BINARY_OPERATOR - AMBIGUOUS_FIRST_ARGUMENT_MINUS - AMBIGUOUS_FIRST_ARGUMENT_PLUS - AMBIGUOUS_PREFIX_AMPERSAND @@ -266,7 +286,7 @@ warnings: - COMPARISON_AFTER_COMPARISON - DOT_DOT_DOT_EOL - EQUAL_IN_CONDITIONAL - - EQUAL_IN_CONDITIONAL_3_3_0 + - EQUAL_IN_CONDITIONAL_3_3 - END_IN_METHOD - DUPLICATED_HASH_KEY - DUPLICATED_WHEN_CLAUSE @@ -279,6 +299,7 @@ warnings: - KEYWORD_EOL - LITERAL_IN_CONDITION_DEFAULT - LITERAL_IN_CONDITION_VERBOSE + - SHAREABLE_CONSTANT_VALUE_LINE - SHEBANG_CARRIAGE_RETURN - UNEXPECTED_CARRIAGE_RETURN - UNREACHABLE_STATEMENT @@ -617,6 +638,8 @@ tokens: flags: - name: ArgumentsNodeFlags values: + - name: CONTAINS_KEYWORDS + comment: "if arguments contain keywords" - name: CONTAINS_KEYWORD_SPLAT comment: "if arguments contain keyword splat" comment: Flags for arguments nodes. @@ -1194,9 +1217,9 @@ nodes: type: constant - name: write_name type: constant - - name: operator + - name: binary_operator type: constant - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node @@ -1352,11 +1375,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a class variable using an operator that isn't `=`. @@ -1462,11 +1485,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a constant using an operator that isn't `=`. @@ -1517,23 +1540,9 @@ nodes: a.b::C ^^^ - - name: child - type: node - kind: - - ConstantReadNode - - MissingNode - comment: | - The right-hand node of the path. Always a `ConstantReadNode` in a - valid Ruby syntax tree. - - ::Foo - ^^^ - - self::Test - ^^^^ - - a.b::C - ^ + - name: name + type: constant? + comment: The name of the constant being accessed. This could be `nil` in the event of a syntax error. - name: delimiter_loc type: location comment: | @@ -1544,6 +1553,16 @@ nodes: One::Two ^^ + - name: name_loc + type: location + comment: | + The location of the name of the constant. + + ::Foo + ^^^ + + One::Two + ^^^ comment: | Represents accessing a constant through a path of `::` operators. @@ -1554,11 +1573,11 @@ nodes: - name: target type: node kind: ConstantPathNode - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a constant path using an operator that isn't `=`. @@ -1583,13 +1602,12 @@ nodes: fields: - name: parent type: node? - - name: child - type: node - kind: - - ConstantReadNode - - MissingNode + - name: name + type: constant? - name: delimiter_loc type: location + - name: name_loc + type: location comment: | Represents writing to a constant path in a context that doesn't have an explicit value. @@ -1926,11 +1944,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to a global variable using an operator that isn't `=`. @@ -2262,9 +2280,9 @@ nodes: type: location - name: block type: node? - - name: operator + - name: binary_operator type: constant - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node @@ -2350,11 +2368,11 @@ nodes: type: constant - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - - name: operator + - name: binary_operator type: constant comment: | Represents assigning to an instance variable using an operator that isn't `=`. @@ -2552,6 +2570,12 @@ nodes: `foo #{bar} baz` ^^^^^^^^^^^^^^^^ + - name: ItLocalVariableReadNode + comment: | + Represents reading from the implicit `it` local variable. + + -> { it } + ^^ - name: ItParametersNode comment: | Represents an implicit set of parameters through the use of the `it` keyword within a block or lambda. @@ -2630,13 +2654,13 @@ nodes: fields: - name: name_loc type: location - - name: operator_loc + - name: binary_operator_loc type: location - name: value type: node - name: name type: constant - - name: operator + - name: binary_operator type: constant - name: depth type: uint32 @@ -2677,10 +2701,6 @@ nodes: _1 # name `:_1` - Finally, for the default `it` block parameter, the name is `0it`. This is to distinguish it from an `it` local variable that is explicitly declared. - - it # name `:0it` - - name: depth type: uint32 comment: | @@ -3071,6 +3091,8 @@ nodes: # On parsing error of `f(**kwargs, ...)` or `f(**nil, ...)`, the keyword_rest value is moved here: - KeywordRestParameterNode - NoKeywordsParameterNode + # On parsing error of `f(..., ...)`, the first forwarding parameter is moved here: + - ForwardingParameterNode - name: keywords type: node[] kind: @@ -3211,8 +3233,21 @@ nodes: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: RationalNode fields: - - name: numeric - type: node + - name: flags + type: flags + kind: IntegerBaseFlags + - name: numerator + type: integer + comment: | + The numerator of the rational number. + + 1.5r # numerator 3 + - name: denominator + type: integer + comment: | + The denominator of the rational number. + + 1.5r # denominator 2 comment: | Represents a rational number literal. diff --git a/prism/extension.c b/prism/extension.c index 7b3f894478..f692ea616c 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -25,13 +25,14 @@ VALUE rb_cPrismParseLexResult; VALUE rb_cPrismDebugEncoding; -ID rb_option_id_command_line; -ID rb_option_id_encoding; -ID rb_option_id_filepath; -ID rb_option_id_frozen_string_literal; -ID rb_option_id_line; -ID rb_option_id_scopes; -ID rb_option_id_version; +ID rb_id_option_command_line; +ID rb_id_option_encoding; +ID rb_id_option_filepath; +ID rb_id_option_frozen_string_literal; +ID rb_id_option_line; +ID rb_id_option_scopes; +ID rb_id_option_version; +ID rb_id_source_for; /******************************************************************************/ /* IO of Ruby code */ @@ -51,7 +52,7 @@ check_string(VALUE value) { // Check if the value is a string. If it's not, then raise a type error. if (!RB_TYPE_P(value, T_STRING)) { - rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(value)); + rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(value)); } // Otherwise, return the value as a C string. @@ -65,7 +66,7 @@ static void input_load_string(pm_string_t *input, VALUE string) { // Check if the string is a string. If it's not, then raise a type error. if (!RB_TYPE_P(string, T_STRING)) { - rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(string)); + rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(string)); } pm_string_constant_init(input, RSTRING_PTR(string), RSTRING_LEN(string)); @@ -134,15 +135,15 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { pm_options_t *options = (pm_options_t *) argument; ID key_id = SYM2ID(key); - if (key_id == rb_option_id_filepath) { + if (key_id == rb_id_option_filepath) { if (!NIL_P(value)) pm_options_filepath_set(options, check_string(value)); - } else if (key_id == rb_option_id_encoding) { + } else if (key_id == rb_id_option_encoding) { if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value))); - } else if (key_id == rb_option_id_line) { + } else if (key_id == rb_id_option_line) { if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value)); - } else if (key_id == rb_option_id_frozen_string_literal) { + } else if (key_id == rb_id_option_frozen_string_literal) { if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value)); - } else if (key_id == rb_option_id_version) { + } else if (key_id == rb_id_option_version) { if (!NIL_P(value)) { const char *version = check_string(value); @@ -150,9 +151,9 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value); } } - } else if (key_id == rb_option_id_scopes) { + } else if (key_id == rb_id_option_scopes) { if (!NIL_P(value)) build_options_scopes(options, value); - } else if (key_id == rb_option_id_command_line) { + } else if (key_id == rb_id_option_command_line) { if (!NIL_P(value)) { const char *string = check_string(value); uint8_t command_line = 0; @@ -599,8 +600,7 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input)); VALUE offsets = rb_ary_new(); - VALUE source_argv[] = { source_string, LONG2NUM(parser.start_line), offsets }; - VALUE source = rb_class_new_instance(3, source_argv, rb_cPrismSource); + VALUE source = rb_funcall(rb_cPrismSource, rb_id_source_for, 3, source_string, LONG2NUM(parser.start_line), offsets); parse_lex_data_t parse_lex_data = { .source = source, @@ -762,6 +762,82 @@ parse(int argc, VALUE *argv, VALUE self) { } /** + * call-seq: + * Prism::parse_file(filepath, **options) -> ParseResult + * + * Parse the given file and return a ParseResult instance. For supported + * options, see Prism::parse. + */ +static VALUE +parse_file(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + file_options(argc, argv, &input, &options); + + VALUE value = parse_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return value; +} + +/** + * Parse the given input and return nothing. + */ +static void +profile_input(pm_string_t *input, const pm_options_t *options) { + pm_parser_t parser; + pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options); + + pm_node_t *node = pm_parse(&parser); + pm_node_destroy(&parser, node); + pm_parser_free(&parser); +} + +/** + * call-seq: + * Prism::profile(source, **options) -> nil + * + * Parse the given string and return nothing. This method is meant to allow + * profilers to avoid the overhead of reifying the AST to Ruby. For supported + * options, see Prism::parse. + */ +static VALUE +profile(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + string_options(argc, argv, &input, &options); + profile_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return Qnil; +} + +/** + * call-seq: + * Prism::profile_file(filepath, **options) -> nil + * + * Parse the given file and return nothing. This method is meant to allow + * profilers to avoid the overhead of reifying the AST to Ruby. For supported + * options, see Prism::parse. + */ +static VALUE +profile_file(int argc, VALUE *argv, VALUE self) { + pm_string_t input; + pm_options_t options = { 0 }; + + file_options(argc, argv, &input, &options); + profile_input(&input, &options); + pm_string_free(&input); + pm_options_free(&options); + + return Qnil; +} + +/** * An implementation of fgets that is suitable for use with Ruby IO objects. */ static char * @@ -816,27 +892,6 @@ parse_stream(int argc, VALUE *argv, VALUE self) { } /** - * call-seq: - * Prism::parse_file(filepath, **options) -> ParseResult - * - * Parse the given file and return a ParseResult instance. For supported - * options, see Prism::parse. - */ -static VALUE -parse_file(int argc, VALUE *argv, VALUE self) { - pm_string_t input; - pm_options_t options = { 0 }; - - file_options(argc, argv, &input, &options); - - VALUE value = parse_input(&input, &options); - pm_string_free(&input); - pm_options_free(&options); - - return value; -} - -/** * Parse the given input and return an array of Comment objects. */ static VALUE @@ -1035,303 +1090,6 @@ parse_file_failure_p(int argc, VALUE *argv, VALUE self) { } /******************************************************************************/ -/* Utility functions exposed to make testing easier */ -/******************************************************************************/ - -/** - * call-seq: - * Debug::named_captures(source) -> Array - * - * Returns an array of strings corresponding to the named capture groups in the - * given source string. If prism was unable to parse the regular expression, - * this function returns nil. - */ -static VALUE -named_captures(VALUE self, VALUE source) { - pm_string_list_t string_list = { 0 }; - - if (!pm_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, PM_ENCODING_UTF_8_ENTRY)) { - pm_string_list_free(&string_list); - return Qnil; - } - - VALUE names = rb_ary_new(); - for (size_t index = 0; index < string_list.length; index++) { - const pm_string_t *string = &string_list.strings[index]; - rb_ary_push(names, rb_str_new((const char *) pm_string_source(string), pm_string_length(string))); - } - - pm_string_list_free(&string_list); - return names; -} - -/** - * call-seq: - * Debug::integer_parse(source) -> [Integer, String] - * - * Parses the given source string and returns the integer it represents, as well - * as a decimal string representation. - */ -static VALUE -integer_parse(VALUE self, VALUE source) { - const uint8_t *start = (const uint8_t *) RSTRING_PTR(source); - size_t length = RSTRING_LEN(source); - - pm_integer_t integer = { 0 }; - pm_integer_parse(&integer, PM_INTEGER_BASE_UNKNOWN, start, start + length); - - pm_buffer_t buffer = { 0 }; - pm_integer_string(&buffer, &integer); - - VALUE string = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer)); - pm_buffer_free(&buffer); - - VALUE result = rb_ary_new_capa(2); - rb_ary_push(result, pm_integer_new(&integer)); - rb_ary_push(result, string); - pm_integer_free(&integer); - - return result; -} - -/** - * call-seq: - * Debug::memsize(source) -> { length: xx, memsize: xx, node_count: xx } - * - * Return a hash of information about the given source string's memory usage. - */ -static VALUE -memsize(VALUE self, VALUE string) { - pm_parser_t parser; - size_t length = RSTRING_LEN(string); - pm_parser_init(&parser, (const uint8_t *) RSTRING_PTR(string), length, NULL); - - pm_node_t *node = pm_parse(&parser); - pm_memsize_t memsize; - pm_node_memsize(node, &memsize); - - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - - VALUE result = rb_hash_new(); - rb_hash_aset(result, ID2SYM(rb_intern("length")), INT2FIX(length)); - rb_hash_aset(result, ID2SYM(rb_intern("memsize")), INT2FIX(memsize.memsize)); - rb_hash_aset(result, ID2SYM(rb_intern("node_count")), INT2FIX(memsize.node_count)); - return result; -} - -/** - * call-seq: - * Debug::profile_file(filepath) -> nil - * - * Parse the file, but do nothing with the result. This is used to profile the - * parser for memory and speed. - */ -static VALUE -profile_file(VALUE self, VALUE filepath) { - pm_string_t input; - - const char *checked = check_string(filepath); - Check_Type(filepath, T_STRING); - - if (!pm_string_mapped_init(&input, checked)) { -#ifdef _WIN32 - int e = rb_w32_map_errno(GetLastError()); -#else - int e = errno; -#endif - - rb_syserr_fail(e, checked); - } - - pm_options_t options = { 0 }; - pm_options_filepath_set(&options, checked); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options); - - pm_node_t *node = pm_parse(&parser); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_options_free(&options); - pm_string_free(&input); - - return Qnil; -} - -#ifndef PRISM_EXCLUDE_PRETTYPRINT - -/** - * call-seq: - * Debug::inspect_node(source) -> inspected - * - * Inspect the AST that represents the given source using the prism pretty print - * as opposed to the Ruby implementation. - */ -static VALUE -inspect_node(VALUE self, VALUE source) { - pm_string_t input; - input_load_string(&input, source); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL); - - pm_node_t *node = pm_parse(&parser); - pm_buffer_t buffer = { 0 }; - - pm_prettyprint(&buffer, &parser, node); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE string = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - - return string; -} - -#endif - -/** - * call-seq: - * Debug::format_errors(source, colorize) -> String - * - * Format the errors that are found when parsing the given source string. - */ -static VALUE -format_errors(VALUE self, VALUE source, VALUE colorize) { - pm_string_t input; - input_load_string(&input, source); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL); - - pm_node_t *node = pm_parse(&parser); - pm_buffer_t buffer = { 0 }; - - pm_parser_errors_format(&parser, &parser.error_list, &buffer, RTEST(colorize), true); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_string_free(&input); - - return result; -} - -/** - * call-seq: - * Debug::static_inspect(source) -> String - * - * Inspect the node as it would be inspected by the warnings used in static - * literal sets. - */ -static VALUE -static_inspect(int argc, VALUE *argv, VALUE self) { - pm_string_t input; - pm_options_t options = { 0 }; - string_options(argc, argv, &input, &options); - - pm_parser_t parser; - pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options); - - pm_node_t *program = pm_parse(&parser); - pm_node_t *node = ((pm_program_node_t *) program)->statements->body.nodes[0]; - - pm_buffer_t buffer = { 0 }; - pm_static_literal_inspect(&buffer, &parser.newline_list, parser.start_line, parser.encoding->name, node); - - rb_encoding *encoding = rb_enc_find(parser.encoding->name); - VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); - - pm_buffer_free(&buffer); - pm_node_destroy(&parser, program); - pm_parser_free(&parser); - pm_string_free(&input); - pm_options_free(&options); - - return result; -} - -/** - * call-seq: Debug::Encoding.all -> Array[Debug::Encoding] - * - * Return an array of all of the encodings that prism knows about. - */ -static VALUE -encoding_all(VALUE self) { - VALUE encodings = rb_ary_new(); - - for (size_t index = 0; index < PM_ENCODING_MAXIMUM; index++) { - const pm_encoding_t *encoding = &pm_encodings[index]; - - VALUE encoding_argv[] = { rb_str_new_cstr(encoding->name), encoding->multibyte ? Qtrue : Qfalse }; - rb_ary_push(encodings, rb_class_new_instance(2, encoding_argv, rb_cPrismDebugEncoding)); - } - - return encodings; -} - -static const pm_encoding_t * -encoding_find(VALUE name) { - const uint8_t *source = (const uint8_t *) RSTRING_PTR(name); - size_t length = RSTRING_LEN(name); - - const pm_encoding_t *encoding = pm_encoding_find(source, source + length); - if (encoding == NULL) { rb_raise(rb_eArgError, "Unknown encoding: %s", source); } - - return encoding; -} - -/** - * call-seq: Debug::Encoding.width(source) -> Integer - * - * Returns the width of the first character in the given string if it is valid - * in the encoding. If it is not, this function returns 0. - */ -static VALUE -encoding_char_width(VALUE self, VALUE name, VALUE value) { - return ULONG2NUM(encoding_find(name)->char_width((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value))); -} - -/** - * call-seq: Debug::Encoding.alnum?(source) -> true | false - * - * Returns true if the first character in the given string is an alphanumeric - * character in the encoding. - */ -static VALUE -encoding_alnum_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->alnum_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse; -} - -/** - * call-seq: Debug::Encoding.alpha?(source) -> true | false - * - * Returns true if the first character in the given string is an alphabetic - * character in the encoding. - */ -static VALUE -encoding_alpha_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->alpha_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse; -} - -/** - * call-seq: Debug::Encoding.upper?(source) -> true | false - * - * Returns true if the first character in the given string is an uppercase - * character in the encoding. - */ -static VALUE -encoding_isupper_char(VALUE self, VALUE name, VALUE value) { - return encoding_find(name)->isupper_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) ? Qtrue : Qfalse; -} - -/******************************************************************************/ /* Initialization of the extension */ /******************************************************************************/ @@ -1364,20 +1122,20 @@ Init_prism(void) { rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject); rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject); rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject); - rb_cPrismResult = rb_define_class_under(rb_cPrism, "Result", rb_cObject); rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cPrismResult); rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult); - // Intern all of the options that we support so that we don't have to do it - // every time we parse. - rb_option_id_command_line = rb_intern_const("command_line"); - rb_option_id_encoding = rb_intern_const("encoding"); - rb_option_id_filepath = rb_intern_const("filepath"); - rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal"); - rb_option_id_line = rb_intern_const("line"); - rb_option_id_scopes = rb_intern_const("scopes"); - rb_option_id_version = rb_intern_const("version"); + // Intern all of the IDs eagerly that we support so that we don't have to do + // it every time we parse. + rb_id_option_command_line = rb_intern_const("command_line"); + rb_id_option_encoding = rb_intern_const("encoding"); + rb_id_option_filepath = rb_intern_const("filepath"); + rb_id_option_frozen_string_literal = rb_intern_const("frozen_string_literal"); + rb_id_option_line = rb_intern_const("line"); + rb_id_option_scopes = rb_intern_const("scopes"); + rb_id_option_version = rb_intern_const("version"); + rb_id_source_for = rb_intern("for"); /** * The version of the prism library. @@ -1388,8 +1146,10 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "lex", lex, -1); rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1); rb_define_singleton_method(rb_cPrism, "parse", parse, -1); - rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1); rb_define_singleton_method(rb_cPrism, "parse_file", parse_file, -1); + rb_define_singleton_method(rb_cPrism, "profile", profile, -1); + rb_define_singleton_method(rb_cPrism, "profile_file", profile_file, -1); + rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1); rb_define_singleton_method(rb_cPrism, "parse_comments", parse_comments, -1); rb_define_singleton_method(rb_cPrism, "parse_file_comments", parse_file_comments, -1); rb_define_singleton_method(rb_cPrism, "parse_lex", parse_lex, -1); @@ -1404,29 +1164,6 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); #endif - // Next, the functions that will be called by the parser to perform various - // internal tasks. We expose these to make them easier to test. - VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug"); - rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1); - rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1); - rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1); - rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1); - rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2); - rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1); - -#ifndef PRISM_EXCLUDE_PRETTYPRINT - rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); -#endif - - // Next, define the functions that are exposed through the private - // Debug::Encoding class. - rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject); - rb_define_singleton_method(rb_cPrismDebugEncoding, "all", encoding_all, 0); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_width", encoding_char_width, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_alnum?", encoding_alnum_char, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_alpha?", encoding_alpha_char, 2); - rb_define_singleton_method(rb_cPrismDebugEncoding, "_upper?", encoding_isupper_char, 2); - // Next, initialize the other APIs. Init_prism_api_node(); Init_prism_pack(); diff --git a/prism/extension.h b/prism/extension.h index 9134959783..b15a2c5e61 100644 --- a/prism/extension.h +++ b/prism/extension.h @@ -1,7 +1,7 @@ #ifndef PRISM_EXT_NODE_H #define PRISM_EXT_NODE_H -#define EXPECTED_PRISM_VERSION "0.27.0" +#define EXPECTED_PRISM_VERSION "0.29.0" #include <ruby.h> #include <ruby/encoding.h> diff --git a/prism/node.h b/prism/node.h index 8736e59a94..e8686a327c 100644 --- a/prism/node.h +++ b/prism/node.h @@ -57,27 +57,6 @@ void pm_node_list_free(pm_node_list_t *list); PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node); /** - * This struct stores the information gathered by the pm_node_memsize function. - * It contains both the memory footprint and additionally metadata about the - * shape of the tree. - */ -typedef struct { - /** The total memory footprint of the node and all of its children. */ - size_t memsize; - - /** The number of children the node has. */ - size_t node_count; -} pm_memsize_t; - -/** - * Calculates the memory footprint of a given node. - * - * @param node The node to calculate the memory footprint of. - * @param memsize The memory footprint of the node and all of its children. - */ -PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize); - -/** * Returns a string representation of the given node type. * * @param node_type The node type to convert to a string. diff --git a/prism/options.c b/prism/options.c index 4d0d6dbc49..664db4f061 100644 --- a/prism/options.c +++ b/prism/options.c @@ -58,8 +58,8 @@ pm_options_version_set(pm_options_t *options, const char *version, size_t length case 5: assert(version != NULL); - if (strncmp(version, "3.3.0", length) == 0) { - options->version = PM_OPTIONS_VERSION_CRUBY_3_3_0; + if ((strncmp(version, "3.3.0", length) == 0) || (strncmp(version, "3.3.1", length) == 0)) { + options->version = PM_OPTIONS_VERSION_CRUBY_3_3; return true; } diff --git a/prism/options.h b/prism/options.h index d0b46a0864..a623ae0b83 100644 --- a/prism/options.h +++ b/prism/options.h @@ -49,8 +49,8 @@ typedef enum { /** The current version of prism. */ PM_OPTIONS_VERSION_LATEST = 0, - /** The vendored version of prism in CRuby 3.3.0. */ - PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1 + /** The vendored version of prism in CRuby 3.3.x. */ + PM_OPTIONS_VERSION_CRUBY_3_3 = 1 } pm_options_version_t; /** diff --git a/prism/parser.h b/prism/parser.h index 8054e332f7..2bae4b0ac3 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -546,6 +546,17 @@ typedef struct pm_locals { pm_local_t *locals; } pm_locals_t; +/** The flags about scope parameters that can be set. */ +typedef uint8_t pm_scope_parameters_t; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20; +static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40; + /** * This struct represents a node in a linked list of scopes. Some scopes can see * into their parent scopes, while others cannot. @@ -558,9 +569,18 @@ typedef struct pm_scope { pm_locals_t locals; /** + * This is a list of the implicit parameters contained within the block. + * These will be processed after the block is parsed to determine the kind + * of parameters node that should be used and to check if any errors need to + * be added. + */ + pm_node_list_t implicit_parameters; + + /** * This is a bitfield that indicates the parameters that are being used in - * this scope. It is a combination of the PM_SCOPE_PARAMS_* constants. There - * are three different kinds of parameters that can be used in a scope: + * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants. + * There are three different kinds of parameters that can be used in a + * scope: * * - Ordinary parameters (e.g., def foo(bar); end) * - Numbered parameters (e.g., def foo; _1; end) @@ -575,15 +595,7 @@ typedef struct pm_scope { * - def foo(&); end * - def foo(...); end */ - uint8_t parameters; - - /** - * An integer indicating the number of numbered parameters on this scope. - * This is necessary to determine if child blocks are allowed to use - * numbered parameters, and to pass information to consumers of the AST - * about how many numbered parameters exist. - */ - int8_t numbered_parameters; + pm_scope_parameters_t parameters; /** * The current state of constant shareability for this scope. This is @@ -598,20 +610,6 @@ typedef struct pm_scope { bool closed; } pm_scope_t; -static const uint8_t PM_SCOPE_PARAMETERS_NONE = 0x0; -static const uint8_t PM_SCOPE_PARAMETERS_ORDINARY = 0x1; -static const uint8_t PM_SCOPE_PARAMETERS_NUMBERED = 0x2; -static const uint8_t PM_SCOPE_PARAMETERS_IT = 0x4; -static const uint8_t PM_SCOPE_PARAMETERS_TYPE_MASK = 0x7; - -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x8; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x10; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x20; -static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x40; - -static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED = -1; -static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_NONE = 0; - /** * A struct that represents a stack of boolean values. */ diff --git a/prism/prism.c b/prism/prism.c index 2815723ebd..c30d693d19 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -423,7 +423,7 @@ lex_mode_pop(pm_parser_t *parser) { * This is the equivalent of IS_lex_state is CRuby. */ static inline bool -lex_state_p(pm_parser_t *parser, pm_lex_state_t state) { +lex_state_p(const pm_parser_t *parser, pm_lex_state_t state) { return parser->lex_state & state; } @@ -672,6 +672,26 @@ pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id #define PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, ...) \ PM_PARSER_WARN_FORMAT(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) +/** + * Add an error for an expected heredoc terminator. This is a special function + * only because it grabs its location off of a lex mode instead of a node or a + * token. + */ +static void +pm_parser_err_heredoc_term(pm_parser_t *parser, pm_lex_mode_t *lex_mode) { + const uint8_t *ident_start = lex_mode->as.heredoc.ident_start; + size_t ident_length = lex_mode->as.heredoc.ident_length; + + PM_PARSER_ERR_FORMAT( + parser, + ident_start, + ident_start + ident_length, + PM_ERR_HEREDOC_TERM, + (int) ident_length, + (const char *) ident_start + ); +} + /******************************************************************************/ /* Scope-related functions */ /******************************************************************************/ @@ -688,7 +708,7 @@ pm_parser_scope_push(pm_parser_t *parser, bool closed) { .previous = parser->current_scope, .locals = { 0 }, .parameters = PM_SCOPE_PARAMETERS_NONE, - .numbered_parameters = PM_SCOPE_NUMBERED_PARAMETERS_NONE, + .implicit_parameters = { 0 }, .shareable_constant = (closed || parser->current_scope == NULL) ? PM_SCOPE_SHAREABLE_CONSTANT_NONE : parser->current_scope->shareable_constant, .closed = closed }; @@ -729,42 +749,97 @@ pm_parser_scope_find(pm_parser_t *parser, uint32_t depth) { return scope; } -static void -pm_parser_scope_forwarding_param_check(pm_parser_t *parser, const pm_token_t * token, const uint8_t mask, pm_diagnostic_id_t diag) { +typedef enum { + PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS, + PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT, + PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL +} pm_scope_forwarding_param_check_result_t; + +static pm_scope_forwarding_param_check_result_t +pm_parser_scope_forwarding_param_check(pm_parser_t *parser, const uint8_t mask) { pm_scope_t *scope = parser->current_scope; - while (scope) { + bool conflict = false; + + while (scope != NULL) { if (scope->parameters & mask) { - if (!scope->closed) { - pm_parser_err_token(parser, token, diag); - return; + if (scope->closed) { + if (conflict) { + return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT; + } else { + return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS; + } } - return; + + conflict = true; } + if (scope->closed) break; scope = scope->previous; } - pm_parser_err_token(parser, token, diag); + return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL; } -static inline void +static void pm_parser_scope_forwarding_block_check(pm_parser_t *parser, const pm_token_t * token) { - pm_parser_scope_forwarding_param_check(parser, token, PM_SCOPE_PARAMETERS_FORWARDING_BLOCK, PM_ERR_ARGUMENT_NO_FORWARDING_AMP); + switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_BLOCK)) { + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS: + // Pass. + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_AMPERSAND); + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND); + break; + } } -static inline void +static void pm_parser_scope_forwarding_positionals_check(pm_parser_t *parser, const pm_token_t * token) { - pm_parser_scope_forwarding_param_check(parser, token, PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS, PM_ERR_ARGUMENT_NO_FORWARDING_STAR); + switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS)) { + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS: + // Pass. + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_STAR); + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_STAR); + break; + } } -static inline void -pm_parser_scope_forwarding_all_check(pm_parser_t *parser, const pm_token_t * token) { - pm_parser_scope_forwarding_param_check(parser, token, PM_SCOPE_PARAMETERS_FORWARDING_ALL, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES); +static void +pm_parser_scope_forwarding_all_check(pm_parser_t *parser, const pm_token_t *token) { + switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_ALL)) { + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS: + // Pass. + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT: + // This shouldn't happen, because ... is not allowed in the + // declaration of blocks. If we get here, we assume we already have + // an error for this. + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES); + break; + } } -static inline void +static void pm_parser_scope_forwarding_keywords_check(pm_parser_t *parser, const pm_token_t * token) { - pm_parser_scope_forwarding_param_check(parser, token, PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS, PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR); + switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS)) { + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS: + // Pass. + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_STAR_STAR); + break; + case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL: + pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR); + break; + } } /** @@ -1108,6 +1183,31 @@ pm_check_value_expression(pm_node_t *node) { return NULL; case PM_BEGIN_NODE: { pm_begin_node_t *cast = (pm_begin_node_t *) node; + + if (cast->statements == NULL && cast->ensure_clause != NULL) { + node = (pm_node_t *) cast->ensure_clause; + } + else { + if (cast->rescue_clause != NULL) { + if (cast->rescue_clause->statements == NULL) { + return NULL; + } + else if (cast->else_clause != NULL) { + node = (pm_node_t *) cast->else_clause; + } + else { + node = (pm_node_t *) cast->statements; + } + } + else { + node = (pm_node_t *) cast->statements; + } + } + + break; + } + case PM_ENSURE_NODE: { + pm_ensure_node_t *cast = (pm_ensure_node_t *) node; node = (pm_node_t *) cast->statements; break; } @@ -1405,7 +1505,7 @@ pm_conditional_predicate_warn_write_literal_p(const pm_node_t *node) { static inline void pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, const pm_node_t *node) { if (pm_conditional_predicate_warn_write_literal_p(node)) { - pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0 : PM_WARN_EQUAL_IN_CONDITIONAL); + pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3 : PM_WARN_EQUAL_IN_CONDITIONAL); } } @@ -1555,7 +1655,7 @@ not_provided(pm_parser_t *parser) { return (pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }; } -#define PM_LOCATION_NULL_VALUE(parser) ((pm_location_t) { .start = parser->start, .end = parser->start }) +#define PM_LOCATION_NULL_VALUE(parser) ((pm_location_t) { .start = (parser)->start, .end = (parser)->start }) #define PM_LOCATION_TOKEN_VALUE(token) ((pm_location_t) { .start = (token)->start, .end = (token)->end }) #define PM_LOCATION_NODE_VALUE(node) ((pm_location_t) { .start = (node)->location.start, .end = (node)->location.end }) #define PM_LOCATION_NODE_BASE_VALUE(node) ((pm_location_t) { .start = (node)->base.location.start, .end = (node)->base.location.end }) @@ -1683,7 +1783,7 @@ char_is_identifier_utf8(const uint8_t *b, const uint8_t *end) { * it's important that it be as fast as possible. */ static inline size_t -char_is_identifier(pm_parser_t *parser, const uint8_t *b) { +char_is_identifier(const pm_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { size_t width; if ((width = parser->encoding->alnum_char(b, parser->end - b)) != 0) { @@ -2752,8 +2852,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location.start = parser->start; - node->base.location.end = parser->start; + node->base.location = PM_LOCATION_NULL_VALUE(parser); node->arguments = arguments; node->name = name; @@ -2924,6 +3023,29 @@ pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const } /** + * Validate that index expressions do not have keywords or blocks if we are + * parsing as Ruby 3.4+. + */ +static void +pm_index_arguments_check(pm_parser_t *parser, const pm_arguments_node_t *arguments, const pm_node_t *block) { + if (parser->version != PM_OPTIONS_VERSION_CRUBY_3_3) { + if (arguments != NULL && PM_NODE_FLAG_P(arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS)) { + pm_node_t *node; + PM_NODE_LIST_FOREACH(&arguments->arguments, index, node) { + if (PM_NODE_TYPE_P(node, PM_KEYWORD_HASH_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_INDEX_KEYWORDS); + break; + } + } + } + + if (block != NULL) { + pm_parser_err_node(parser, block, PM_ERR_UNEXPECTED_INDEX_BLOCK); + } + } +} + +/** * Allocate and initialize a new IndexAndWriteNode node. */ static pm_index_and_write_node_t * @@ -2931,6 +3053,8 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons assert(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL); pm_index_and_write_node_t *node = PM_ALLOC_NODE(parser, pm_index_and_write_node_t); + pm_index_arguments_check(parser, target->arguments, target->block); + *node = (pm_index_and_write_node_t) { { .type = PM_INDEX_AND_WRITE_NODE, @@ -2980,8 +3104,8 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3002,6 +3126,8 @@ static pm_index_operator_write_node_t * pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) { pm_index_operator_write_node_t *node = PM_ALLOC_NODE(parser, pm_index_operator_write_node_t); + pm_index_arguments_check(parser, target->arguments, target->block); + *node = (pm_index_operator_write_node_t) { { .type = PM_INDEX_OPERATOR_WRITE_NODE, @@ -3017,8 +3143,8 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = target->block, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3075,6 +3201,8 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const assert(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL); pm_index_or_write_node_t *node = PM_ALLOC_NODE(parser, pm_index_or_write_node_t); + pm_index_arguments_check(parser, target->arguments, target->block); + *node = (pm_index_or_write_node_t) { { .type = PM_INDEX_OR_WRITE_NODE, @@ -3139,6 +3267,8 @@ pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { pm_index_target_node_t *node = PM_ALLOC_NODE(parser, pm_index_target_node_t); pm_node_flags_t flags = target->base.flags; + pm_index_arguments_check(parser, target->arguments, target->block); + *node = (pm_index_target_node_t) { { .type = PM_INDEX_TARGET_NODE, @@ -3358,9 +3488,9 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -3474,9 +3604,9 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat } }, .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -3510,22 +3640,27 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node * Allocate and initialize a new ConstantPathNode node. */ static pm_constant_path_node_t * -pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_token_t *delimiter, pm_node_t *child) { +pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_token_t *delimiter, const pm_token_t *name_token) { pm_assert_value_expression(parser, parent); - pm_constant_path_node_t *node = PM_ALLOC_NODE(parser, pm_constant_path_node_t); + pm_constant_id_t name = PM_CONSTANT_ID_UNSET; + if (name_token->type == PM_TOKEN_CONSTANT) { + name = pm_parser_constant_id_token(parser, name_token); + } + *node = (pm_constant_path_node_t) { { .type = PM_CONSTANT_PATH_NODE, .location = { .start = parent == NULL ? delimiter->start : parent->location.start, - .end = child->location.end + .end = name_token->end }, }, .parent = parent, - .child = child, - .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter) + .name = name, + .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter), + .name_loc = PM_LOCATION_TOKEN_VALUE(name_token) }; return node; @@ -3596,9 +3731,9 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -4180,7 +4315,7 @@ pm_float_node_imaginary_create(pm_parser_t *parser, const pm_token_t *token) { } /** - * Allocate and initialize a new FloatNode node from a FLOAT_RATIONAL token. + * Allocate and initialize a new RationalNode node from a FLOAT_RATIONAL token. */ static pm_rational_node_t * pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) { @@ -4190,16 +4325,44 @@ pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) { *node = (pm_rational_node_t) { { .type = PM_RATIONAL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, + .flags = PM_INTEGER_BASE_FLAGS_DECIMAL | PM_NODE_FLAG_STATIC_LITERAL, .location = PM_LOCATION_TOKEN_VALUE(token) }, - .numeric = (pm_node_t *) pm_float_node_create(parser, &((pm_token_t) { - .type = PM_TOKEN_FLOAT, - .start = token->start, - .end = token->end - 1 - })) + .numerator = { 0 }, + .denominator = { 0 } }; + const uint8_t *start = token->start; + const uint8_t *end = token->end - 1; // r + + while (start < end && *start == '0') start++; // 0.1 -> .1 + while (end > start && end[-1] == '0') end--; // 1.0 -> 1. + + size_t length = (size_t) (end - start); + if (length == 1) { + node->denominator.value = 1; + return node; + } + + const uint8_t *point = memchr(start, '.', length); + assert(point && "should have a decimal point"); + + uint8_t *digits = malloc(length); + if (digits == NULL) { + fputs("[pm_float_node_rational_create] Failed to allocate memory", stderr); + abort(); + } + + memcpy(digits, start, (unsigned long) (point - start)); + memcpy(digits + (point - start), point + 1, (unsigned long) (end - point - 1)); + pm_integer_parse(&node->numerator, PM_INTEGER_BASE_DEFAULT, digits, digits + length - 1); + + digits[0] = '1'; + if (end - point > 1) memset(digits + 1, '0', (size_t) (end - point - 1)); + pm_integer_parse(&node->denominator, PM_INTEGER_BASE_DEFAULT, digits, digits + (end - point)); + free(digits); + + pm_integers_reduce(&node->numerator, &node->denominator); return node; } @@ -4449,9 +4612,9 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta }, .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -4510,7 +4673,7 @@ pm_global_variable_read_node_synthesized_create(pm_parser_t *parser, pm_constant *node = (pm_global_variable_read_node_t) { { .type = PM_GLOBAL_VARIABLE_READ_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, .name = name }; @@ -4552,11 +4715,11 @@ pm_global_variable_write_node_synthesized_create(pm_parser_t *parser, pm_constan *node = (pm_global_variable_write_node_t) { { .type = PM_GLOBAL_VARIABLE_WRITE_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, .name = name, - .name_loc = { .start = parser->start, .end = parser->start }, - .operator_loc = { .start = parser->start, .end = parser->start }, + .name_loc = PM_LOCATION_NULL_VALUE(parser), + .operator_loc = PM_LOCATION_NULL_VALUE(parser), .value = value }; @@ -4833,7 +4996,7 @@ pm_integer_node_imaginary_create(pm_parser_t *parser, pm_node_flags_t base, cons } /** - * Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL + * Allocate and initialize a new RationalNode node from an INTEGER_RATIONAL * token. */ static pm_rational_node_t * @@ -4844,16 +5007,24 @@ pm_integer_node_rational_create(pm_parser_t *parser, pm_node_flags_t base, const *node = (pm_rational_node_t) { { .type = PM_RATIONAL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, + .flags = base | PM_NODE_FLAG_STATIC_LITERAL, .location = PM_LOCATION_TOKEN_VALUE(token) }, - .numeric = (pm_node_t *) pm_integer_node_create(parser, base, &((pm_token_t) { - .type = PM_TOKEN_INTEGER, - .start = token->start, - .end = token->end - 1 - })) + .numerator = { 0 }, + .denominator = { .value = 1, 0 } }; + pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL; + switch (base) { + case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break; + case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break; + case PM_INTEGER_BASE_FLAGS_DECIMAL: break; + case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break; + default: assert(false && "unreachable"); break; + } + + pm_integer_parse(&node->numerator, integer_base, token->start, token->end - 1); + return node; } @@ -4957,9 +5128,9 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance }, .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -5034,6 +5205,50 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable } /** + * Append a part into a list of string parts. Importantly this handles nested + * interpolated strings by not necessarily removing the marker for static + * literals. + */ +static void +pm_interpolated_node_append(pm_node_t *node, pm_node_list_t *parts, pm_node_t *part) { + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + break; + case PM_EMBEDDED_STATEMENTS_NODE: { + pm_embedded_statements_node_t *cast = (pm_embedded_statements_node_t *) part; + pm_node_t *embedded = (cast->statements != NULL && cast->statements->body.size == 1) ? cast->statements->body.nodes[0] : NULL; + + if (embedded == NULL) { + // If there are no statements or more than one statement, then + // we lose the static literal flag. + pm_node_flag_unset(node, PM_NODE_FLAG_STATIC_LITERAL); + } else if (PM_NODE_TYPE_P(embedded, PM_STRING_NODE)) { + // If the embedded statement is a string, then we can keep the + // static literal flag and mark the string as frozen. + pm_node_flag_set(embedded, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } else if (PM_NODE_TYPE_P(embedded, PM_INTERPOLATED_STRING_NODE) && PM_NODE_FLAG_P(embedded, PM_NODE_FLAG_STATIC_LITERAL)) { + // If the embedded statement is an interpolated string and it's + // a static literal, then we can keep the static literal flag. + } else { + // Otherwise we lose the static literal flag. + pm_node_flag_unset(node, PM_NODE_FLAG_STATIC_LITERAL); + } + + break; + } + case PM_EMBEDDED_VARIABLE_NODE: + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + break; + default: + assert(false && "unexpected node type"); + break; + } + + pm_node_list_append(parts, part); +} + +/** * Allocate a new InterpolatedRegularExpressionNode node. */ static pm_interpolated_regular_expression_node_t * @@ -5066,54 +5281,113 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio node->base.location.end = part->location.end; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - - pm_node_list_append(&node->parts, part); + pm_interpolated_node_append((pm_node_t *) node, &node->parts, part); } static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); node->base.location.end = closing->end; - pm_node_flag_set((pm_node_t *)node, pm_regular_expression_flags_create(parser, closing)); + pm_node_flag_set((pm_node_t *) node, pm_regular_expression_flags_create(parser, closing)); } /** * Append a part to an InterpolatedStringNode node. + * + * This has some somewhat complicated semantics, because we need to update + * multiple flags that have somewhat confusing interactions. + * + * PM_NODE_FLAG_STATIC_LITERAL indicates that the node should be treated as a + * single static literal string that can be pushed onto the stack on its own. + * Note that this doesn't necessarily mean that the string will be frozen or + * not; the instructions in CRuby will be either putobject or putstring, + * depending on the combination of `--enable-frozen-string-literal`, + * `# frozen_string_literal: true`, and whether or not there is interpolation. + * + * PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN indicates that the string should be + * explicitly frozen. This will only happen if the string is comprised entirely + * of string parts that are themselves static literals and frozen. + * + * PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE indicates that the string should + * be explicitly marked as mutable. This will happen from + * `--disable-frozen-string-literal` or `# frozen_string_literal: false`. This + * is necessary to indicate that the string should be left up to the runtime, + * which could potentially use a chilled string otherwise. */ static inline void -pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_node_t *node, pm_node_t *part) { +pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_t *part) { +#define CLEAR_FLAGS(node) \ + node->base.flags = (pm_node_flags_t) (node->base.flags & ~(PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE)) + +#define MUTABLE_FLAGS(node) \ + node->base.flags = (pm_node_flags_t) ((node->base.flags | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE) & ~PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); + if (node->parts.size == 0 && node->opening_loc.start == NULL) { node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } + node->base.location.end = MAX(node->base.location.end, part->location.end); - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + break; + case PM_INTERPOLATED_STRING_NODE: + if (PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + // If the string that we're concatenating is a static literal, + // then we can keep the static literal flag for this string. + } else { + // Otherwise, we lose the static literal flag here and we should + // also clear the mutability flags. + CLEAR_FLAGS(node); + } + break; + case PM_EMBEDDED_STATEMENTS_NODE: { + pm_embedded_statements_node_t *cast = (pm_embedded_statements_node_t *) part; + pm_node_t *embedded = (cast->statements != NULL && cast->statements->body.size == 1) ? cast->statements->body.nodes[0] : NULL; + + if (embedded == NULL) { + // If we're embedding multiple statements or no statements, then + // the string is not longer a static literal. + CLEAR_FLAGS(node); + } else if (PM_NODE_TYPE_P(embedded, PM_STRING_NODE)) { + // If the embedded statement is a string, then we can make that + // string as frozen and static literal, and not touch the static + // literal status of this string. + pm_node_flag_set(embedded, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + MUTABLE_FLAGS(node); + } + } else if (PM_NODE_TYPE_P(embedded, PM_INTERPOLATED_STRING_NODE) && PM_NODE_FLAG_P(embedded, PM_NODE_FLAG_STATIC_LITERAL)) { + // If the embedded statement is an interpolated string, but that + // string is marked as static literal, then we can keep our + // static literal status for this string. + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + MUTABLE_FLAGS(node); + } + } else { + // In all other cases, we lose the static literal flag here and + // become mutable. + CLEAR_FLAGS(node); + } + + break; + } + case PM_EMBEDDED_VARIABLE_NODE: + // Embedded variables clear static literal, which means we also + // should clear the mutability flags. + CLEAR_FLAGS(node); + break; + default: + assert(false && "unexpected node type"); + break; } pm_node_list_append(&node->parts, part); - node->base.location.end = MAX(node->base.location.end, part->location.end); - if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { - switch (parser->frozen_string_literal) { - case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); - break; - case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); - break; - } - } +#undef CLEAR_FLAGS +#undef MUTABLE_FLAGS } /** @@ -5122,11 +5396,21 @@ pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_n static pm_interpolated_string_node_t * pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_node_list_t *parts, const pm_token_t *closing) { pm_interpolated_string_node_t *node = PM_ALLOC_NODE(parser, pm_interpolated_string_node_t); + pm_node_flags_t flags = PM_NODE_FLAG_STATIC_LITERAL; + + switch (parser->frozen_string_literal) { + case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: + flags |= PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE; + break; + case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: + flags |= PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN; + break; + } *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, + .flags = flags, .location = { .start = opening->start, .end = closing->end, @@ -5140,7 +5424,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin if (parts != NULL) { pm_node_t *part; PM_NODE_LIST_FOREACH(parts, index, part) { - pm_interpolated_string_node_append(parser, node, part); + pm_interpolated_string_node_append(node, part); } } @@ -5162,15 +5446,7 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - - pm_node_list_append(&node->parts, part); + pm_interpolated_node_append((pm_node_t *) node, &node->parts, part); node->base.location.end = MAX(node->base.location.end, part->location.end); } @@ -5236,11 +5512,7 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - pm_node_list_append(&node->parts, part); + pm_interpolated_node_append((pm_node_t *) node, &node->parts, part); node->base.location.end = part->location.end; } @@ -5251,6 +5523,23 @@ pm_interpolated_xstring_node_closing_set(pm_interpolated_x_string_node_t *node, } /** + * Create a local variable read that is reading the implicit 'it' variable. + */ +static pm_it_local_variable_read_node_t * +pm_it_local_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name) { + pm_it_local_variable_read_node_t *node = PM_ALLOC_NODE(parser, pm_it_local_variable_read_node_t); + + *node = (pm_it_local_variable_read_node_t) { + { + .type = PM_IT_LOCAL_VARIABLE_READ_NODE, + .location = PM_LOCATION_TOKEN_VALUE(name) + } + }; + + return node; +} + +/** * Allocate and initialize a new ItParametersNode node. */ static pm_it_parameters_node_t * @@ -5452,10 +5741,10 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar } }, .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value, .name = name, - .operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), .depth = depth }; @@ -5563,28 +5852,6 @@ pm_token_is_it(const uint8_t *start, const uint8_t *end) { } /** - * Returns true if the given node is `it` default parameter. - */ -static inline bool -pm_node_is_it(pm_parser_t *parser, pm_node_t *node) { - // Check if it's a local variable reference - if (node->type != PM_CALL_NODE) { - return false; - } - - // Check if it's a variable call - pm_call_node_t *call_node = (pm_call_node_t *) node; - if (!PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - return false; - } - - // Check if it's called `it` - pm_constant_id_t id = ((pm_call_node_t *)node)->name; - pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, id); - return pm_token_is_it(constant->start, constant->start + constant->length); -} - -/** * Returns true if the given bounds comprise a numbered parameter (i.e., they * are of the form /^_\d$/). */ @@ -6734,7 +7001,7 @@ pm_statements_node_body_append(pm_parser_t *parser, pm_statements_node_t *node, case PM_REDO_NODE: case PM_RETRY_NODE: case PM_RETURN_NODE: - pm_parser_warn_node(parser, previous, PM_WARN_UNREACHABLE_STATEMENT); + pm_parser_warn_node(parser, statement, PM_WARN_UNREACHABLE_STATEMENT); break; default: break; @@ -6841,7 +7108,8 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument } /** - * Read through the contents of a string and check if it consists solely of US ASCII code points. + * Read through the contents of a string and check if it consists solely of + * US-ASCII code points. */ static bool pm_ascii_only_p(const pm_string_t *contents) { @@ -6856,26 +7124,71 @@ pm_ascii_only_p(const pm_string_t *contents) { } /** + * Validate that the contents of the given symbol are all valid UTF-8. + */ +static void +parse_symbol_encoding_validate_utf8(pm_parser_t *parser, const pm_token_t *location, const pm_string_t *contents) { + for (const uint8_t *cursor = pm_string_source(contents), *end = cursor + pm_string_length(contents); cursor < end;) { + size_t width = pm_encoding_utf_8_char_width(cursor, end - cursor); + + if (width == 0) { + pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + break; + } + + cursor += width; + } +} + +/** + * Validate that the contents of the given symbol are all valid in the encoding + * of the parser. + */ +static void +parse_symbol_encoding_validate_other(pm_parser_t *parser, const pm_token_t *location, const pm_string_t *contents) { + const pm_encoding_t *encoding = parser->encoding; + + for (const uint8_t *cursor = pm_string_source(contents), *end = cursor + pm_string_length(contents); cursor < end;) { + size_t width = encoding->char_width(cursor, end - cursor); + + if (width == 0) { + pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + break; + } + + cursor += width; + } +} + +/** * Ruby "downgrades" the encoding of Symbols to US-ASCII if the associated * encoding is ASCII-compatible and the Symbol consists only of US-ASCII code * points. Otherwise, the encoding may be explicitly set with an escape * sequence. + * + * If the validate flag is set, then it will check the contents of the symbol + * to ensure that all characters are valid in the encoding. */ static inline pm_node_flags_t -parse_symbol_encoding(const pm_parser_t *parser, const pm_string_t *contents) { +parse_symbol_encoding(pm_parser_t *parser, const pm_token_t *location, const pm_string_t *contents, bool validate) { if (parser->explicit_encoding != NULL) { // A Symbol may optionally have its encoding explicitly set. This will // happen if an escape sequence results in a non-ASCII code point. if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { + if (validate) parse_symbol_encoding_validate_utf8(parser, location, contents); return PM_SYMBOL_FLAGS_FORCED_UTF8_ENCODING; } else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { return PM_SYMBOL_FLAGS_FORCED_BINARY_ENCODING; + } else if (validate) { + parse_symbol_encoding_validate_other(parser, location, contents); } } else if (pm_ascii_only_p(contents)) { // Ruby stipulates that all source files must use an ASCII-compatible // encoding. Thus, all symbols appearing in source are eligible for // "downgrading" to US-ASCII. return PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING; + } else if (validate) { + parse_symbol_encoding_validate_other(parser, location, contents); } return 0; @@ -7043,7 +7356,7 @@ pm_symbol_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_t */ static pm_symbol_node_t * pm_symbol_node_create_current_string(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing) { - pm_symbol_node_t *node = pm_symbol_node_create_unescaped(parser, opening, value, closing, &parser->current_string, parse_symbol_encoding(parser, &parser->current_string)); + pm_symbol_node_t *node = pm_symbol_node_create_unescaped(parser, opening, value, closing, &parser->current_string, parse_symbol_encoding(parser, value, &parser->current_string, false)); parser->current_string = PM_STRING_EMPTY; return node; } @@ -7065,7 +7378,7 @@ pm_symbol_node_label_create(pm_parser_t *parser, const pm_token_t *token) { assert((label.end - label.start) >= 0); pm_string_shared_init(&node->unescaped, label.start, label.end); - pm_node_flag_set((pm_node_t *) node, parse_symbol_encoding(parser, &node->unescaped)); + pm_node_flag_set((pm_node_t *) node, parse_symbol_encoding(parser, &label, &node->unescaped, false)); break; } @@ -7097,9 +7410,9 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { { .type = PM_SYMBOL_NODE, .flags = PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, - .value_loc = { .start = parser->start, .end = parser->start }, + .value_loc = PM_LOCATION_NULL_VALUE(parser), .unescaped = { 0 } }; @@ -7150,7 +7463,8 @@ pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const .unescaped = node->unescaped }; - pm_node_flag_set((pm_node_t *)new_node, parse_symbol_encoding(parser, &node->unescaped)); + pm_token_t content = { .type = PM_TOKEN_IDENTIFIER, .start = node->content_loc.start, .end = node->content_loc.end }; + pm_node_flag_set((pm_node_t *) new_node, parse_symbol_encoding(parser, &content, &node->unescaped, true)); // We are explicitly _not_ using pm_node_destroy here because we don't want // to trash the unescaped string. We could instead copy the string if we @@ -7499,10 +7813,10 @@ pm_while_node_synthesized_create(pm_parser_t *parser, pm_node_t *predicate, pm_s *node = (pm_while_node_t) { { .type = PM_WHILE_NODE, - .location = { .start = parser->start, .end = parser->start } + .location = PM_LOCATION_NULL_VALUE(parser) }, - .keyword_loc = { .start = parser->start, .end = parser->start }, - .closing_loc = { .start = parser->start, .end = parser->start }, + .keyword_loc = PM_LOCATION_NULL_VALUE(parser), + .closing_loc = PM_LOCATION_NULL_VALUE(parser), .predicate = predicate, .statements = statements }; @@ -7658,51 +7972,6 @@ pm_parser_local_add_constant(pm_parser_t *parser, const char *start, size_t leng } /** - * Create a local variable read that is reading the implicit 'it' variable. - */ -static pm_local_variable_read_node_t * -pm_local_variable_read_node_create_it(pm_parser_t *parser, const pm_token_t *name) { - if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_ORDINARY) { - pm_parser_err_token(parser, name, PM_ERR_IT_NOT_ALLOWED_ORDINARY); - return NULL; - } - - if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED) { - pm_parser_err_token(parser, name, PM_ERR_IT_NOT_ALLOWED_NUMBERED); - return NULL; - } - - parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_IT; - - pm_constant_id_t name_id = pm_parser_constant_id_constant(parser, "0it", 3); - pm_parser_local_add(parser, name_id, name->start, name->end, 0); - - return pm_local_variable_read_node_create_constant_id(parser, name, name_id, 0, false); -} - -/** - * Convert a `it` variable call node to a node for `it` default parameter. - */ -static pm_node_t * -pm_node_check_it(pm_parser_t *parser, pm_node_t *node) { - if ( - (parser->version != PM_OPTIONS_VERSION_CRUBY_3_3_0) && - !parser->current_scope->closed && - (parser->current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED) && - pm_node_is_it(parser, node) - ) { - pm_local_variable_read_node_t *read = pm_local_variable_read_node_create_it(parser, &parser->previous); - - if (read != NULL) { - pm_node_destroy(parser, node); - node = (pm_node_t *) read; - } - } - - return node; -} - -/** * Add a parameter name to the current scope and check whether the name of the * parameter is unique or not. * @@ -7737,6 +8006,7 @@ pm_parser_scope_pop(pm_parser_t *parser) { pm_scope_t *scope = parser->current_scope; parser->current_scope = scope->previous; pm_locals_free(&scope->locals); + pm_node_list_free(&scope->implicit_parameters); xfree(scope); } @@ -7808,7 +8078,7 @@ pm_do_loop_stack_p(pm_parser_t *parser) { * is beyond the end of the source then return '\0'. */ static inline uint8_t -peek_at(pm_parser_t *parser, const uint8_t *cursor) { +peek_at(const pm_parser_t *parser, const uint8_t *cursor) { if (cursor < parser->end) { return *cursor; } else { @@ -7831,7 +8101,7 @@ peek_offset(pm_parser_t *parser, ptrdiff_t offset) { * that position is beyond the end of the source then return '\0'. */ static inline uint8_t -peek(pm_parser_t *parser) { +peek(const pm_parser_t *parser) { return peek_at(parser, parser->current.end); } @@ -7897,6 +8167,14 @@ next_newline(const uint8_t *cursor, ptrdiff_t length) { } /** + * This is equivalent to the predicate of warn_balanced in CRuby. + */ +static inline bool +ambiguous_operator_p(const pm_parser_t *parser, bool space_seen) { + return !lex_state_p(parser, PM_LEX_STATE_CLASS | PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME | PM_LEX_STATE_ENDFN) && space_seen && !pm_char_is_whitespace(peek(parser)); +} + +/** * Here we're going to check if this is a "magic" comment, and perform whatever * actions are necessary for it here. */ @@ -8135,7 +8413,12 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // If we have hit a ractor pragma, attempt to lex that. uint32_t value_length = (uint32_t) (value_end - value_start); if (key_length == 24 && pm_strncasecmp(key_source, (const uint8_t *) "shareable_constant_value", 24) == 0) { - if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "none", 4) == 0) { + const uint8_t *cursor = parser->current.start; + while ((cursor > parser->start) && ((cursor[-1] == ' ') || (cursor[-1] == '\t'))) cursor--; + + if (!((cursor == parser->start) || (cursor[-1] == '\n'))) { + pm_parser_warn_token(parser, &parser->current, PM_WARN_SHAREABLE_CONSTANT_VALUE_LINE); + } else if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "none", 4) == 0) { pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_NONE); } else if (value_length == 7 && pm_strncasecmp(value_start, (const uint8_t *) "literal", 7) == 0) { pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_LITERAL); @@ -8410,10 +8693,11 @@ context_human(pm_context_t context) { /* Specific token lexers */ /******************************************************************************/ -static void -pm_strspn_number_validate(pm_parser_t *parser, const uint8_t *invalid) { +static inline void +pm_strspn_number_validate(pm_parser_t *parser, const uint8_t *string, size_t length, const uint8_t *invalid) { if (invalid != NULL) { - pm_parser_err(parser, invalid, invalid + 1, PM_ERR_INVALID_NUMBER_UNDERSCORE); + pm_diagnostic_id_t diag_id = (invalid == (string + length - 1)) ? PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING : PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER; + pm_parser_err(parser, invalid, invalid + 1, diag_id); } } @@ -8421,7 +8705,7 @@ static size_t pm_strspn_binary_number_validate(pm_parser_t *parser, const uint8_t *string) { const uint8_t *invalid = NULL; size_t length = pm_strspn_binary_number(string, parser->end - string, &invalid); - pm_strspn_number_validate(parser, invalid); + pm_strspn_number_validate(parser, string, length, invalid); return length; } @@ -8429,7 +8713,7 @@ static size_t pm_strspn_octal_number_validate(pm_parser_t *parser, const uint8_t *string) { const uint8_t *invalid = NULL; size_t length = pm_strspn_octal_number(string, parser->end - string, &invalid); - pm_strspn_number_validate(parser, invalid); + pm_strspn_number_validate(parser, string, length, invalid); return length; } @@ -8437,7 +8721,7 @@ static size_t pm_strspn_decimal_number_validate(pm_parser_t *parser, const uint8_t *string) { const uint8_t *invalid = NULL; size_t length = pm_strspn_decimal_number(string, parser->end - string, &invalid); - pm_strspn_number_validate(parser, invalid); + pm_strspn_number_validate(parser, string, length, invalid); return length; } @@ -8445,7 +8729,7 @@ static size_t pm_strspn_hexadecimal_number_validate(pm_parser_t *parser, const uint8_t *string) { const uint8_t *invalid = NULL; size_t length = pm_strspn_hexadecimal_number(string, parser->end - string, &invalid); - pm_strspn_number_validate(parser, invalid); + pm_strspn_number_validate(parser, string, length, invalid); return length; } @@ -8591,6 +8875,16 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) { type = lex_optional_float_suffix(parser, seen_e); } + // At this point we have a completed number, but we want to provide the user + // with a good experience if they put an additional .xxx fractional + // component on the end, so we'll check for that here. + if (peek_offset(parser, 0) == '.' && pm_char_is_decimal_digit(peek_offset(parser, 1))) { + const uint8_t *fraction_start = parser->current.end; + const uint8_t *fraction_end = parser->current.end + 2; + fraction_end += pm_strspn_decimal_digit(fraction_end, parser->end - fraction_end); + pm_parser_err(parser, fraction_start, fraction_end, PM_ERR_INVALID_NUMBER_FRACTION); + } + return type; } @@ -8683,7 +8977,7 @@ lex_global_variable(pm_parser_t *parser) { } while (parser->current.end < parser->end && (width = char_is_identifier(parser, parser->current.end)) > 0); // $0 isn't allowed to be followed by anything. - pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; + pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); } @@ -8719,9 +9013,9 @@ lex_global_variable(pm_parser_t *parser) { } else { // If we get here, then we have a $ followed by something that // isn't recognized as a global variable. - pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; - size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; + const uint8_t *end = parser->current.end + parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_FORMAT(parser, parser->current.start, end, diag_id, (int) (end - parser->current.start), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -9092,12 +9386,20 @@ escape_hexadecimal_digit(const uint8_t value) { * validated. */ static inline uint32_t -escape_unicode(const uint8_t *string, size_t length) { +escape_unicode(pm_parser_t *parser, const uint8_t *string, size_t length) { uint32_t value = 0; for (size_t index = 0; index < length; index++) { if (index != 0) value <<= 4; value |= escape_hexadecimal_digit(string[index]); } + + // Here we're going to verify that the value is actually a valid Unicode + // codepoint and not a surrogate pair. + if (value >= 0xD800 && value <= 0xDFFF) { + pm_parser_err(parser, string, string + length, PM_ERR_ESCAPE_INVALID_UNICODE); + return 0xFFFD; + } + return value; } @@ -9106,7 +9408,7 @@ escape_unicode(const uint8_t *string, size_t length) { */ static inline uint8_t escape_byte(uint8_t value, const uint8_t flags) { - if (flags & PM_ESCAPE_FLAG_CONTROL) value &= 0x1f; + if (flags & PM_ESCAPE_FLAG_CONTROL) value &= 0x9f; if (flags & PM_ESCAPE_FLAG_META) value |= 0x80; return value; } @@ -9206,22 +9508,7 @@ escape_write_escape_encoded(pm_parser_t *parser, pm_buffer_t *buffer) { static inline void escape_write_byte(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expression_buffer, uint8_t flags, uint8_t byte) { if (flags & PM_ESCAPE_FLAG_REGEXP) { - pm_buffer_append_bytes(regular_expression_buffer, (const uint8_t *) "\\x", 2); - - uint8_t byte1 = (uint8_t) ((byte >> 4) & 0xF); - uint8_t byte2 = (uint8_t) (byte & 0xF); - - if (byte1 >= 0xA) { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) ((byte1 - 0xA) + 'A')); - } else { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte1 + '0')); - } - - if (byte2 >= 0xA) { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte2 - 0xA + 'A')); - } else { - pm_buffer_append_byte(regular_expression_buffer, (uint8_t) (byte2 + '0')); - } + pm_buffer_append_format(regular_expression_buffer, "\\x%02X", byte); } escape_write_byte_encoded(parser, buffer, byte); @@ -9256,57 +9543,57 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre switch (peek(parser)) { case '\\': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\\', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\\', flags)); return; } case '\'': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\'', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\'', flags)); return; } case 'a': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\a', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\a', flags)); return; } case 'b': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\b', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\b', flags)); return; } case 'e': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\033', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\033', flags)); return; } case 'f': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\f', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\f', flags)); return; } case 'n': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\n', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\n', flags)); return; } case 'r': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\r', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\r', flags)); return; } case 's': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte(' ', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte(' ', flags)); return; } case 't': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\t', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\t', flags)); return; } case 'v': { parser->current.end++; - escape_write_byte_encoded(parser, buffer, escape_byte('\v', flags)); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, escape_byte('\v', flags)); return; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { @@ -9323,7 +9610,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre } } - escape_write_byte_encoded(parser, buffer, value); + escape_write_byte(parser, buffer, regular_expression_buffer, flags, value); return; } case 'x': { @@ -9342,8 +9629,13 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; } + value = escape_byte(value, flags); if (flags & PM_ESCAPE_FLAG_REGEXP) { - pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); + if (flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) { + pm_buffer_append_format(regular_expression_buffer, "\\x%02X", value); + } else { + pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); + } } escape_write_byte_encoded(parser, buffer, value); @@ -9357,22 +9649,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre const uint8_t *start = parser->current.end - 1; parser->current.end++; - if ( - (parser->current.end + 4 <= parser->end) && - pm_char_is_hexadecimal_digit(parser->current.end[0]) && - pm_char_is_hexadecimal_digit(parser->current.end[1]) && - pm_char_is_hexadecimal_digit(parser->current.end[2]) && - pm_char_is_hexadecimal_digit(parser->current.end[3]) - ) { - uint32_t value = escape_unicode(parser->current.end, 4); - - if (flags & PM_ESCAPE_FLAG_REGEXP) { - pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end + 4 - start)); - } - escape_write_unicode(parser, buffer, flags, start, parser->current.end + 4, value); - - parser->current.end += 4; - } else if (peek(parser) == '{') { + if (peek(parser) == '{') { const uint8_t *unicode_codepoints_start = parser->current.end - 2; parser->current.end++; @@ -9390,7 +9667,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE_LONG); } else if (hexadecimal_length == 0) { // there are not hexadecimal characters - pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); return; } @@ -9400,7 +9678,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre extra_codepoints_start = unicode_start; } - uint32_t value = escape_unicode(unicode_start, hexadecimal_length); + uint32_t value = escape_unicode(parser, unicode_start, hexadecimal_length); escape_write_unicode(parser, buffer, flags, unicode_start, parser->current.end, value); parser->current.end += pm_strspn_whitespace(parser->current.end, parser->end - parser->current.end); @@ -9422,7 +9700,21 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre pm_buffer_append_bytes(regular_expression_buffer, unicode_codepoints_start, (size_t) (parser->current.end - unicode_codepoints_start)); } } else { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_UNICODE); + size_t length = pm_strspn_hexadecimal_digit(parser->current.end, MIN(parser->end - parser->current.end, 4)); + + if (length == 4) { + uint32_t value = escape_unicode(parser, parser->current.end, 4); + + if (flags & PM_ESCAPE_FLAG_REGEXP) { + pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end + 4 - start)); + } + + escape_write_unicode(parser, buffer, flags, start, parser->current.end + 4, value); + parser->current.end += 4; + } else { + parser->current.end += length; + pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_UNICODE); + } } return; @@ -9447,6 +9739,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_CONTROL); return; case ' ': @@ -9474,7 +9772,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre case 'C': { parser->current.end++; if (peek(parser) != '-') { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9497,6 +9796,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_CONTROL); return; case ' ': @@ -9511,7 +9816,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; default: { if (!char_is_ascii_printable(peeked)) { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9524,7 +9830,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre case 'M': { parser->current.end++; if (peek(parser) != '-') { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); return; } @@ -9542,6 +9849,12 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; } parser->current.end++; + + if (match(parser, 'u') || match(parser, 'U')) { + pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + return; + } + escape_read(parser, buffer, regular_expression_buffer, flags | PM_ESCAPE_FLAG_META); return; case ' ': @@ -9556,7 +9869,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre return; default: if (!char_is_ascii_printable(peeked)) { - pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); return; } @@ -9676,8 +9990,8 @@ lex_at_variable(pm_parser_t *parser) { } } else if (parser->current.end < parser->end && pm_char_is_decimal_digit(*parser->current.end)) { pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE; - if (parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0) { - diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0 : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0; + if (parser->version == PM_OPTIONS_VERSION_CRUBY_3_3) { + diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3 : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3; } size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); @@ -10517,6 +10831,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_USTAR_STAR; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR_STAR; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10540,6 +10856,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_USTAR; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10656,13 +10974,17 @@ parser_lex(pm_parser_t *parser) { // If we have quotes, then we're going to go until we find the // end quote. while ((parser->current.end < parser->end) && quote != (pm_heredoc_quote_t) (*parser->current.end)) { + if (*parser->current.end == '\r' || *parser->current.end == '\n') break; parser->current.end++; } } size_t ident_length = (size_t) (parser->current.end - ident_start); + bool ident_error = false; + if (quote != PM_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) { - // TODO: handle unterminated heredoc + pm_parser_err(parser, ident_start, ident_start + ident_length, PM_ERR_HEREDOC_IDENTIFIER); + ident_error = true; } parser->explicit_encoding = NULL; @@ -10687,7 +11009,7 @@ parser_lex(pm_parser_t *parser) { // this is not a valid heredoc declaration. In this case we // will add an error, but we will still return a heredoc // start. - pm_parser_err_current(parser, PM_ERR_HEREDOC_TERM); + if (!ident_error) pm_parser_err_heredoc_term(parser, parser->lex_modes.current); body_start = parser->end; } else { // Otherwise, we want to indicate that the body of the @@ -10710,6 +11032,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_LESS_LESS_EQUAL); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); + } + if (lex_state_operator_p(parser)) { lex_state_set(parser, PM_LEX_STATE_ARG); } else { @@ -10823,6 +11149,8 @@ parser_lex(pm_parser_t *parser) { type = PM_TOKEN_UAMPERSAND; } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_UAMPERSAND; + } else if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10897,6 +11225,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_UPLUS); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); + } + lex_state_set(parser, PM_LEX_STATE_BEG); LEX(PM_TOKEN_PLUS); } @@ -10934,6 +11266,10 @@ parser_lex(pm_parser_t *parser) { LEX(pm_char_is_decimal_digit(peek(parser)) ? PM_TOKEN_UMINUS_NUM : PM_TOKEN_UMINUS); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); + } + lex_state_set(parser, PM_LEX_STATE_BEG); LEX(PM_TOKEN_MINUS); } @@ -11032,6 +11368,10 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_REGEXP_BEGIN); } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); + } + if (lex_state_operator_p(parser)) { lex_state_set(parser, PM_LEX_STATE_ARG); } else { @@ -11067,7 +11407,7 @@ parser_lex(pm_parser_t *parser) { // operator because we don't want to move into the string // lex mode unnecessarily. if ((lex_state_beg_p(parser) || lex_state_arg_p(parser)) && (parser->current.end >= parser->end)) { - pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT); + pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT_EOF); LEX(PM_TOKEN_PERCENT); } @@ -11086,10 +11426,7 @@ parser_lex(pm_parser_t *parser) { const uint8_t delimiter = pm_lex_percent_delimiter(parser); lex_mode_push_string(parser, true, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter)); - - if (parser->current.end < parser->end) { - LEX(PM_TOKEN_STRING_BEGIN); - } + LEX(PM_TOKEN_STRING_BEGIN); } // Delimiters for %-literals cannot be alphanumeric. We @@ -11216,6 +11553,10 @@ parser_lex(pm_parser_t *parser) { } } + if (ambiguous_operator_p(parser, space_seen)) { + PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); + } + lex_state_set(parser, lex_state_operator_p(parser) ? PM_LEX_STATE_ARG : PM_LEX_STATE_BEG); LEX(PM_TOKEN_PERCENT); } @@ -12014,7 +12355,7 @@ parser_lex(pm_parser_t *parser) { // terminator) but still continue parsing so that content after the // declaration of the heredoc can be parsed. if (parser->current.end >= parser->end) { - pm_parser_err_current(parser, PM_ERR_HEREDOC_TERM); + pm_parser_err_heredoc_term(parser, lex_mode); parser->next_start = lex_mode->as.heredoc.next_start; parser->heredoc_end = parser->current.end; lex_state_set(parser, PM_LEX_STATE_END); @@ -12653,6 +12994,23 @@ expect3(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_to parser->previous.type = PM_TOKEN_MISSING; } +/** + * A special expect1 that expects a heredoc terminator and handles popping the + * lex mode accordingly. + */ +static void +expect1_heredoc_term(pm_parser_t *parser, pm_lex_mode_t *lex_mode) { + if (match1(parser, PM_TOKEN_HEREDOC_END)) { + lex_mode_pop(parser); + parser_lex(parser); + } else { + pm_parser_err_heredoc_term(parser, lex_mode); + lex_mode_pop(parser); + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + } +} + static pm_node_t * parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool accepts_command_call, pm_diagnostic_id_t diag_id); @@ -12781,24 +13139,95 @@ parse_write_name(pm_parser_t *parser, pm_constant_id_t *name_field) { } /** + * Certain expressions are not targetable, but in order to provide a better + * experience we give a specific error message. In order to maintain as much + * information in the tree as possible, we replace them with local variable + * writes. + */ +static pm_node_t * +parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) { + switch (PM_NODE_TYPE(target)) { + case PM_SOURCE_ENCODING_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_ENCODING); break; + case PM_FALSE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_FALSE); break; + case PM_SOURCE_FILE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_FILE); break; + case PM_SOURCE_LINE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_LINE); break; + case PM_NIL_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_NIL); break; + case PM_SELF_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_SELF); break; + case PM_TRUE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE); break; + default: break; + } + + pm_constant_id_t name = pm_parser_constant_id_location(parser, target->location.start, target->location.end); + pm_local_variable_target_node_t *result = pm_local_variable_target_node_create(parser, &target->location, name, 0); + + pm_node_destroy(parser, target); + return (pm_node_t *) result; +} + +/** + * When an implicit local variable is written to or targeted, it becomes a + * regular, named local variable. This function removes it from the list of + * implicit parameters when that happens. + */ +static void +parse_target_implicit_parameter(pm_parser_t *parser, pm_node_t *node) { + pm_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters; + + for (size_t index = 0; index < implicit_parameters->size; index++) { + if (implicit_parameters->nodes[index] == node) { + // If the node is not the last one in the list, we need to shift the + // remaining nodes down to fill the gap. This is extremely unlikely + // to happen. + if (index != implicit_parameters->size - 1) { + memcpy(&implicit_parameters->nodes[index], &implicit_parameters->nodes[index + 1], (implicit_parameters->size - index - 1) * sizeof(pm_node_t *)); + } + + implicit_parameters->size--; + break; + } + } +} + +/** * Convert the given node into a valid target node. */ static pm_node_t * -parse_target(pm_parser_t *parser, pm_node_t *target) { +parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple) { switch (PM_NODE_TYPE(target)) { case PM_MISSING_NODE: return target; + case PM_SOURCE_ENCODING_NODE: + case PM_FALSE_NODE: + case PM_SOURCE_FILE_NODE: + case PM_SOURCE_LINE_NODE: + case PM_NIL_NODE: + case PM_SELF_NODE: + case PM_TRUE_NODE: { + // In these special cases, we have specific error messages and we + // will replace them with local variable writes. + return parse_unwriteable_target(parser, target); + } case PM_CLASS_VARIABLE_READ_NODE: assert(sizeof(pm_class_variable_target_node_t) == sizeof(pm_class_variable_read_node_t)); target->type = PM_CLASS_VARIABLE_TARGET_NODE; return target; case PM_CONSTANT_PATH_NODE: + if (context_def_p(parser)) { + pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_IN_METHOD); + } + assert(sizeof(pm_constant_path_target_node_t) == sizeof(pm_constant_path_node_t)); target->type = PM_CONSTANT_PATH_TARGET_NODE; + return target; case PM_CONSTANT_READ_NODE: + if (context_def_p(parser)) { + pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_IN_METHOD); + } + assert(sizeof(pm_constant_target_node_t) == sizeof(pm_constant_read_node_t)); target->type = PM_CONSTANT_TARGET_NODE; + return target; case PM_BACK_REFERENCE_READ_NODE: case PM_NUMBERED_REFERENCE_READ_NODE: @@ -12809,7 +13238,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target) { target->type = PM_GLOBAL_VARIABLE_TARGET_NODE; return target; case PM_LOCAL_VARIABLE_READ_NODE: { - pm_refute_numbered_parameter(parser, target->location.start, target->location.end); + if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, target->location.start); + parse_target_implicit_parameter(parser, target); + } const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) target; uint32_t name = cast->name; @@ -12821,6 +13253,15 @@ parse_target(pm_parser_t *parser, pm_node_t *target) { return target; } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); + pm_node_t *node = (pm_node_t *) pm_local_variable_target_node_create(parser, &target->location, name, 0); + + parse_target_implicit_parameter(parser, target); + pm_node_destroy(parser, target); + + return node; + } case PM_INSTANCE_VARIABLE_READ_NODE: assert(sizeof(pm_instance_variable_target_node_t) == sizeof(pm_instance_variable_read_node_t)); target->type = PM_INSTANCE_VARIABLE_TARGET_NODE; @@ -12831,7 +13272,7 @@ parse_target(pm_parser_t *parser, pm_node_t *target) { pm_splat_node_t *splat = (pm_splat_node_t *) target; if (splat->expression != NULL) { - splat->expression = parse_target(parser, splat->expression); + splat->expression = parse_target(parser, splat->expression, multiple); } return (pm_node_t *) splat; @@ -12869,6 +13310,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target) { } if (*call->message_loc.start == '_' || parser->encoding->alnum_char(call->message_loc.start, call->message_loc.end - call->message_loc.start)) { + if (multiple && PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { + pm_parser_err_node(parser, (const pm_node_t *) call, PM_ERR_UNEXPECTED_SAFE_NAVIGATION); + } + parse_write_name(parser, &call->name); return (pm_node_t *) pm_call_target_node_create(parser, call); } @@ -12896,8 +13341,8 @@ parse_target(pm_parser_t *parser, pm_node_t *target) { * assignment. */ static pm_node_t * -parse_target_validate(pm_parser_t *parser, pm_node_t *target) { - pm_node_t *result = parse_target(parser, target); +parse_target_validate(pm_parser_t *parser, pm_node_t *target, bool multiple) { + pm_node_t *result = parse_target(parser, target, multiple); // Ensure that we have one of an =, an 'in' in for indexes, and a ')' in parens after the targets. if ( @@ -12942,13 +13387,20 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod } case PM_CONSTANT_PATH_NODE: { pm_node_t *node = (pm_node_t *) pm_constant_path_write_node_create(parser, (pm_constant_path_node_t *) target, operator, value); + + if (context_def_p(parser)) { + pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_IN_METHOD); + } + return parse_shareable_constant_write(parser, node); } case PM_CONSTANT_READ_NODE: { pm_node_t *node = (pm_node_t *) pm_constant_write_node_create(parser, (pm_constant_read_node_t *) target, operator, value); + if (context_def_p(parser)) { pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_IN_METHOD); } + pm_node_destroy(parser, target); return parse_shareable_constant_write(parser, node); } @@ -12962,18 +13414,34 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod return (pm_node_t *) node; } case PM_LOCAL_VARIABLE_READ_NODE: { - pm_refute_numbered_parameter(parser, target->location.start, target->location.end); pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target; pm_constant_id_t name = local_read->name; + pm_location_t name_loc = target->location; + uint32_t depth = local_read->depth; - pm_locals_unread(&pm_parser_scope_find(parser, depth)->locals, name); + pm_scope_t *scope = pm_parser_scope_find(parser, depth); - pm_location_t name_loc = target->location; + if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED; + PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, diag_id, target->location.start); + parse_target_implicit_parameter(parser, target); + } + + pm_locals_unread(&scope->locals, name); pm_node_destroy(parser, target); return (pm_node_t *) pm_local_variable_write_node_create(parser, name, depth, value, &name_loc, operator); } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); + pm_node_t *node = (pm_node_t *) pm_local_variable_write_node_create(parser, name, 0, value, &target->location, operator); + + parse_target_implicit_parameter(parser, target); + pm_node_destroy(parser, target); + + return node; + } case PM_INSTANCE_VARIABLE_READ_NODE: { pm_node_t *write_node = (pm_node_t *) pm_instance_variable_write_node_create(parser, (pm_instance_variable_read_node_t *) target, operator, value); pm_node_destroy(parser, target); @@ -13127,7 +13595,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b bool has_rest = PM_NODE_TYPE_P(first_target, PM_SPLAT_NODE); pm_multi_target_node_t *result = pm_multi_target_node_create(parser); - pm_multi_target_node_targets_append(parser, result, parse_target(parser, first_target)); + pm_multi_target_node_targets_append(parser, result, parse_target(parser, first_target, true)); while (accept1(parser, PM_TOKEN_COMMA)) { if (accept1(parser, PM_TOKEN_USTAR)) { @@ -13143,7 +13611,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b if (token_begins_expression_p(parser->current.type)) { name = parse_expression(parser, binding_power, false, PM_ERR_EXPECT_EXPRESSION_AFTER_STAR); - name = parse_target(parser, name); + name = parse_target(parser, name, true); } pm_node_t *splat = (pm_node_t *) pm_splat_node_create(parser, &star_operator, name); @@ -13151,7 +13619,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b has_rest = true; } else if (token_begins_expression_p(parser->current.type)) { pm_node_t *target = parse_expression(parser, binding_power, false, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA); - target = parse_target(parser, target); + target = parse_target(parser, target, true); pm_multi_target_node_targets_append(parser, result, target); } else if (!match1(parser, PM_TOKEN_EOF)) { @@ -13188,8 +13656,8 @@ parse_targets_validate(pm_parser_t *parser, pm_node_t *first_target, pm_binding_ */ static pm_statements_node_t * parse_statements(pm_parser_t *parser, pm_context_t context) { - // First, skip past any optional terminators that might be at the beginning of - // the statements. + // First, skip past any optional terminators that might be at the beginning + // of the statements. while (accept2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)); // If we have a terminator, then we can just return NULL. @@ -13205,20 +13673,20 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_STATEMENT, true, PM_ERR_CANNOT_PARSE_EXPRESSION); pm_statements_node_body_append(parser, statements, node); - // If we're recovering from a syntax error, then we need to stop parsing the - // statements now. + // If we're recovering from a syntax error, then we need to stop parsing + // the statements now. if (parser->recovering) { - // If this is the level of context where the recovery has happened, then - // we can mark the parser as done recovering. + // If this is the level of context where the recovery has happened, + // then we can mark the parser as done recovering. if (context_terminator(context, &parser->current)) parser->recovering = false; break; } - // If we have a terminator, then we will parse all consecutive terminators - // and then continue parsing the statements list. + // If we have a terminator, then we will parse all consecutive + // terminators and then continue parsing the statements list. if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - // If we have a terminator, then we will continue parsing the statements - // list. + // If we have a terminator, then we will continue parsing the + // statements list. while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)); if (context_terminator(context, &parser->current)) break; @@ -13226,27 +13694,28 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { continue; } - // At this point we have a list of statements that are not terminated by a - // newline or semicolon. At this point we need to check if we're at the end - // of the statements list. If we are, then we should break out of the loop. + // At this point we have a list of statements that are not terminated by + // a newline or semicolon. At this point we need to check if we're at + // the end of the statements list. If we are, then we should break out + // of the loop. if (context_terminator(context, &parser->current)) break; // At this point, we have a syntax error, because the statement was not // terminated by a newline or semicolon, and we're not at the end of the - // statements list. Ideally we should scan forward to determine if we should - // insert a missing terminator or break out of parsing the statements list - // at this point. + // statements list. Ideally we should scan forward to determine if we + // should insert a missing terminator or break out of parsing the + // statements list at this point. // - // We don't have that yet, so instead we'll do a more naive approach. If we - // were unable to parse an expression, then we will skip past this token and - // continue parsing the statements list. Otherwise we'll add an error and - // continue parsing the statements list. + // We don't have that yet, so instead we'll do a more naive approach. If + // we were unable to parse an expression, then we will skip past this + // token and continue parsing the statements list. Otherwise we'll add + // an error and continue parsing the statements list. if (PM_NODE_TYPE_P(node, PM_MISSING_NODE)) { parser_lex(parser); while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)); if (context_terminator(context, &parser->current)) break; - } else if (!accept1(parser, PM_TOKEN_NEWLINE)) { + } else if (!accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_EOF)) { // This is an inlined version of accept1 because the error that we // want to add has varargs. If this happens again, we should // probably extract a helper function. @@ -13268,7 +13737,7 @@ parse_statements(pm_parser_t *parser, pm_context_t context) { */ static void pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node); + const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, true); if (duplicated != NULL) { pm_buffer_t buffer = { 0 }; @@ -13294,13 +13763,16 @@ pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *liter */ static void pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node) != NULL) { + pm_node_t *previous; + + if ((previous = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, node->location.start, node->location.end, PM_WARN_DUPLICATED_WHEN_CLAUSE, - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line, + pm_newline_list_line_column(&parser->newline_list, previous->location.start, parser->start_line).line ); } } @@ -13486,9 +13958,10 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for bool contains_keyword_splat = parse_assocs(parser, &hash_keys, (pm_node_t *) hash); parse_arguments_append(parser, arguments, argument); - if (contains_keyword_splat) { - pm_node_flag_set((pm_node_t *) arguments->arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT); - } + + pm_node_flags_t flags = PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS; + if (contains_keyword_splat) flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT; + pm_node_flag_set((pm_node_t *) arguments->arguments, flags); pm_static_literals_free(&hash_keys); parsed_bare_hash = true; @@ -13566,7 +14039,9 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for argument = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, !parsed_first_argument, PM_ERR_EXPECT_ARGUMENT); } + bool contains_keywords = false; bool contains_keyword_splat = false; + if (pm_symbol_node_label_p(argument) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { if (parsed_bare_hash) { pm_parser_err_previous(parser, PM_ERR_ARGUMENT_BARE_HASH); @@ -13580,6 +14055,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for } pm_keyword_hash_node_t *bare_hash = pm_keyword_hash_node_create(parser); + contains_keywords = true; // Create the set of static literals for this hash. pm_static_literals_t hash_keys = { 0 }; @@ -13608,9 +14084,12 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for } parse_arguments_append(parser, arguments, argument); - if (contains_keyword_splat) { - pm_node_flag_set((pm_node_t *)arguments->arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT); - } + + pm_node_flags_t flags = 0; + if (contains_keywords) flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS; + if (contains_keyword_splat) flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT; + pm_node_flag_set((pm_node_t *) arguments->arguments, flags); + break; } } @@ -13723,7 +14202,6 @@ typedef enum { PM_PARAMETERS_ORDER_OPTIONAL, PM_PARAMETERS_ORDER_NAMED, PM_PARAMETERS_ORDER_NONE, - } pm_parameters_order_t; /** @@ -13748,31 +14226,37 @@ static pm_parameters_order_t parameters_ordering[PM_TOKEN_MAXIMUM] = { * Check if current parameter follows valid parameters ordering. If not it adds * an error to the list without stopping the parsing, otherwise sets the * parameters state to the one corresponding to the current parameter. + * + * It returns true if it was successful, and false otherwise. */ -static void +static bool update_parameter_state(pm_parser_t *parser, pm_token_t *token, pm_parameters_order_t *current) { pm_parameters_order_t state = parameters_ordering[token->type]; - if (state == PM_PARAMETERS_NO_CHANGE) return; + if (state == PM_PARAMETERS_NO_CHANGE) return true; // If we see another ordered argument after a optional argument // we only continue parsing ordered arguments until we stop seeing ordered arguments. if (*current == PM_PARAMETERS_ORDER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) { *current = PM_PARAMETERS_ORDER_AFTER_OPTIONAL; - return; + return true; } else if (*current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) { - return; + return true; } if (token->type == PM_TOKEN_USTAR && *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL) { pm_parser_err_token(parser, token, PM_ERR_PARAMETER_STAR); - } - - if (*current == PM_PARAMETERS_ORDER_NOTHING_AFTER || state > *current) { + return false; + } else if (token->type == PM_TOKEN_UDOT_DOT_DOT && (*current >= PM_PARAMETERS_ORDER_KEYWORDS_REST && *current <= PM_PARAMETERS_ORDER_AFTER_OPTIONAL)) { + pm_parser_err_token(parser, token, *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL ? PM_ERR_PARAMETER_FORWARDING_AFTER_REST : PM_ERR_PARAMETER_ORDER); + return false; + } else if (*current == PM_PARAMETERS_ORDER_NOTHING_AFTER || state > *current) { // We know what transition we failed on, so we can provide a better error here. pm_parser_err_token(parser, token, PM_ERR_PARAMETER_ORDER); - } else if (state < *current) { - *current = state; + return false; } + + if (state < *current) *current = state; + return true; } /** @@ -13841,27 +14325,22 @@ parse_parameters( pm_parser_err_current(parser, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES); } - if (order > PM_PARAMETERS_ORDER_NOTHING_AFTER) { - update_parameter_state(parser, &parser->current, &order); - parser_lex(parser); + bool succeeded = update_parameter_state(parser, &parser->current, &order); + parser_lex(parser); - parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL; + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL; + pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous); - pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous); - if (params->keyword_rest != NULL) { - // If we already have a keyword rest parameter, then we replace it with the - // forwarding parameter and move the keyword rest parameter to the posts list. - pm_node_t *keyword_rest = params->keyword_rest; - pm_parameters_node_posts_append(params, keyword_rest); - pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_FWD); - params->keyword_rest = NULL; - } - pm_parameters_node_keyword_rest_set(params, (pm_node_t *)param); - } else { - update_parameter_state(parser, &parser->current, &order); - parser_lex(parser); + if (params->keyword_rest != NULL) { + // If we already have a keyword rest parameter, then we replace it with the + // forwarding parameter and move the keyword rest parameter to the posts list. + pm_node_t *keyword_rest = params->keyword_rest; + pm_parameters_node_posts_append(params, keyword_rest); + if (succeeded) pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_FWD); + params->keyword_rest = NULL; } + pm_parameters_node_keyword_rest_set(params, (pm_node_t *) param); break; } case PM_TOKEN_CLASS_VARIABLE: @@ -13956,6 +14435,12 @@ parse_parameters( pm_token_t local = name; local.end -= 1; + if (parser->encoding_changed ? parser->encoding->isupper_char(local.start, local.end - local.start) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start)) { + pm_parser_err(parser, local.start, local.end, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + } else if (local.end[-1] == '!' || local.end[-1] == '?') { + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); + } + bool repeated = pm_parser_parameter_name_check(parser, &local); pm_parser_local_add_token(parser, &local, 1); @@ -14031,6 +14516,7 @@ parse_parameters( pm_token_t operator = parser->previous; pm_token_t name; bool repeated = false; + if (accept1(parser, PM_TOKEN_IDENTIFIER)) { name = parser->previous; repeated = pm_parser_parameter_name_check(parser, &name); @@ -14044,6 +14530,7 @@ parse_parameters( if (repeated) { pm_node_flag_set_repeated_parameter(param); } + if (params->rest == NULL) { pm_parameters_node_rest_set(params, param); } else { @@ -14055,6 +14542,7 @@ parse_parameters( } case PM_TOKEN_STAR_STAR: case PM_TOKEN_USTAR_STAR: { + pm_parameters_order_t previous_order = order; update_parameter_state(parser, &parser->current, &order); parser_lex(parser); @@ -14062,6 +14550,10 @@ parse_parameters( pm_node_t *param; if (accept1(parser, PM_TOKEN_KEYWORD_NIL)) { + if (previous_order <= PM_PARAMETERS_ORDER_KEYWORDS) { + pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_NO_KW); + } + param = (pm_node_t *) pm_no_keywords_parameter_node_create(parser, &operator, &parser->previous); } else { pm_token_t name; @@ -14159,7 +14651,7 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, pm_rescues_type pm_rescue_node_operator_set(rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, PM_ERR_RESCUE_VARIABLE); - reference = parse_target(parser, reference); + reference = parse_target(parser, reference, false); pm_rescue_node_reference_set(rescue, reference); break; @@ -14189,7 +14681,7 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, pm_rescues_type pm_rescue_node_operator_set(rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, PM_ERR_RESCUE_VARIABLE); - reference = parse_target(parser, reference); + reference = parse_target(parser, reference, false); pm_rescue_node_reference_set(rescue, reference); break; @@ -14395,37 +14887,107 @@ parse_block_parameters( } /** + * Return true if any of the visible scopes to the current context are using + * numbered parameters. + */ +static bool +outer_scope_using_numbered_parameters_p(pm_parser_t *parser) { + for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { + if (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) return true; + } + + return false; +} + +/** + * These are the names of the various numbered parameters. We have them here so + * that when we insert them into the constant pool we can use a constant string + * and not have to allocate. + */ +static const char * const pm_numbered_parameter_names[] = { + "_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9" +}; + +/** * Return the node that should be used in the parameters field of a block-like * (block or lambda) node, depending on the kind of parameters that were * declared in the current scope. */ static pm_node_t * parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_token_t *opening, const pm_token_t *closing) { - uint8_t masked = parser->current_scope->parameters & PM_SCOPE_PARAMETERS_TYPE_MASK; + pm_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters; + + // If we have ordinary parameters, then we will return them as the set of + // parameters. + if (parameters != NULL) { + // If we also have implicit parameters, then this is an error. + if (implicit_parameters->size > 0) { + pm_node_t *node = implicit_parameters->nodes[0]; + + if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_ORDINARY); + } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_ORDINARY); + } else { + assert(false && "unreachable"); + } + } - if (masked == PM_SCOPE_PARAMETERS_NONE) { - assert(parameters == NULL); - return NULL; - } else if (masked == PM_SCOPE_PARAMETERS_ORDINARY) { - assert(parameters != NULL); return parameters; - } else if (masked == PM_SCOPE_PARAMETERS_NUMBERED) { - assert(parameters == NULL); + } + + // If we don't have any implicit parameters, then the set of parameters is + // NULL. + if (implicit_parameters->size == 0) { + return NULL; + } + + // If we don't have ordinary parameters, then we now must validate our set + // of implicit parameters. We can only have numbered parameters or it, but + // they cannot be mixed. + uint8_t numbered_parameter = 0; + bool it_parameter = false; + + for (size_t index = 0; index < implicit_parameters->size; index++) { + pm_node_t *node = implicit_parameters->nodes[index]; + + if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) { + if (it_parameter) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_IT); + } else if (outer_scope_using_numbered_parameters_p(parser)) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK); + } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) { + pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK); + } else if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + numbered_parameter = MAX(numbered_parameter, (uint8_t) (node->location.start[1] - '0')); + } else { + assert(false && "unreachable"); + } + } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + if (numbered_parameter > 0) { + pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_NUMBERED); + } else { + it_parameter = true; + } + } + } - int8_t maximum = parser->current_scope->numbered_parameters; - if (maximum > 0) { - const pm_location_t location = { .start = opening->start, .end = closing->end }; - return (pm_node_t *) pm_numbered_parameters_node_create(parser, &location, (uint8_t) maximum); + if (numbered_parameter > 0) { + // Go through the parent scopes and mark them as being disallowed from + // using numbered parameters because this inner scope is using them. + for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { + scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_INNER; } - return NULL; - } else if (masked == PM_SCOPE_PARAMETERS_IT) { - assert(parameters == NULL); + const pm_location_t location = { .start = opening->start, .end = closing->end }; + return (pm_node_t *) pm_numbered_parameters_node_create(parser, &location, numbered_parameter); + } + + if (it_parameter) { return (pm_node_t *) pm_it_parameters_node_create(parser, opening, closing); - } else { - assert(false && "unreachable"); - return NULL; } + + return NULL; } /** @@ -14442,9 +15004,6 @@ parse_block(pm_parser_t *parser) { pm_block_parameters_node_t *block_parameters = NULL; if (accept1(parser, PM_TOKEN_PIPE)) { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_token_t block_parameters_opening = parser->previous; if (match1(parser, PM_TOKEN_PIPE)) { block_parameters = pm_block_parameters_node_create(parser, NULL, &block_parameters_opening); @@ -14513,7 +15072,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); } else { pm_accepts_block_stack_push(parser, true); - parse_arguments(parser, arguments, true, PM_TOKEN_PARENTHESIS_RIGHT); + parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); @@ -14531,7 +15090,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept // If we get here, then the subsequent token cannot be used as an infix // operator. In this case we assume the subsequent token is part of an // argument to this method call. - parse_arguments(parser, arguments, true, PM_TOKEN_EOF); + parse_arguments(parser, arguments, accepts_block, PM_TOKEN_EOF); // If we have done with the arguments and still not consumed the comma, // then we have a trailing comma where we need to check whether it is @@ -14562,11 +15121,8 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (arguments->block == NULL && !arguments->has_forwarding) { arguments->block = (pm_node_t *) block; } else { - if (arguments->has_forwarding) { - pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_FORWARDING); - } else { - pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_MULTI); - } + pm_parser_err_node(parser, (pm_node_t *) block, PM_ERR_ARGUMENT_BLOCK_MULTI); + if (arguments->block != NULL) { if (arguments->arguments == NULL) { arguments->arguments = pm_arguments_node_create(parser); @@ -14945,7 +15501,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context) { #define PM_CASE_WRITABLE PM_CLASS_VARIABLE_READ_NODE: case PM_CONSTANT_PATH_NODE: \ case PM_CONSTANT_READ_NODE: case PM_GLOBAL_VARIABLE_READ_NODE: case PM_LOCAL_VARIABLE_READ_NODE: \ case PM_INSTANCE_VARIABLE_READ_NODE: case PM_MULTI_TARGET_NODE: case PM_BACK_REFERENCE_READ_NODE: \ - case PM_NUMBERED_REFERENCE_READ_NODE + case PM_NUMBERED_REFERENCE_READ_NODE: case PM_IT_LOCAL_VARIABLE_READ_NODE // Assert here that the flags are the same so that we can safely switch the type // of the node without having to move the flags. @@ -15003,6 +15559,10 @@ parse_string_part(pm_parser_t *parser) { // "aaa #{bbb} #@ccc ddd" // ^^^^^^ case PM_TOKEN_EMBEXPR_BEGIN: { + // Ruby disallows seeing encoding around interpolation in strings, + // even though it is known at parse time. + parser->explicit_encoding = NULL; + pm_lex_state_t state = parser->lex_state; int brace_nesting = parser->brace_nesting; @@ -15025,6 +15585,13 @@ parse_string_part(pm_parser_t *parser) { expect1(parser, PM_TOKEN_EMBEXPR_END, PM_ERR_EMBEXPR_END); pm_token_t closing = parser->previous; + // If this set of embedded statements only contains a single + // statement, then Ruby does not consider it as a possible statement + // that could emit a line event. + if (statements != NULL && statements->body.size == 1) { + pm_node_flag_unset(statements->body.nodes[0], PM_NODE_FLAG_NEWLINE); + } + return (pm_node_t *) pm_embedded_statements_node_create(parser, &opening, statements, &closing); } @@ -15035,6 +15602,10 @@ parse_string_part(pm_parser_t *parser) { // "aaa #{bbb} #@ccc ddd" // ^^^^^ case PM_TOKEN_EMBVAR: { + // Ruby disallows seeing encoding around interpolation in strings, + // even though it is known at parse time. + parser->explicit_encoding = NULL; + lex_state_set(parser, PM_LEX_STATE_BEG); parser_lex(parser); @@ -15158,7 +15729,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); - pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &symbol->unescaped)); + pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); return (pm_node_t *) symbol; } @@ -15258,7 +15829,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC); } - return (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &unescaped)); + return (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, false)); } /** @@ -15283,7 +15854,7 @@ parse_undef_argument(pm_parser_t *parser) { pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); - pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &symbol->unescaped)); + pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); return (pm_node_t *) symbol; } @@ -15324,7 +15895,7 @@ parse_alias_argument(pm_parser_t *parser, bool first) { pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); - pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &symbol->unescaped)); + pm_node_flag_set((pm_node_t *) symbol, parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); return (pm_node_t *) symbol; } @@ -15350,74 +15921,43 @@ parse_alias_argument(pm_parser_t *parser, bool first) { } /** - * Return true if any of the visible scopes to the current context are using - * numbered parameters. - */ -static bool -outer_scope_using_numbered_parameters_p(pm_parser_t *parser) { - for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { - if (scope->numbered_parameters > 0) return true; - } - - return false; -} - -/** - * These are the names of the various numbered parameters. We have them here so - * that when we insert them into the constant pool we can use a constant string - * and not have to allocate. - */ -static const char * const pm_numbered_parameter_names[] = { - "_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9" -}; - -/** * Parse an identifier into either a local variable read. If the local variable * is not found, it returns NULL instead. */ -static pm_local_variable_read_node_t * +static pm_node_t * parse_variable(pm_parser_t *parser) { + pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; - if ((depth = pm_parser_local_depth(parser, &parser->previous)) != -1) { - return pm_local_variable_read_node_create(parser, &parser->previous, (uint32_t) depth); + + if ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1) { + return (pm_node_t *) pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false); } pm_scope_t *current_scope = parser->current_scope; - if (!current_scope->closed && current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED && pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end)) { - // Now that we know we have a numbered parameter, we need to check - // if it's allowed in this context. If it is, then we will create a - // local variable read. If it's not, then we'll create a normal call - // node but add an error. - if (current_scope->parameters & PM_SCOPE_PARAMETERS_ORDINARY) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_ORDINARY); - } else if (current_scope->parameters & PM_SCOPE_PARAMETERS_IT) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_IT); - } else if (outer_scope_using_numbered_parameters_p(parser)) { - pm_parser_err_previous(parser, PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE); - } else { - // Indicate that this scope is using numbered params so that child - // scopes cannot. We subtract the value for the character '0' to get - // the actual integer value of the number (only _1 through _9 are - // valid). - int8_t numbered_parameters = (int8_t) (parser->previous.start[1] - '0'); - current_scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED; - - if (numbered_parameters > current_scope->numbered_parameters) { - current_scope->numbered_parameters = numbered_parameters; + if (!current_scope->closed && !(current_scope->parameters & PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED)) { + if (pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end)) { + // When you use a numbered parameter, it implies the existence of + // all of the locals that exist before it. For example, referencing + // _2 means that _1 must exist. Therefore here we loop through all + // of the possibilities and add them into the constant pool. + uint8_t maximum = (uint8_t) (parser->previous.start[1] - '0'); + for (uint8_t number = 1; number <= maximum; number++) { + pm_parser_local_add_constant(parser, pm_numbered_parameter_names[number - 1], 2); } - // When you use a numbered parameter, it implies the existence - // of all of the locals that exist before it. For example, - // referencing _2 means that _1 must exist. Therefore here we - // loop through all of the possibilities and add them into the - // constant pool. - for (int8_t numbered_param = 1; numbered_param <= numbered_parameters - 1; numbered_param++) { - pm_parser_local_add_constant(parser, pm_numbered_parameter_names[numbered_param - 1], 2); + if (!match1(parser, PM_TOKEN_EQUAL)) { + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_FOUND; } - // Finally we can create the local variable read node. - pm_constant_id_t name_id = pm_parser_local_add_constant(parser, pm_numbered_parameter_names[numbered_parameters - 1], 2); - return pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, 0, false); + pm_node_t *node = (pm_node_t *) pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, 0, false); + pm_node_list_append(¤t_scope->implicit_parameters, node); + + return node; + } else if ((parser->version != PM_OPTIONS_VERSION_CRUBY_3_3) && pm_token_is_it(parser->previous.start, parser->previous.end)) { + pm_node_t *node = (pm_node_t *) pm_it_local_variable_read_node_create(parser, &parser->previous); + pm_node_list_append(¤t_scope->implicit_parameters, node); + + return node; } } @@ -15432,8 +15972,8 @@ parse_variable_call(pm_parser_t *parser) { pm_node_flags_t flags = 0; if (!match1(parser, PM_TOKEN_PARENTHESIS_LEFT) && (parser->previous.end[-1] != '!') && (parser->previous.end[-1] != '?')) { - pm_local_variable_read_node_t *node = parse_variable(parser); - if (node != NULL) return (pm_node_t *) node; + pm_node_t *node = parse_variable(parser); + if (node != NULL) return node; flags |= PM_CALL_NODE_FLAGS_VARIABLE_CALL; } @@ -15551,8 +16091,12 @@ parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_w nodes->size = write_index; } +#define PM_PARSE_PATTERN_SINGLE 0 +#define PM_PARSE_PATTERN_TOP 1 +#define PM_PARSE_PATTERN_MULTI 2 + static pm_node_t * -parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id); +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id); /** * Add the newly created local to the list of captures for this pattern matching @@ -15581,9 +16125,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures while (accept1(parser, PM_TOKEN_COLON_COLON)) { pm_token_t delimiter = parser->previous; expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); - - pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - node = (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, child); + node = (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, &parser->previous); } // If there is a [ or ( that follows, then this is part of a larger pattern @@ -15602,7 +16144,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); } @@ -15614,7 +16156,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); } @@ -15763,20 +16305,51 @@ parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) } /** + * Check that the slice of the source given by the bounds parameters constitutes + * a valid local variable name. + */ +static bool +pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { + ptrdiff_t length = end - start; + if (length == 0) return false; + + // First ensure that it starts with a valid identifier starting character. + size_t width = char_is_identifier_start(parser, start); + if (width == 0) return false; + + // Next, ensure that it's not an uppercase character. + if (parser->encoding_changed) { + if (parser->encoding->isupper_char(start, length)) return false; + } else { + if (pm_encoding_utf_8_isupper_char(start, length)) return false; + } + + // Next, iterate through all of the bytes of the string to ensure that they + // are all valid identifier characters. + const uint8_t *cursor = start + width; + while ((cursor < end) && (width = char_is_identifier(parser, cursor))) cursor += width; + return cursor == end; +} + +/** * Create an implicit node for the value of a hash pattern that has omitted the * value. This will use an implicit local variable target. */ static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); int depth = -1; - if (value_loc->end[-1] == '!' || value_loc->end[-1] == '?') { - pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); - PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); - } else { + + if (pm_slice_is_valid_local(parser, value_loc->start, value_loc->end)) { depth = pm_parser_local_depth_constant_id(parser, constant_id); + } else { + pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); + + if ((value_loc->end > value_loc->start) && ((value_loc->end[-1] == '!') || (value_loc->end[-1] == '?'))) { + PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); + } } if (depth == -1) { @@ -15800,7 +16373,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca */ static void parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node) != NULL) { + if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node, true) != NULL) { pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); } } @@ -15831,7 +16404,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node } else { // Here we have a value for the first assoc in the list, so // we will parse it now. - value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -15846,7 +16419,8 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node // If we get anything else, then this is an error. For this we'll // create a missing node for the value and create an assoc node for // the first node in the list. - pm_parser_err_node(parser, first_node, PM_ERR_PATTERN_HASH_KEY_LABEL); + pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL; + pm_parser_err_node(parser, first_node, diag_id); pm_token_t operator = not_provided(parser); pm_node_t *value = (pm_node_t *) pm_missing_node_create(parser, first_node->location.start, first_node->location.end); @@ -15883,7 +16457,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -15940,7 +16514,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm // Otherwise, we'll parse the inner pattern, then deal with it depending // on the type it returns. - pm_node_t *inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + pm_node_t *inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); @@ -16007,11 +16581,11 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm first_node = parse_pattern_keyword_rest(parser, captures); break; case PM_TOKEN_STRING_BEGIN: - first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY); + first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY_LABEL); break; default: { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); parser_lex(parser); - pm_parser_err_previous(parser, PM_ERR_PATTERN_HASH_KEY); first_node = (pm_node_t *) pm_missing_node_create(parser, parser->previous.start, parser->previous.end); break; @@ -16087,19 +16661,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *variable = (pm_node_t *) parse_variable(parser); if (variable == NULL) { - if ( - (parser->version != PM_OPTIONS_VERSION_CRUBY_3_3_0) && - !parser->current_scope->closed && - (parser->current_scope->numbered_parameters != PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED) && - pm_token_is_it(parser->previous.start, parser->previous.end) - ) { - pm_local_variable_read_node_t *read = pm_local_variable_read_node_create_it(parser, &parser->previous); - if (read == NULL) read = pm_local_variable_read_node_create(parser, &parser->previous, 0); - variable = (pm_node_t *) read; - } else { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); - variable = (pm_node_t *) pm_local_variable_read_node_missing_create(parser, &parser->previous, 0); - } + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); + variable = (pm_node_t *) pm_local_variable_read_node_missing_create(parser, &parser->previous, 0); } return (pm_node_t *) pm_pinned_variable_node_create(parser, &operator, variable); @@ -16162,8 +16725,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm parser_lex(parser); expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); - pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - pm_constant_path_node_t *node = pm_constant_path_node_create(parser, NULL, &delimiter, child); + pm_constant_path_node_t *node = pm_constant_path_node_create(parser, NULL, &delimiter, &parser->previous); return parse_pattern_constant_path(parser, captures, (pm_node_t *) node); } @@ -16214,7 +16776,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p pm_token_t opening = parser->current; parser_lex(parser); - pm_node_t *body = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + pm_node_t *body = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); pm_node_t *right = (pm_node_t *) pm_parentheses_node_create(parser, &opening, body, &parser->previous); @@ -16273,7 +16835,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p * Parse a pattern matching expression. */ static pm_node_t * -parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id) { +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; bool leading_rest = false; @@ -16283,14 +16845,26 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pat case PM_TOKEN_LABEL: { parser_lex(parser); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); - return (pm_node_t *) parse_pattern_hash(parser, captures, key); + node = (pm_node_t *) parse_pattern_hash(parser, captures, key); + + if (!(flags & PM_PARSE_PATTERN_TOP)) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT); + } + + return node; } case PM_TOKEN_USTAR_STAR: { node = parse_pattern_keyword_rest(parser, captures); - return (pm_node_t *) parse_pattern_hash(parser, captures, node); + node = (pm_node_t *) parse_pattern_hash(parser, captures, node); + + if (!(flags & PM_PARSE_PATTERN_TOP)) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT); + } + + return node; } case PM_TOKEN_USTAR: { - if (top_pattern) { + if (flags & (PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI)) { parser_lex(parser); node = (pm_node_t *) parse_pattern_rest(parser, captures); leading_rest = true; @@ -16309,7 +16883,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pat return (pm_node_t *) parse_pattern_hash(parser, captures, node); } - if (top_pattern && match1(parser, PM_TOKEN_COMMA)) { + if ((flags & PM_PARSE_PATTERN_MULTI) && match1(parser, PM_TOKEN_COMMA)) { // If we have a comma, then we are now parsing either an array pattern or a // find pattern. We need to parse all of the patterns, put them into a big // list, and then determine which type of node we have. @@ -16383,10 +16957,12 @@ parse_negative_numeric(pm_node_t *node) { cast->value = -cast->value; break; } - case PM_RATIONAL_NODE: - node->location.start--; - parse_negative_numeric(((pm_rational_node_t *) node)->numeric); + case PM_RATIONAL_NODE: { + pm_rational_node_t *cast = (pm_rational_node_t *) node; + cast->base.location.start--; + cast->numerator.negative = true; break; + } case PM_IMAGINARY_NODE: node->location.start--; parse_negative_numeric(((pm_imaginary_node_t *) node)->numeric); @@ -16451,6 +17027,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { // start with a single string content node. pm_string_t unescaped; pm_token_t content; + if (match1(parser, PM_TOKEN_EOF)) { unescaped = PM_STRING_EMPTY; content = not_provided(parser); @@ -16489,7 +17066,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { pm_node_list_free(&parts); } else if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { - node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &unescaped)); + node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); } else if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_STRING_LITERAL_EOF); node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); @@ -16513,9 +17090,21 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { node = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped); pm_node_flag_set(node, parse_unescaped_encoding(parser)); - expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); + + // Kind of odd behavior, but basically if we have an + // unterminated string and it ends in a newline, we back up one + // character so that the error message is on the last line of + // content in the string. + if (!accept1(parser, PM_TOKEN_STRING_END)) { + const uint8_t *location = parser->previous.end; + if (location > parser->start && location[-1] == '\n') location--; + pm_parser_err(parser, location, location, PM_ERR_STRING_LITERAL_EOF); + + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + } } else if (accept1(parser, PM_TOKEN_LABEL_END)) { - node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &unescaped)); + node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true)); } else { // If we get here, then we have interpolation so we'll need // to create a string or symbol node with interpolation. @@ -16597,11 +17186,11 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { pm_token_t bounds = not_provided(parser); pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); - pm_interpolated_string_node_append(parser, container, current); + pm_interpolated_string_node_append(container, current); current = (pm_node_t *) container; } - pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, node); + pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, node); } } @@ -16620,6 +17209,11 @@ pm_parser_err_prefix(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, pm_token_type_human(parser->previous.type)); break; } + case PM_ERR_HASH_VALUE: + case PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR: { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + break; + } case PM_ERR_UNARY_RECEIVER: { const char *human = (parser->current.type == PM_TOKEN_EOF ? "end-of-input" : pm_token_type_human(parser->current.type)); PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, human, parser->previous.start[0]); @@ -16818,8 +17412,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } - if (pm_array_node_size(array) != 0) { - expect1(parser, PM_TOKEN_COMMA, PM_ERR_ARRAY_SEPARATOR); + // Ensure that we have a comma between elements in the array. + if ((pm_array_node_size(array) != 0) && !accept1(parser, PM_TOKEN_COMMA)) { + const uint8_t *location = parser->previous.end; + PM_PARSER_ERR_FORMAT(parser, location, location, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); + + parser->previous.start = location; + parser->previous.type = PM_TOKEN_MISSING; } // If we have a right bracket immediately following a comma, @@ -16976,7 +17575,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b return (pm_node_t *) multi_target; } - return parse_target_validate(parser, (pm_node_t *) multi_target); + return parse_target_validate(parser, (pm_node_t *) multi_target, false); } // If we have a single statement and are ending on a right parenthesis @@ -16997,7 +17596,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we didn't find a terminator and we didn't find a right // parenthesis, then this is a syntax error. - if (!terminator_found) { + if (!terminator_found && !match1(parser, PM_TOKEN_EOF)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } @@ -17026,7 +17625,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) break; } else if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { break; - } else { + } else if (!match1(parser, PM_TOKEN_EOF)) { + // If we're at the end of the file, then we're going to add + // an error after this for the ) anyway. PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } } @@ -17137,12 +17738,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } case PM_TOKEN_UCOLON_COLON: { parser_lex(parser); - pm_token_t delimiter = parser->previous; - expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); - pm_node_t *constant = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - pm_node_t *node = (pm_node_t *)pm_constant_path_node_create(parser, NULL, &delimiter, constant); + expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); + pm_node_t *node = (pm_node_t *) pm_constant_path_node_create(parser, NULL, &delimiter, &parser->previous); if ((binding_power == PM_BINDING_POWER_STATEMENT) && match1(parser, PM_TOKEN_COMMA)) { node = parse_targets_validate(parser, node, PM_BINDING_POWER_INDEX); @@ -17247,8 +17846,28 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b ) { pm_arguments_t arguments = { 0 }; parse_arguments_list(parser, &arguments, true, accepts_command_call); - pm_call_node_t *fcall = pm_call_node_fcall_create(parser, &identifier, &arguments); + + if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { + // If we're about to convert an 'it' implicit local + // variable read into a method call, we need to remove + // it from the list of implicit local variables. + parse_target_implicit_parameter(parser, node); + } else { + // Otherwise, we're about to convert a regular local + // variable read into a method call, in which case we + // need to indicate that this was not a read for the + // purposes of warnings. + assert(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)); + + if (pm_token_is_numbered_parameter(identifier.start, identifier.end)) { + parse_target_implicit_parameter(parser, node); + } else { + pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node; + pm_locals_unread(&pm_parser_scope_find(parser, cast->depth)->locals, cast->name); + } + } + pm_node_destroy(parser, node); return (pm_node_t *) fcall; } @@ -17256,31 +17875,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if ((binding_power == PM_BINDING_POWER_STATEMENT) && match1(parser, PM_TOKEN_COMMA)) { node = parse_targets_validate(parser, node, PM_BINDING_POWER_INDEX); - } else { - // Check if `it` is not going to be assigned. - switch (parser->current.type) { - case PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL: - case PM_TOKEN_AMPERSAND_EQUAL: - case PM_TOKEN_CARET_EQUAL: - case PM_TOKEN_EQUAL: - case PM_TOKEN_GREATER_GREATER_EQUAL: - case PM_TOKEN_LESS_LESS_EQUAL: - case PM_TOKEN_MINUS_EQUAL: - case PM_TOKEN_PARENTHESIS_RIGHT: - case PM_TOKEN_PERCENT_EQUAL: - case PM_TOKEN_PIPE_EQUAL: - case PM_TOKEN_PIPE_PIPE_EQUAL: - case PM_TOKEN_PLUS_EQUAL: - case PM_TOKEN_SLASH_EQUAL: - case PM_TOKEN_STAR_EQUAL: - case PM_TOKEN_STAR_STAR_EQUAL: - break; - default: - // Once we know it's neither a method call nor an - // assignment, we can finally create `it` default - // parameter. - node = pm_node_check_it(parser, node); - } } return node; @@ -17302,8 +17896,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) { // If we get here, then we have an empty heredoc. We'll create // an empty content token and return an empty string node. - lex_mode_pop(parser); - expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); + expect1_heredoc_term(parser, lex_mode); pm_token_t content = parse_strings_empty_content(parser->previous.start); if (quote == PM_HEREDOC_QUOTE_BACKTICK) { @@ -17344,8 +17937,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } node = (pm_node_t *) cast; - lex_mode_pop(parser); - expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); + expect1_heredoc_term(parser, lex_mode); } else { // If we get here, then we have multiple parts in the heredoc, // so we'll need to create an interpolated string node to hold @@ -17367,20 +17959,18 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_interpolated_x_string_node_t *cast = pm_interpolated_xstring_node_create(parser, &opening, &opening); cast->parts = parts; - lex_mode_pop(parser); - expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); - + expect1_heredoc_term(parser, lex_mode); pm_interpolated_xstring_node_closing_set(cast, &parser->previous); + cast->base.location = cast->opening_loc; node = (pm_node_t *) cast; } else { pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &opening); pm_node_list_free(&parts); - lex_mode_pop(parser); - expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); - + expect1_heredoc_term(parser, lex_mode); pm_interpolated_string_node_closing_set(cast, &parser->previous); + cast->base.location = cast->opening_loc; node = (pm_node_t *) cast; } @@ -17545,6 +18135,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // as frozen because when clause strings are frozen. if (PM_NODE_TYPE_P(condition, PM_STRING_NODE)) { pm_node_flag_set(condition, PM_STRING_FLAGS_FROZEN | PM_NODE_FLAG_STATIC_LITERAL); + } else if (PM_NODE_TYPE_P(condition, PM_SOURCE_FILE_NODE)) { + pm_node_flag_set(condition, PM_NODE_FLAG_STATIC_LITERAL); } pm_when_clause_static_literals_add(parser, &literals, condition); @@ -17601,7 +18193,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t in_keyword = parser->previous; pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -17630,7 +18222,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b then_keyword = not_provided(parser); } } else { - expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_WHEN_DELIMITER); + expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_IN_DELIMITER); then_keyword = parser->previous; } @@ -17950,7 +18542,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match2(parser, PM_TOKEN_DOT, PM_TOKEN_COLON_COLON)) { receiver = parse_variable_call(parser); - receiver = pm_node_check_it(parser, receiver); pm_parser_scope_push(parser, true); lex_state_set(parser, PM_LEX_STATE_FNAME); @@ -18084,7 +18675,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; - expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_DEF_PARAMS_TERM_PAREN); + if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); + parser->previous.start = parser->previous.end; + parser->previous.type = PM_TOKEN_MISSING; + } + rparen = parser->previous; break; } @@ -18282,7 +18878,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_COMMA)) { index = parse_targets(parser, index, PM_BINDING_POWER_INDEX); } else { - index = parse_target(parser, index); + index = parse_target(parser, index, false); } context_pop(parser); @@ -18404,9 +19000,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t double_colon = parser->previous; expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); - pm_node_t *constant = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - - constant_path = (pm_node_t *) pm_constant_path_node_create(parser, constant_path, &double_colon, constant); + constant_path = (pm_node_t *) pm_constant_path_node_create(parser, constant_path, &double_colon, &parser->previous); } // Here we retrieve the name of the module. If it wasn't a constant, @@ -18786,15 +19380,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit string content and the current node is // an interpolated string, then we need to append // the string content to the list of child nodes. - pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, string); + pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, string); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit string content and the current node is // a string node, then we need to convert the // current node into an interpolated string and add // the string content to the list of child nodes. pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(parser, interpolated, current); - pm_interpolated_string_node_append(parser, interpolated, string); + pm_interpolated_string_node_append(interpolated, current); + pm_interpolated_string_node_append(interpolated, string); current = (pm_node_t *) interpolated; } else { assert(false && "unreachable"); @@ -18819,7 +19413,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(parser, interpolated, current); + pm_interpolated_string_node_append(interpolated, current); current = (pm_node_t *) interpolated; } else { // If we hit an embedded variable and the current @@ -18828,7 +19422,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); break; } case PM_TOKEN_EMBEXPR_BEGIN: { @@ -18848,7 +19442,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(parser, interpolated, current); + pm_interpolated_string_node_append(interpolated, current); current = (pm_node_t *) interpolated; } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { // If we hit an embedded expression and the current @@ -18859,7 +19453,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); break; } default: @@ -18935,6 +19529,14 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &opening, &parser->previous, &closing, &unescaped); + + if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { + // This is extremely strange, but the first string part of a + // regular expression will always be tagged as binary if we + // are in a US-ASCII file, no matter its contents. + pm_node_flag_set(part, PM_STRING_FLAGS_FORCED_BINARY_ENCODING); + } + pm_interpolated_regular_expression_node_append(interpolated, part); } else { // If the first part of the body of the regular expression is not a @@ -19063,7 +19665,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_COMMA)) { return parse_targets_validate(parser, splat, PM_BINDING_POWER_INDEX); } else { - return parse_target_validate(parser, splat); + return parse_target_validate(parser, splat, true); } } case PM_TOKEN_BANG: { @@ -19135,9 +19737,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b switch (parser->current.type) { case PM_TOKEN_PARENTHESIS_LEFT: { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_token_t opening = parser->current; parser_lex(parser); @@ -19154,9 +19753,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_CASE_PARAMETER: { - assert(parser->current_scope->parameters == PM_SCOPE_PARAMETERS_NONE); - parser->current_scope->parameters = PM_SCOPE_PARAMETERS_ORDINARY; - pm_accepts_block_stack_push(parser, false); pm_token_t opening = not_provided(parser); block_parameters = parse_block_parameters(parser, false, &opening, true); @@ -19409,39 +20005,6 @@ parse_call_operator_write(pm_parser_t *parser, pm_call_node_t *call_node, const } /** - * Returns true if the name of the capture group is a valid local variable that - * can be written to. - */ -static bool -parse_regular_expression_named_capture(pm_parser_t *parser, const uint8_t *source, size_t length) { - if (length == 0) { - return false; - } - - // First ensure that it starts with a valid identifier starting character. - size_t width = char_is_identifier_start(parser, source); - if (!width) { - return false; - } - - // Next, ensure that it's not an uppercase character. - if (parser->encoding_changed) { - if (parser->encoding->isupper_char(source, (ptrdiff_t) length)) return false; - } else { - if (pm_encoding_utf_8_isupper_char(source, (ptrdiff_t) length)) return false; - } - - // Next, iterate through all of the bytes of the string to ensure that they - // are all valid identifier characters. - const uint8_t *cursor = source + width; - while (cursor < source + length && (width = char_is_identifier(parser, cursor))) { - cursor += width; - } - - return cursor == source + length; -} - -/** * Potentially change a =~ with a regular expression with named captures into a * match write node. */ @@ -19467,7 +20030,7 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * // If the name of the capture group isn't a valid identifier, we do // not add it to the local table. - if (!parse_regular_expression_named_capture(parser, source, length)) continue; + if (!pm_slice_is_valid_local(parser, source, source + length)) continue; if (content->type == PM_STRING_SHARED) { // If the unescaped string is a slice of the source, then we can @@ -19641,7 +20204,6 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_CALL_NODE: { - parser_lex(parser); pm_call_node_t *cast = (pm_call_node_t *) node; // If we have a vcall (a method with no arguments and no @@ -19652,6 +20214,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + parser_lex(parser); + pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ); pm_node_t *result = (pm_node_t *) pm_local_variable_and_write_node_create(parser, (pm_node_t *) cast, &token, value, constant_id, 0); @@ -19659,6 +20223,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } + // Move past the token here so that we have already added + // the local variable by this point. + parser_lex(parser); + // If there is no call operator and the message is "[]" then // this is an aref expression, and we can transform it into // an aset expression. @@ -19754,7 +20322,6 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_CALL_NODE: { - parser_lex(parser); pm_call_node_t *cast = (pm_call_node_t *) node; // If we have a vcall (a method with no arguments and no @@ -19765,6 +20332,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + parser_lex(parser); + pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ); pm_node_t *result = (pm_node_t *) pm_local_variable_or_write_node_create(parser, (pm_node_t *) cast, &token, value, constant_id, 0); @@ -19772,6 +20341,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } + // Move past the token here so that we have already added + // the local variable by this point. + parser_lex(parser); + // If there is no call operator and the message is "[]" then // this is an aref expression, and we can transform it into // an aset expression. @@ -19925,7 +20498,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // In this case we have an operator but we don't know what it's for. // We need to treat it as an error. For now, we'll mark it as an error // and just skip right past it. - pm_parser_err_previous(parser, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR); + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); return node; } } @@ -20181,7 +20754,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if ( (parser->current.type == PM_TOKEN_PARENTHESIS_LEFT) || - (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR)) + (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR))) ) { // If we have a constant immediately following a '::' operator, then // this can either be a constant path or a method call, depending on @@ -20196,8 +20769,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t path = (pm_node_t *) pm_call_node_call_create(parser, node, &delimiter, &message, &arguments); } else { // Otherwise, this is a constant path. That would look like Foo::Bar. - pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - path = (pm_node_t *)pm_constant_path_node_create(parser, node, &delimiter, child); + path = (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, &parser->previous); } // If this is followed by a comma then it is a multiple assignment. @@ -20236,9 +20808,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return (pm_node_t *) pm_call_node_shorthand_create(parser, node, &delimiter, &arguments); } default: { - pm_parser_err_token(parser, &delimiter, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); - pm_node_t *child = (pm_node_t *) pm_missing_node_create(parser, delimiter.start, delimiter.end); - return (pm_node_t *)pm_constant_path_node_create(parser, node, &delimiter, child); + expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); + return (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, &parser->previous); } } } @@ -20309,7 +20880,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -20326,7 +20897,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_constant_id_list_t captures = { 0 }; - pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); parser->pattern_matching_newlines = previous_pattern_matching_newlines; pm_constant_id_list_free(&captures); @@ -20339,6 +20910,10 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t } } +#undef PM_PARSE_PATTERN_SINGLE +#undef PM_PARSE_PATTERN_TOP +#undef PM_PARSE_PATTERN_MULTI + /** * Parse an expression at the given point of the parser using the given binding * power to parse subsequent chains. If this function finds a syntax error, it @@ -20722,7 +21297,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // Scopes given from the outside are not allowed to have numbered // parameters. - parser->current_scope->numbered_parameters = PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED; + parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED; for (size_t local_index = 0; local_index < scope->locals_count; local_index++) { const pm_string_t *local = pm_options_scope_local_get(scope, local_index); @@ -21110,328 +21685,3 @@ pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t s } #endif - -/** An error that is going to be formatted into the output. */ -typedef struct { - /** A pointer to the diagnostic that was generated during parsing. */ - pm_diagnostic_t *error; - - /** The start line of the diagnostic message. */ - int32_t line; - - /** The column start of the diagnostic message. */ - uint32_t column_start; - - /** The column end of the diagnostic message. */ - uint32_t column_end; -} pm_error_t; - -/** The format that will be used to format the errors into the output. */ -typedef struct { - /** The prefix that will be used for line numbers. */ - const char *number_prefix; - - /** The prefix that will be used for blank lines. */ - const char *blank_prefix; - - /** The divider that will be used between sections of source code. */ - const char *divider; - - /** The length of the blank prefix. */ - size_t blank_prefix_length; - - /** The length of the divider. */ - size_t divider_length; -} pm_error_format_t; - -#define PM_COLOR_GRAY "\033[38;5;102m" -#define PM_COLOR_RED "\033[1;31m" -#define PM_COLOR_RESET "\033[m" - -static inline pm_error_t * -pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { - pm_error_t *errors = xcalloc(error_list->size, sizeof(pm_error_t)); - if (errors == NULL) return NULL; - - int32_t start_line = parser->start_line; - for (pm_diagnostic_t *error = (pm_diagnostic_t *) error_list->head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { - pm_line_column_t start = pm_newline_list_line_column(newline_list, error->location.start, start_line); - pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.end, start_line); - - // We're going to insert this error into the array in sorted order. We - // do this by finding the first error that has a line number greater - // than the current error and then inserting the current error before - // that one. - size_t index = 0; - while ( - (index < error_list->size) && - (errors[index].error != NULL) && - ( - (errors[index].line < start.line) || - ((errors[index].line == start.line) && (errors[index].column_start < start.column)) - ) - ) index++; - - // Now we're going to shift all of the errors after this one down one - // index to make room for the new error. - if (index + 1 < error_list->size) { - memmove(&errors[index + 1], &errors[index], sizeof(pm_error_t) * (error_list->size - index - 1)); - } - - // Finally, we'll insert the error into the array. - uint32_t column_end; - if (start.line == end.line) { - column_end = end.column; - } else { - column_end = (uint32_t) (newline_list->offsets[start.line - start_line + 1] - newline_list->offsets[start.line - start_line] - 1); - } - - // Ensure we have at least one column of error. - if (start.column == column_end) column_end++; - - errors[index] = (pm_error_t) { - .error = error, - .line = start.line, - .column_start = start.column, - .column_end = column_end - }; - } - - return errors; -} - -static inline void -pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, pm_buffer_t *buffer) { - int32_t line_delta = line - parser->start_line; - assert(line_delta >= 0); - - size_t index = (size_t) line_delta; - assert(index < newline_list->size); - - const uint8_t *start = &parser->start[newline_list->offsets[index]]; - const uint8_t *end; - - if (index >= newline_list->size - 1) { - end = parser->end; - } else { - end = &parser->start[newline_list->offsets[index + 1]]; - } - - pm_buffer_append_format(buffer, number_prefix, line); - pm_buffer_append_string(buffer, (const char *) start, (size_t) (end - start)); - - if (end == parser->end && end[-1] != '\n') { - pm_buffer_append_string(buffer, "\n", 1); - } -} - -/** - * Format the errors on the parser into the given buffer. - */ -PRISM_EXPORTED_FUNCTION void -pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { - assert(error_list->size != 0); - - // First, we're going to sort all of the errors by line number using an - // insertion sort into a newly allocated array. - const int32_t start_line = parser->start_line; - const pm_newline_list_t *newline_list = &parser->newline_list; - - pm_error_t *errors = pm_parser_errors_format_sort(parser, error_list, newline_list); - if (errors == NULL) return; - - // Now we're going to determine how we're going to format line numbers and - // blank lines based on the maximum number of digits in the line numbers - // that are going to be displaid. - pm_error_format_t error_format; - int32_t first_line_number = errors[0].line; - int32_t last_line_number = errors[error_list->size - 1].line; - - // If we have a maximum line number that is negative, then we're going to - // use the absolute value for comparison but multiple by 10 to additionally - // have a column for the negative sign. - if (first_line_number < 0) first_line_number = (-first_line_number) * 10; - if (last_line_number < 0) last_line_number = (-last_line_number) * 10; - int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; - - if (max_line_number < 10) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%1" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%1" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~\n" - }; - } - } else if (max_line_number < 100) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%2" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%2" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~\n" - }; - } - } else if (max_line_number < 1000) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%3" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%3" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~\n" - }; - } - } else if (max_line_number < 10000) { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%4" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%4" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~~\n" - }; - } - } else { - if (colorize) { - error_format = (pm_error_format_t) { - .number_prefix = PM_COLOR_GRAY "%5" PRIi32 " | " PM_COLOR_RESET, - .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, - .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" - }; - } else { - error_format = (pm_error_format_t) { - .number_prefix = "%5" PRIi32 " | ", - .blank_prefix = " | ", - .divider = " ~~~~~~~~\n" - }; - } - } - - error_format.blank_prefix_length = strlen(error_format.blank_prefix); - error_format.divider_length = strlen(error_format.divider); - - // Now we're going to iterate through every error in our error list and - // display it. While we're iterating, we will display some padding lines of - // the source before the error to give some context. We'll be careful not to - // display the same line twice in case the errors are close enough in the - // source. - int32_t last_line = parser->start_line - 1; - const pm_encoding_t *encoding = parser->encoding; - - for (size_t index = 0; index < error_list->size; index++) { - pm_error_t *error = &errors[index]; - - // Here we determine how many lines of padding of the source to display, - // based on the difference from the last line that was displaid. - if (error->line - last_line > 1) { - if (error->line - last_line > 2) { - if ((index != 0) && (error->line - last_line > 3)) { - pm_buffer_append_string(buffer, error_format.divider, error_format.divider_length); - } - - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 2, buffer); - } - - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 1, buffer); - } - - // If this is the first error or we're on a new line, then we'll display - // the line that has the error in it. - if ((index == 0) || (error->line != last_line)) { - if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); - } else { - pm_buffer_append_string(buffer, "> ", 2); - } - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, buffer); - } - - const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; - if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); - - // Now we'll display the actual error message. We'll do this by first - // putting the prefix to the line, then a bunch of blank spaces - // depending on the column, then as many carets as we need to display - // the width of the error, then the error message itself. - // - // Note that this doesn't take into account the width of the actual - // character when displaid in the terminal. For some east-asian - // languages or emoji, this means it can be thrown off pretty badly. We - // will need to solve this eventually. - pm_buffer_append_string(buffer, " ", 2); - pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); - - size_t column = 0; - while (column < error->column_end) { - if (column < error->column_start) { - pm_buffer_append_byte(buffer, ' '); - } else { - const uint8_t caret = column == error->column_start ? '^' : '~'; - - if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED, 7); - pm_buffer_append_byte(buffer, caret); - pm_buffer_append_string(buffer, PM_COLOR_RESET, 3); - } else { - pm_buffer_append_byte(buffer, caret); - } - } - - size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); - column += (char_width == 0 ? 1 : char_width); - } - - if (inline_messages) { - pm_buffer_append_byte(buffer, ' '); - assert(error->error != NULL); - - const char *message = error->error->message; - pm_buffer_append_string(buffer, message, strlen(message)); - } - - pm_buffer_append_byte(buffer, '\n'); - - // Here we determine how many lines of padding to display after the - // error, depending on where the next error is in source. - last_line = error->line; - int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; - - if (next_line - last_line > 1) { - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, buffer); - } - - if (next_line - last_line > 1) { - pm_buffer_append_string(buffer, " ", 2); - pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, buffer); - } - } - - // Finally, we'll free the array of errors that we allocated. - xfree(errors); -} - -#undef PM_COLOR_GRAY -#undef PM_COLOR_RED -#undef PM_COLOR_RESET diff --git a/prism/prism.h b/prism/prism.h index 59067c3021..755c38fca2 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -219,17 +219,6 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_ty */ const char * pm_token_type_human(pm_token_type_t token_type); -/** - * Format the errors on the parser into the given buffer. - * - * @param parser The parser to format the errors for. - * @param error_list The list of errors to format. - * @param buffer The buffer to format the errors into. - * @param colorize Whether or not to colorize the errors with ANSI escape sequences. - * @param inline_messages Whether or not to inline the messages with the source. - */ -PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages); - // We optionally support dumping to JSON. For systems that don't want or need // this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define. #ifndef PRISM_EXCLUDE_JSON diff --git a/prism/static_literals.c b/prism/static_literals.c index 08f61c81fa..b8604321c9 100644 --- a/prism/static_literals.c +++ b/prism/static_literals.c @@ -59,6 +59,25 @@ murmur_hash(const uint8_t *key, size_t length) { } /** + * Hash the value of an integer and return it. + */ +static uint32_t +integer_hash(const pm_integer_t *integer) { + uint32_t hash; + if (integer->values) { + hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length); + } else { + hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t)); + } + + if (integer->negative) { + hash ^= murmur_scramble((uint32_t) 1); + } + + return hash; +} + +/** * Return the hash of the given node. It is important that nodes that have * equivalent static literal values have the same hash. This is because we use * these hashes to look for duplicates. @@ -68,19 +87,8 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: { // Integers hash their value. - const pm_integer_t *integer = &((const pm_integer_node_t *) node)->value; - uint32_t hash; - if (integer->values) { - hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length); - } else { - hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t)); - } - - if (integer->negative) { - hash ^= murmur_scramble((uint32_t) 1); - } - - return hash; + const pm_integer_node_t *cast = (const pm_integer_node_t *) node; + return integer_hash(&cast->value); } case PM_SOURCE_LINE_NODE: { // Source lines hash their line number. @@ -94,11 +102,9 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) return murmur_hash((const uint8_t *) value, sizeof(double)); } case PM_RATIONAL_NODE: { - // Rationals hash their numeric value. Because their numeric value - // is stored as a subnode, we hash that node and then mix in the - // fact that this is a rational node. - const pm_node_t *numeric = ((const pm_rational_node_t *) node)->numeric; - return node_hash(metadata, numeric) ^ murmur_scramble((uint32_t) node->type); + // Rationals hash their numerator and denominator. + const pm_rational_node_t *cast = (const pm_rational_node_t *) node; + return integer_hash(&cast->numerator) ^ integer_hash(&cast->denominator) ^ murmur_scramble((uint32_t) cast->base.type); } case PM_IMAGINARY_NODE: { // Imaginaries hash their numeric value. Because their numeric value @@ -148,7 +154,7 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) * and must be able to compare all node types that will be stored in this hash. */ static pm_node_t * -pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) { +pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, bool replace, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) { // If we are out of space, we need to resize the hash. This will cause all // of the nodes to be rehashed and reinserted into the new hash. if (hash->size * 2 >= hash->capacity) { @@ -196,9 +202,14 @@ pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *m // already in the hash. Otherwise, we can just increment the size and insert // the new node. pm_node_t *result = hash->nodes[index]; - if (result == NULL) hash->size++; - hash->nodes[index] = node; + if (result == NULL) { + hash->size++; + hash->nodes[index] = node; + } else if (replace) { + hash->nodes[index] = node; + } + return result; } @@ -275,8 +286,15 @@ pm_compare_number_nodes(const pm_static_literals_metadata_t *metadata, const pm_ switch (PM_NODE_TYPE(left)) { case PM_IMAGINARY_NODE: return pm_compare_number_nodes(metadata, ((const pm_imaginary_node_t *) left)->numeric, ((const pm_imaginary_node_t *) right)->numeric); - case PM_RATIONAL_NODE: - return pm_compare_number_nodes(metadata, ((const pm_rational_node_t *) left)->numeric, ((const pm_rational_node_t *) right)->numeric); + case PM_RATIONAL_NODE: { + const pm_rational_node_t *left_rational = (const pm_rational_node_t *) left; + const pm_rational_node_t *right_rational = (const pm_rational_node_t *) right; + + int result = pm_integer_compare(&left_rational->denominator, &right_rational->denominator); + if (result != 0) return result; + + return pm_integer_compare(&left_rational->numerator, &right_rational->numerator); + } case PM_INTEGER_NODE: return pm_compare_integer_nodes(metadata, left, right); case PM_FLOAT_NODE: @@ -335,7 +353,7 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_static_liter * Add a node to the set of static literals. */ pm_node_t * -pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node) { +pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -347,6 +365,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_integer_nodes ); case PM_FLOAT_NODE: @@ -358,6 +377,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_float_nodes ); case PM_RATIONAL_NODE: @@ -370,6 +390,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_number_nodes ); case PM_STRING_NODE: @@ -382,6 +403,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_string_nodes ); case PM_REGULAR_EXPRESSION_NODE: @@ -393,6 +415,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_regular_expression_nodes ); case PM_SYMBOL_NODE: @@ -404,26 +427,27 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line .encoding_name = NULL }, node, + replace, pm_compare_string_nodes ); case PM_TRUE_NODE: { pm_node_t *duplicated = literals->true_node; - literals->true_node = node; + if ((duplicated == NULL) || replace) literals->true_node = node; return duplicated; } case PM_FALSE_NODE: { pm_node_t *duplicated = literals->false_node; - literals->false_node = node; + if ((duplicated == NULL) || replace) literals->false_node = node; return duplicated; } case PM_NIL_NODE: { pm_node_t *duplicated = literals->nil_node; - literals->nil_node = node; + if ((duplicated == NULL) || replace) literals->nil_node = node; return duplicated; } case PM_SOURCE_ENCODING_NODE: { pm_node_t *duplicated = literals->source_encoding_node; - literals->source_encoding_node = node; + if ((duplicated == NULL) || replace) literals->source_encoding_node = node; return duplicated; } default: @@ -456,7 +480,7 @@ pm_static_literal_positive_p(const pm_node_t *node) { case PM_INTEGER_NODE: return !((const pm_integer_node_t *) node)->value.negative; case PM_RATIONAL_NODE: - return pm_static_literal_positive_p(((const pm_rational_node_t *) node)->numeric); + return !((const pm_rational_node_t *) node)->numerator.negative; case PM_IMAGINARY_NODE: return pm_static_literal_positive_p(((const pm_imaginary_node_t *) node)->numeric); default: @@ -466,43 +490,6 @@ pm_static_literal_positive_p(const pm_node_t *node) { } /** - * Inspect a rational node that wraps a float node. This is going to be a - * poor-man's version of the Ruby `Rational#to_s` method, because we're not - * going to try to reduce the rational by finding the GCD. We'll leave that for - * a future improvement. - */ -static void -pm_rational_inspect(pm_buffer_t *buffer, pm_rational_node_t *node) { - const uint8_t *start = node->base.location.start; - const uint8_t *end = node->base.location.end - 1; // r - - while (start < end && *start == '0') start++; // 0.1 -> .1 - while (end > start && end[-1] == '0') end--; // 1.0 -> 1. - size_t length = (size_t) (end - start); - - const uint8_t *point = memchr(start, '.', length); - assert(point && "should have a decimal point"); - - uint8_t *digits = malloc(length - 1); - if (digits == NULL) return; - - memcpy(digits, start, (unsigned long) (point - start)); - memcpy(digits + (point - start), point + 1, (unsigned long) (end - point - 1)); - - pm_integer_t numerator = { 0 }; - pm_integer_parse(&numerator, PM_INTEGER_BASE_DECIMAL, digits, digits + length - 1); - - pm_buffer_append_byte(buffer, '('); - pm_integer_string(buffer, &numerator); - pm_buffer_append_string(buffer, "/1", 2); - for (size_t index = 0; index < (size_t) (end - point - 1); index++) pm_buffer_append_byte(buffer, '0'); - pm_buffer_append_byte(buffer, ')'); - - pm_integer_free(&numerator); - free(digits); -} - -/** * Create a string-based representation of the given static literal. */ static inline void @@ -544,7 +531,9 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met pm_buffer_append_string(buffer, "(0", 2); if (pm_static_literal_positive_p(numeric)) pm_buffer_append_byte(buffer, '+'); pm_static_literal_inspect_node(buffer, metadata, numeric); - if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE)) pm_buffer_append_byte(buffer, '*'); + if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE)) { + pm_buffer_append_byte(buffer, '*'); + } pm_buffer_append_string(buffer, "i)", 2); break; } @@ -555,22 +544,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met pm_buffer_append_string(buffer, "nil", 3); break; case PM_RATIONAL_NODE: { - const pm_node_t *numeric = ((const pm_rational_node_t *) node)->numeric; - - switch (PM_NODE_TYPE(numeric)) { - case PM_INTEGER_NODE: - pm_buffer_append_byte(buffer, '('); - pm_static_literal_inspect_node(buffer, metadata, numeric); - pm_buffer_append_string(buffer, "/1)", 3); - break; - case PM_FLOAT_NODE: - pm_rational_inspect(buffer, (pm_rational_node_t *) node); - break; - default: - assert(false && "unreachable"); - break; - } - + const pm_rational_node_t *rational = (const pm_rational_node_t *) node; + pm_buffer_append_byte(buffer, '('); + pm_integer_string(buffer, &rational->numerator); + pm_buffer_append_byte(buffer, '/'); + pm_integer_string(buffer, &rational->denominator); + pm_buffer_append_byte(buffer, ')'); break; } case PM_REGULAR_EXPRESSION_NODE: { @@ -624,7 +603,7 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met /** * Create a string-based representation of the given static literal. */ -PRISM_EXPORTED_FUNCTION void +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node) { pm_static_literal_inspect_node( buffer, diff --git a/prism/static_literals.h b/prism/static_literals.h index 72706054c1..bd29761899 100644 --- a/prism/static_literals.h +++ b/prism/static_literals.h @@ -95,9 +95,10 @@ typedef struct { * @param start_line The line number that the parser starts on. * @param literals The set of static literals to add the node to. * @param node The node to add to the set. + * @param replace Whether to replace the previous node if one already exists. * @return A pointer to the node that is being overwritten, if there is one. */ -pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node); +pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); /** * Free the internal memory associated with the given static literals set. @@ -115,6 +116,6 @@ void pm_static_literals_free(pm_static_literals_t *literals); * @param encoding_name The name of the encoding of the source being parsed. * @param node The node to create a string representation of. */ -PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); #endif diff --git a/prism/templates/ext/prism/api_node.c.erb b/prism/templates/ext/prism/api_node.c.erb index 419236ef78..0e3e4d63cc 100644 --- a/prism/templates/ext/prism/api_node.c.erb +++ b/prism/templates/ext/prism/api_node.c.erb @@ -76,8 +76,7 @@ pm_source_new(const pm_parser_t *parser, rb_encoding *encoding) { rb_ary_push(offsets, ULONG2NUM(parser->newline_list.offsets[index])); } - VALUE source_argv[] = { source_string, LONG2NUM(parser->start_line), offsets }; - return rb_class_new_instance(3, source_argv, rb_cPrismSource); + return rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets); } typedef struct pm_node_stack_node { diff --git a/prism/templates/lib/prism/dsl.rb.erb b/prism/templates/lib/prism/dsl.rb.erb index 8dbb540952..eff0d1c4fc 100644 --- a/prism/templates/lib/prism/dsl.rb.erb +++ b/prism/templates/lib/prism/dsl.rb.erb @@ -2,7 +2,7 @@ module Prism # The DSL module provides a set of methods that can be used to create prism # nodes in a more concise manner. For example, instead of writing: # - # source = Prism::Source.new("[1]") + # source = Prism::Source.for("[1]") # # Prism::ArrayNode.new( # [ @@ -20,7 +20,7 @@ module Prism # # you could instead write: # - # source = Prism::Source.new("[1]") + # source = Prism::Source.for("[1]") # # ArrayNode( # IntegerNode(Prism::IntegerBaseFlags::DECIMAL, 1, Location(source, 1, 1)), source), diff --git a/prism/templates/lib/prism/inspect_visitor.rb.erb b/prism/templates/lib/prism/inspect_visitor.rb.erb index 8e7902f0f1..9328da636b 100644 --- a/prism/templates/lib/prism/inspect_visitor.rb.erb +++ b/prism/templates/lib/prism/inspect_visitor.rb.erb @@ -116,13 +116,8 @@ module Prism # Compose a header for the given node. def inspect_node(name, node) - result = +"@ #{name} (" - location = node.location - result << "location: (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})" - result << ", newline: true" if node.newline? - - result << ")\n" + "@ #{name} (location: (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column}))\n" end # Compose a string representing the given inner location field. diff --git a/prism/templates/lib/prism/node.rb.erb b/prism/templates/lib/prism/node.rb.erb index ddbe53e594..f0ce226def 100644 --- a/prism/templates/lib/prism/node.rb.erb +++ b/prism/templates/lib/prism/node.rb.erb @@ -28,17 +28,14 @@ module Prism location.is_a?(Location) ? location.end_offset : ((location >> 32) + (location & 0xFFFFFFFF)) end - def newline? # :nodoc: - @newline ? true : false + # Returns all of the lines of the source code associated with this node. + def source_lines + location.source_lines end - def set_newline_flag(newline_marked) # :nodoc: - line = location.start_line - unless newline_marked[line] - newline_marked[line] = true - @newline = true - end - end + # An alias for source_lines, used to mimic the API from + # RubyVM::AbstractSyntaxTree to make it easier to migrate. + alias script_lines source_lines # Slice the location of the node from the source. def slice @@ -67,6 +64,43 @@ module Prism DotVisitor.new.tap { |visitor| accept(visitor) }.to_dot end + # Returns a list of nodes that are descendants of this node that contain the + # given line and column. This is useful for locating a node that is selected + # based on the line and column of the source code. + # + # Important to note is that the column given to this method should be in + # bytes, as opposed to characters or code units. + def tunnel(line, column) + queue = [self] #: Array[Prism::node] + result = [] + + while (node = queue.shift) + result << node + + node.compact_child_nodes.each do |child_node| + child_location = child_node.location + + start_line = child_location.start_line + end_line = child_location.end_line + + if start_line == end_line + if line == start_line && column >= child_location.start_column && column < child_location.end_column + queue << child_node + break + end + elsif (line == start_line && column >= child_location.start_column) || (line == end_line && column < child_location.end_column) + queue << child_node + break + elsif line > start_line && line < end_line + queue << child_node + break + end + end + end + + result + end + # Returns a list of the fields that exist for this node class. Fields # describe the structure of the node. This kind of reflection is useful for # things like recursively visiting each node _and_ field in the tree. @@ -135,7 +169,6 @@ module Prism # def initialize: (<%= (node.fields.map { |field| "#{field.rbs_class} #{field.name}" } + ["Location location"]).join(", ") %>) -> void def initialize(source, <%= (node.fields.map(&:name) + ["location"]).join(", ") %>) @source = source - @newline = false @location = location <%- node.fields.each do |field| -%> <%- if Prism::Template::CHECK_FIELD_KIND && field.respond_to?(:check_field_kind) -%> @@ -149,25 +182,6 @@ module Prism def accept(visitor) visitor.visit_<%= node.human %>(self) end - <%- if node.newline == false -%> - - def set_newline_flag(newline_marked) # :nodoc: - # Never mark <%= node.name %> with a newline flag, mark children instead - end - <%- elsif node.newline.is_a?(String) -%> - - def set_newline_flag(newline_marked) # :nodoc: - <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> - <%- case field -%> - <%- when Prism::Template::NodeField -%> - <%= field.name %>.set_newline_flag(newline_marked) - <%- when Prism::Template::NodeListField -%> - first = <%= field.name %>.first - first.set_newline_flag(newline_marked) if first - <%- else raise field.class.name -%> - <%- end -%> - end - <%- end -%> # def child_nodes: () -> Array[nil | Node] def child_nodes diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index c31a319e5f..756821cf7d 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -1,5 +1,5 @@ require "stringio" -require_relative "polyfill/string" +require_relative "polyfill/unpack1" module Prism # A module responsible for deserializing parse results. @@ -10,7 +10,7 @@ module Prism # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 27 + MINOR_VERSION = 29 # The patch version of prism that we are expecting to find in the serialized # strings. @@ -19,7 +19,7 @@ module Prism # Deserialize the AST represented by the given string into a parse result. def self.load(input, serialized) input = input.dup - source = Source.new(input) + source = Source.for(input) loader = Loader.new(source, serialized) result = loader.load_result diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 839013714b..22beae922e 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -91,26 +91,28 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "both block arg and actual block given; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_CONFLICT_AMPERSAND] = { "unexpected `&`; anonymous block parameter is also used within block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_CONFLICT_STAR] = { "unexpected `*`; anonymous rest parameter is also used within block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_CONFLICT_STAR_STAR] = { "unexpected `**`; anonymous keyword rest parameter is also used within block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND] = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected ... when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*`; no anonymous rest parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**`; no anonymous keyword rest parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = { "unexpected `*` splat argument after a `*` splat argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_TERM_PAREN] = { "unexpected %s; expected a `)` to close the arguments", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected `{` after a method call without parenthesis", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected '{' after a method call without parenthesis", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_SEPARATOR] = { "unexpected %s; expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -146,7 +148,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_NAME] = { "unexpected %s; expected a method name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "unexpected %s; expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -156,16 +158,16 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hexadecimal escape sequence", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hex escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; Multiple codepoints at single character literal are disallowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "unterminated Unicode escape", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_SYNTAX }, @@ -174,11 +176,12 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "unexpected %s; expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_IN_DELIMITER] = { "expected a delimiter after the patterns of an `in` clause", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_MESSAGE] = { "unexpected %s; expecting a message to send to the receiver", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_SYNTAX }, @@ -194,6 +197,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EXPRESSION_NOT_WRITABLE_FILE] = { "Can't assign to __FILE__", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_LINE] = { "Can't assign to __LINE__", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_NIL] = { "Can't assign to nil", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED] = { "Can't assign to numbered parameter %.2s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_SELF] = { "Can't change the value of self", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE] = { "Can't assign to true", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_SYNTAX }, @@ -206,48 +210,54 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_HEREDOC_TERM] = { "could not find a terminator for the heredoc", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_VALUE] = { "unexpected %s; expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HEREDOC_IDENTIFIER] = { "unterminated here document identifier", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HEREDOC_TERM] = { "unterminated heredoc; can't find string \"%.*s\" anywhere before EOF", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INSTANCE_VARIABLE_BARE] = { "'@' without identifiers is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_BLOCK_EXIT] = { "Invalid %s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_ESCAPE_CHARACTER] = { "Invalid escape character syntax", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_LOCAL_VARIABLE_READ] = { "identifier %.*s is not valid to get", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_LOCAL_VARIABLE_WRITE] = { "identifier %.*s is not valid to set", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number; numeric literal without digits", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number; numeric literal without digits", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_FRACTION] = { "unexpected fraction part after numeric literal", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = { "invalid hexadecimal number; numeric literal without digits", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number; numeric literal without digits", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_NUMBER_UNDERSCORE] = { "invalid underscore placement in number", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER] = { "invalid underscore placement in number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING] = { "trailing '_' in number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_CHARACTER] = { "Invalid char '\\x%02X' in expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_SYNTAX }, // TODO WHAT? + [PM_ERR_INVALID_PERCENT] = { "unknown type of %string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PERCENT_EOF] = { "unterminated quoted string meets end of file", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_AFTER_ELSE] = { "Invalid retry after else", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_AFTER_ENSURE] = { "Invalid retry after ensure", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_RETRY_WITHOUT_RESCUE] = { "Invalid retry without rescue", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_SYMBOL] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_YIELD] = { "Invalid yield", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when a numbered parameter is already used", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LAMBDA_TERM_END] = { "expected a lambda block beginning with `do` to end with `end`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LIST_I_LOWER_ELEMENT] = { "expected a symbol in a `%i` list", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_LIST_I_LOWER_TERM] = { "expected a closing delimiter for the `%i` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_LOWER_TERM] = { "unterminated list; expected a closing delimiter for the `%i`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LIST_I_UPPER_ELEMENT] = { "expected a symbol in a `%I` list", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_LIST_I_UPPER_TERM] = { "expected a closing delimiter for the `%I` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_UPPER_TERM] = { "unterminated list; expected a closing delimiter for the `%I`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LIST_W_LOWER_ELEMENT] = { "expected a string in a `%w` list", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_LIST_W_LOWER_TERM] = { "expected a closing delimiter for the `%w` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_LOWER_TERM] = { "unterminated list; expected a closing delimiter for the `%w`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_LIST_W_UPPER_ELEMENT] = { "expected a string in a `%W` list", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_UPPER_TERM] = { "unterminated list; expected a closing delimiter for the `%W`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in method body", PM_ERROR_LEVEL_SYNTAX }, @@ -258,15 +268,17 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK] = { "numbered parameter is already used in inner block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when 'it' is already used", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK] = { "numbered parameter is already used in outer block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_CIRCULAR] = { "circular argument reference - %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_FORWARDING_AFTER_REST] = { "... after rest argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NAME_DUPLICATED] = { "duplicated argument name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_SYNTAX }, @@ -277,6 +289,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_UNEXPECTED_NO_KW] = { "unexpected **nil; no keywords marker disallowed after keywords", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_CAPTURE_DUPLICATE] = { "duplicated variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_SYNTAX }, @@ -288,9 +301,11 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_IMPLICIT] = { "unexpected implicit hash in pattern; use '{' to delineate", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY] = { "unexpected %s; expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY_DUPLICATE] = { "duplicated key name", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, // TODO // THIS // AND // ABOVE // IS WEIRD + [PM_ERR_PATTERN_HASH_KEY_INTERPOLATED] = { "symbol literal with interpolation is not allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY_LOCALS] = { "key must be valid as local variables", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, @@ -304,7 +319,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_TERM] = { "unterminated regexp meets end of file; expected a closing delimiter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_SYNTAX }, @@ -317,18 +332,21 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_INTERPOLATED_TERM] = { "unterminated string; expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, // TODO expected symbol? prism.c ~9719 - [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "unterminated quoted string; expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "unterminated symbol; expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_BLOCK_ARGUMENT] = { "block argument should not be given", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_INDEX_BLOCK] = { "unexpected block arg given in index; blocks are not allowed in index expressions", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_INDEX_KEYWORDS] = { "unexpected keyword arg given in index; keywords are not allowed in index expressions", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_SAFE_NAVIGATION] = { "&. inside multiple assignment destination", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNTIL_TERM] = { "expected an `end` to close the `until` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -340,6 +358,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_SYNTAX }, // Warnings + [PM_WARN_AMBIGUOUS_BINARY_OPERATOR] = { "'%s' after local variable or literal is interpreted as binary operator even though it seems like %s", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = { "ambiguous first argument; put parentheses or a space even after `-` operator", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS] = { "ambiguous first argument; put parentheses or a space even after `+` operator", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_PREFIX_AMPERSAND] = { "ambiguous `&` has been interpreted as an argument prefix", PM_WARNING_LEVEL_VERBOSE }, @@ -349,8 +368,8 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_COMPARISON_AFTER_COMPARISON] = { "comparison '%.*s' after comparison", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_DOT_DOT_DOT_EOL] = { "... at EOL, should be parenthesized?", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_DUPLICATED_HASH_KEY] = { "key %.*s is duplicated and overwritten on line %" PRIi32, PM_WARNING_LEVEL_DEFAULT }, - [PM_WARN_DUPLICATED_WHEN_CLAUSE] = { "duplicated 'when' clause with line %" PRIi32 " is ignored", PM_WARNING_LEVEL_VERBOSE }, - [PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0] = { "found `= literal' in conditional, should be ==", PM_WARNING_LEVEL_DEFAULT }, + [PM_WARN_DUPLICATED_WHEN_CLAUSE] = { "'when' clause on line %" PRIi32 " duplicates 'when' clause on line %" PRIi32 " and is ignored", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_EQUAL_IN_CONDITIONAL_3_3] = { "found `= literal' in conditional, should be ==", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_EQUAL_IN_CONDITIONAL] = { "found '= literal' in conditional, should be ==", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_END_IN_METHOD] = { "END in method; use at_exit", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_FLOAT_OUT_OF_RANGE] = { "Float %.*s%s out of range", PM_WARNING_LEVEL_VERBOSE }, @@ -362,6 +381,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_KEYWORD_EOL] = { "`%.*s` at the end of line without an expression", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_LITERAL_IN_CONDITION_DEFAULT] = { "%sliteral in %s", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_LITERAL_IN_CONDITION_VERBOSE] = { "%sliteral in %s", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_SHAREABLE_CONSTANT_VALUE_LINE] = { "'shareable_constant_value' is ignored unless in comment-only line", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_SHEBANG_CARRIAGE_RETURN] = { "shebang line ending with \\r may cause problems", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNEXPECTED_CARRIAGE_RETURN] = { "encountered \\r in middle of line, treated as a mere space", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNREACHABLE_STATEMENT] = { "statement not reached", PM_WARNING_LEVEL_VERBOSE }, diff --git a/prism/templates/src/node.c.erb b/prism/templates/src/node.c.erb index e1c35f5a45..deb2ea49fc 100644 --- a/prism/templates/src/node.c.erb +++ b/prism/templates/src/node.c.erb @@ -1,19 +1,6 @@ #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" #include "prism/node.h" -static void -pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize); - -/** - * Calculate the size of the node list in bytes. - */ -static size_t -pm_node_list_memsize(pm_node_list_t *node_list, pm_memsize_t *memsize) { - pm_node_t *node; - PM_NODE_LIST_FOREACH(node_list, index, node) pm_node_memsize_node(node, memsize); - return sizeof(pm_node_list_t) + (node_list->capacity * sizeof(pm_node_t *)); -} - /** * Attempts to grow the node list to the next size. If there is already * capacity in the list, this function does nothing. Otherwise it reallocates @@ -156,57 +143,6 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { xfree(node); } -static void -pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize) { - memsize->node_count++; - - switch (PM_NODE_TYPE(node)) { - // We do not calculate memsize of a ScopeNode - // as it should never be generated - case PM_SCOPE_NODE: - return; - <%- nodes.each do |node| -%> -#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - case <%= node.type %>: { - pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; - memsize->memsize += sizeof(*cast); - <%- node.fields.each do |field| -%> - <%- case field -%> - <%- when Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::FlagsField, Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::DoubleField -%> - <%- when Prism::Template::NodeField -%> - pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize); - <%- when Prism::Template::OptionalNodeField -%> - if (cast-><%= field.name %> != NULL) { - pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize); - } - <%- when Prism::Template::StringField -%> - memsize->memsize += (pm_string_memsize(&cast-><%= field.name %>) - sizeof(pm_string_t)); - <%- when Prism::Template::NodeListField -%> - memsize->memsize += (pm_node_list_memsize(&cast-><%= field.name %>, memsize) - sizeof(pm_node_list_t)); - <%- when Prism::Template::ConstantListField -%> - memsize->memsize += (pm_constant_id_list_memsize(&cast-><%= field.name %>) - sizeof(pm_constant_id_list_t)); - <%- when Prism::Template::IntegerField -%> - memsize->memsize += (pm_integer_memsize(&cast-><%= field.name %>) - sizeof(pm_integer_t)); - <%- else -%> - <%- raise -%> - <%- end -%> - <%- end -%> - break; - } - <%- end -%> -#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - } -} - -/** - * Calculates the memory footprint of a given node. - */ -PRISM_EXPORTED_FUNCTION void -pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize) { - *memsize = (pm_memsize_t) { .memsize = 0, .node_count = 0 }; - pm_node_memsize_node(node, memsize); -} - /** * Returns a string representation of the given node type. */ diff --git a/prism/templates/src/token_type.c.erb b/prism/templates/src/token_type.c.erb index 1aeecd72b2..f196393ee1 100644 --- a/prism/templates/src/token_type.c.erb +++ b/prism/templates/src/token_type.c.erb @@ -30,7 +30,7 @@ const char * pm_token_type_human(pm_token_type_t token_type) { switch (token_type) { case PM_TOKEN_EOF: - return "end of file"; + return "end-of-input"; case PM_TOKEN_MISSING: return "missing token"; case PM_TOKEN_NOT_PROVIDED: @@ -90,9 +90,9 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_DOT: return "'.'"; case PM_TOKEN_DOT_DOT: - return "'..'"; + return ".."; case PM_TOKEN_DOT_DOT_DOT: - return "'...'"; + return "..."; case PM_TOKEN_EMBDOC_BEGIN: return "'=begin'"; case PM_TOKEN_EMBDOC_END: @@ -130,9 +130,9 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_GREATER_EQUAL: return "'>='"; case PM_TOKEN_GREATER_GREATER: - return "'>>'"; + return ">>"; case PM_TOKEN_GREATER_GREATER_EQUAL: - return "'>>='"; + return ">>="; case PM_TOKEN_HEREDOC_END: return "heredoc ending"; case PM_TOKEN_HEREDOC_START: @@ -258,9 +258,9 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_LESS_EQUAL_GREATER: return "'<=>'"; case PM_TOKEN_LESS_LESS: - return "'<<'"; + return "<<"; case PM_TOKEN_LESS_LESS_EQUAL: - return "'<<='"; + return "<<="; case PM_TOKEN_METHOD_NAME: return "method name"; case PM_TOKEN_MINUS: @@ -352,7 +352,7 @@ pm_token_type_human(pm_token_type_t token_type) { case PM_TOKEN_USTAR: return "*"; case PM_TOKEN_USTAR_STAR: - return "'**'"; + return "**"; case PM_TOKEN_WORDS_SEP: return "string separator"; case PM_TOKEN___END__: diff --git a/prism/util/pm_char.h b/prism/util/pm_char.h index 32f698a42b..deeafd6321 100644 --- a/prism/util/pm_char.h +++ b/prism/util/pm_char.h @@ -34,8 +34,7 @@ size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length); * @return The number of characters at the start of the string that are * whitespace. */ -size_t -pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); +size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); /** * Returns the number of characters at the start of the string that are inline diff --git a/prism/util/pm_constant_pool.c b/prism/util/pm_constant_pool.c index 2a3203f4c4..624002cec9 100644 --- a/prism/util/pm_constant_pool.c +++ b/prism/util/pm_constant_pool.c @@ -62,14 +62,6 @@ pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id) { } /** - * Get the memory size of a list of constant ids. - */ -size_t -pm_constant_id_list_memsize(pm_constant_id_list_t *list) { - return sizeof(pm_constant_id_list_t) + (list->capacity * sizeof(pm_constant_id_t)); -} - -/** * Free the memory associated with a list of constant ids. */ void diff --git a/prism/util/pm_constant_pool.h b/prism/util/pm_constant_pool.h index 0fe16858a0..6df23f8f50 100644 --- a/prism/util/pm_constant_pool.h +++ b/prism/util/pm_constant_pool.h @@ -88,14 +88,6 @@ void pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_co bool pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id); /** - * Get the memory size of a list of constant ids. - * - * @param list The list to get the memory size of. - * @return The memory size of the list. - */ -size_t pm_constant_id_list_memsize(pm_constant_id_list_t *list); - -/** * Free the memory associated with a list of constant ids. * * @param list The list to free. diff --git a/prism/util/pm_integer.c b/prism/util/pm_integer.c index e523bae90b..137f736536 100644 --- a/prism/util/pm_integer.c +++ b/prism/util/pm_integer.c @@ -471,15 +471,18 @@ pm_integer_parse_big(pm_integer_t *integer, uint32_t multiplier, const uint8_t * * has already been validated, as internal validation checks are not performed * here. */ -PRISM_EXPORTED_FUNCTION void +void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end) { - // Ignore unary +. Unary + is parsed differently and will not end up here. + // Ignore unary +. Unary - is parsed differently and will not end up here. // Instead, it will modify the parsed integer later. if (*start == '+') start++; // Determine the multiplier from the base, and skip past any prefixes. uint32_t multiplier = 10; switch (base) { + case PM_INTEGER_BASE_DEFAULT: + while (*start == '0') start++; // 01 -> 1 + break; case PM_INTEGER_BASE_BINARY: start += 2; // 0b multiplier = 2; @@ -534,14 +537,6 @@ pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *s } /** - * Return the memory size of the integer. - */ -size_t -pm_integer_memsize(const pm_integer_t *integer) { - return sizeof(pm_integer_t) + integer->length * sizeof(uint32_t); -} - -/** * Compare two integers. This function returns -1 if the left integer is less * than the right integer, 0 if they are equal, and 1 if the left integer is * greater than the right integer. @@ -573,6 +568,39 @@ pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right) { } /** + * Reduce a ratio of integers to its simplest form. + */ +void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator) { + // If either the numerator or denominator do not fit into a 32-bit integer, + // then this function is a no-op. In the future, we may consider reducing + // even the larger numbers, but for now we're going to keep it simple. + if ( + // If the numerator doesn't fit into a 32-bit integer, return early. + numerator->length != 0 || + // If the denominator doesn't fit into a 32-bit integer, return early. + denominator->length != 0 || + // If the numerator is 0, then return early. + numerator->value == 0 || + // If the denominator is 1, then return early. + denominator->value == 1 + ) return; + + // Find the greatest common divisor of the numerator and denominator. + uint32_t divisor = numerator->value; + uint32_t remainder = denominator->value; + + while (remainder != 0) { + uint32_t temporary = remainder; + remainder = divisor % remainder; + divisor = temporary; + } + + // Divide the numerator and denominator by the greatest common divisor. + numerator->value /= divisor; + denominator->value /= divisor; +} + +/** * Convert an integer to a decimal string. */ PRISM_EXPORTED_FUNCTION void diff --git a/prism/util/pm_integer.h b/prism/util/pm_integer.h index 7f172988b3..91b28ad2f3 100644 --- a/prism/util/pm_integer.h +++ b/prism/util/pm_integer.h @@ -48,6 +48,9 @@ typedef struct { * from the string itself. */ typedef enum { + /** The default decimal base, with no prefix. Leading 0s will be ignored. */ + PM_INTEGER_BASE_DEFAULT, + /** The binary base, indicated by a 0b or 0B prefix. */ PM_INTEGER_BASE_BINARY, @@ -79,15 +82,7 @@ typedef enum { * @param start The start of the string. * @param end The end of the string. */ -PRISM_EXPORTED_FUNCTION void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end); - -/** - * Return the memory size of the integer. - * - * @param integer The integer to get the memory size of. - * @return The size of the memory associated with the integer. - */ -size_t pm_integer_memsize(const pm_integer_t *integer); +void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end); /** * Compare two integers. This function returns -1 if the left integer is less @@ -101,6 +96,18 @@ size_t pm_integer_memsize(const pm_integer_t *integer); int pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right); /** + * Reduce a ratio of integers to its simplest form. + * + * If either the numerator or denominator do not fit into a 32-bit integer, then + * this function is a no-op. In the future, we may consider reducing even the + * larger numbers, but for now we're going to keep it simple. + * + * @param numerator The numerator of the ratio. + * @param denominator The denominator of the ratio. + */ +void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator); + +/** * Convert an integer to a decimal string. * * @param buffer The buffer to append the string to. diff --git a/prism/util/pm_string.c b/prism/util/pm_string.c index 8342edc34e..dfc121b6a2 100644 --- a/prism/util/pm_string.c +++ b/prism/util/pm_string.c @@ -246,18 +246,6 @@ pm_string_file_init(pm_string_t *string, const char *filepath) { } /** - * Returns the memory size associated with the string. - */ -size_t -pm_string_memsize(const pm_string_t *string) { - size_t size = sizeof(pm_string_t); - if (string->type == PM_STRING_OWNED) { - size += string->length; - } - return size; -} - -/** * Ensure the string is owned. If it is not, then reinitialize it as owned and * copy over the previous source. */ diff --git a/prism/util/pm_string.h b/prism/util/pm_string.h index a68e2a7c91..d23792c0ba 100644 --- a/prism/util/pm_string.h +++ b/prism/util/pm_string.h @@ -121,14 +121,6 @@ PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const ch PRISM_EXPORTED_FUNCTION bool pm_string_file_init(pm_string_t *string, const char *filepath); /** - * Returns the memory size associated with the string. - * - * @param string The string to get the memory size of. - * @return The size of the memory associated with the string. - */ -size_t pm_string_memsize(const pm_string_t *string); - -/** * Ensure the string is owned. If it is not, then reinitialize it as owned and * copy over the previous source. * diff --git a/prism/util/pm_strpbrk.c b/prism/util/pm_strpbrk.c index 6c8dea1836..916a4cc3fd 100644 --- a/prism/util/pm_strpbrk.c +++ b/prism/util/pm_strpbrk.c @@ -9,6 +9,27 @@ pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start } /** + * Set the explicit encoding for the parser to the current encoding. + */ +static inline void +pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) { + if (parser->explicit_encoding != NULL) { + if (parser->explicit_encoding == parser->encoding) { + // Okay, we already locked to this encoding. + } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { + // Not okay, we already found a Unicode escape sequence and this + // conflicts. + pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name); + } else { + // Should not be anything else. + assert(false && "unreachable"); + } + } + + parser->explicit_encoding = parser->encoding; +} + +/** * This is the default path. */ static inline const uint8_t * @@ -52,7 +73,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars * This is the path when the encoding is ASCII-8BIT. */ static inline const uint8_t * -pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) { +pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; while (index < maximum) { @@ -60,6 +81,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi return source + index; } + if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1); index++; } @@ -72,6 +94,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi static inline const uint8_t * pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; + const pm_encoding_t *encoding = parser->encoding; while (index < maximum) { if (strchr((const char *) charset, source[index]) != NULL) { @@ -81,7 +104,8 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t if (source[index] < 0x80) { index++; } else { - size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width); if (width > 0) { index += width; @@ -96,7 +120,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t do { index++; - } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); + } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); } @@ -113,6 +137,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t static inline const uint8_t * pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) { size_t index = 0; + const pm_encoding_t *encoding = parser->encoding; while (index < maximum) { if (strchr((const char *) charset, source[index]) != NULL) { @@ -122,7 +147,8 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t if (source[index] < 0x80 || !validate) { index++; } else { - size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); + pm_strpbrk_explicit_encoding_set(parser, source, width); if (width > 0) { index += width; @@ -135,7 +161,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t do { index++; - } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); + } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); } @@ -171,7 +197,7 @@ pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, p } else if (!parser->encoding_changed) { return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate); } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) { - return pm_strpbrk_ascii_8bit(source, charset, (size_t) length); + return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate); } else if (parser->encoding->multibyte) { return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate); } else { diff --git a/prism/version.h b/prism/version.h index 0d25b2883c..154e967944 100644 --- a/prism/version.h +++ b/prism/version.h @@ -14,7 +14,7 @@ /** * The minor version of the Prism library as an int. */ -#define PRISM_VERSION_MINOR 27 +#define PRISM_VERSION_MINOR 29 /** * The patch version of the Prism library as an int. @@ -24,6 +24,6 @@ /** * The version of the Prism library as a constant string. */ -#define PRISM_VERSION "0.27.0" +#define PRISM_VERSION "0.29.0" #endif diff --git a/prism_compile.c b/prism_compile.c index d207c6bf07..ea4c4fa054 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -152,12 +152,11 @@ pm_location_line_number(const pm_parser_t *parser, const pm_location_t *location } /** - * Convert the value of an integer node into a Ruby Integer. + * Parse the value of a pm_integer_t into a Ruby Integer. */ static VALUE -parse_integer(const pm_integer_node_t *node) +parse_integer_value(const pm_integer_t *integer) { - const pm_integer_t *integer = &node->value; VALUE result; if (integer->values == NULL) { @@ -188,6 +187,15 @@ parse_integer(const pm_integer_node_t *node) } /** + * Convert the value of an integer node into a Ruby Integer. + */ +static inline VALUE +parse_integer(const pm_integer_node_t *node) +{ + return parse_integer_value(&node->value); +} + +/** * Convert the value of a float node into a Ruby Float. */ static VALUE @@ -205,36 +213,9 @@ parse_float(const pm_float_node_t *node) static VALUE parse_rational(const pm_rational_node_t *node) { - VALUE result; - - if (PM_NODE_TYPE_P(node->numeric, PM_FLOAT_NODE)) { - const uint8_t *start = node->base.location.start; - const uint8_t *end = node->base.location.end - 1; - size_t length = end - start; - - char *buffer = malloc(length + 1); - memcpy(buffer, start, length); - - buffer[length] = '\0'; - - char *decimal = memchr(buffer, '.', length); - RUBY_ASSERT(decimal); - size_t seen_decimal = decimal - buffer; - size_t fraclen = length - seen_decimal - 1; - memmove(decimal, decimal + 1, fraclen + 1); - - VALUE numerator = rb_cstr_to_inum(buffer, 10, false); - result = rb_rational_new(numerator, rb_int_positive_pow(10, fraclen)); - - free(buffer); - } - else { - RUBY_ASSERT(PM_NODE_TYPE_P(node->numeric, PM_INTEGER_NODE)); - VALUE numerator = parse_integer((const pm_integer_node_t *) node->numeric); - result = rb_rational_raw(numerator, INT2FIX(1)); - } - - return result; + VALUE numerator = parse_integer_value(&node->numerator); + VALUE denominator = parse_integer_value(&node->denominator); + return rb_rational_new(numerator, denominator); } /** @@ -279,7 +260,7 @@ parse_string(const pm_scope_node_t *scope_node, const pm_string_t *string) * creating those strings based on the flags set on the owning node. */ static inline VALUE -parse_string_encoded(const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *string) +parse_string_encoded(const pm_node_t *node, const pm_string_t *string, rb_encoding *default_encoding) { rb_encoding *encoding; @@ -290,7 +271,7 @@ parse_string_encoded(const pm_scope_node_t *scope_node, const pm_node_t *node, c encoding = rb_utf8_encoding(); } else { - encoding = scope_node->encoding; + encoding = default_encoding; } return rb_enc_str_new((const char *) pm_string_source(string), pm_string_length(string), encoding); @@ -301,10 +282,10 @@ parse_static_literal_string(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, { rb_encoding *encoding; - if (node->flags & PM_ENCODING_FLAGS_FORCED_BINARY_ENCODING) { + if (node->flags & PM_STRING_FLAGS_FORCED_BINARY_ENCODING) { encoding = rb_ascii8bit_encoding(); } - else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { + else if (node->flags & PM_STRING_FLAGS_FORCED_UTF8_ENCODING) { encoding = rb_utf8_encoding(); } else { @@ -351,91 +332,46 @@ pm_optimizable_range_item_p(const pm_node_t *node) return (!node || PM_NODE_TYPE_P(node, PM_INTEGER_NODE) || PM_NODE_TYPE_P(node, PM_NIL_NODE)); } -static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); - -static int -pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +/** Raise an error corresponding to the invalid regular expression. */ +static VALUE +parse_regexp_error(rb_iseq_t *iseq, int32_t line_number, const char *fmt, ...) { - int stack_size = 0; - size_t parts_size = parts->size; - bool interpolated = false; - - if (parts_size > 0) { - VALUE current_string = Qnil; - - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - const pm_string_node_t *string_node = (const pm_string_node_t *) part; - VALUE string_value = parse_string_encoded(scope_node, (const pm_node_t *) string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else { - interpolated = true; - - if ( - PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && - ((const pm_embedded_statements_node_t *) part)->statements != NULL && - ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && - PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE) - ) { - const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; - VALUE string_value = parse_string_encoded(scope_node, (const pm_node_t *) string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else { - if (!RTEST(current_string)) { - current_string = rb_enc_str_new(NULL, 0, scope_node->encoding); - } - - PUSH_INSN1(ret, *node_location, putobject, rb_fstring(current_string)); - PM_COMPILE_NOT_POPPED(part); - PUSH_INSN(ret, *node_location, dup); - PUSH_INSN1(ret, *node_location, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); - PUSH_INSN(ret, *node_location, anytostring); - - current_string = Qnil; - stack_size += 2; - } - } - } - - if (RTEST(current_string)) { - current_string = rb_fstring(current_string); + va_list args; + va_start(args, fmt); + VALUE error = rb_syntax_error_append(Qnil, rb_iseq_path(iseq), line_number, -1, NULL, "%" PRIsVALUE, args); + va_end(args); + rb_exc_raise(error); +} - if (stack_size == 0 && interpolated) { - PUSH_INSN1(ret, *node_location, putstring, current_string); - } - else { - PUSH_INSN1(ret, *node_location, putobject, current_string); - } +static VALUE +parse_regexp_string_part(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped, rb_encoding *implicit_regexp_encoding, rb_encoding *explicit_regexp_encoding) +{ + // If we were passed an explicit regexp encoding, then we need to double + // check that it's okay here for this fragment of the string. + rb_encoding *encoding; - current_string = Qnil; - stack_size++; - } + if (explicit_regexp_encoding != NULL) { + encoding = explicit_regexp_encoding; + } + else if (node->flags & PM_STRING_FLAGS_FORCED_BINARY_ENCODING) { + encoding = rb_ascii8bit_encoding(); + } + else if (node->flags & PM_STRING_FLAGS_FORCED_UTF8_ENCODING) { + encoding = rb_utf8_encoding(); } else { - PUSH_INSN(ret, *node_location, putnil); + encoding = implicit_regexp_encoding; } - return stack_size; + VALUE string = rb_enc_str_new((const char *) pm_string_source(unescaped), pm_string_length(unescaped), encoding); + VALUE error = rb_reg_check_preprocess(string); + + if (error != Qnil) parse_regexp_error(iseq, pm_node_line_number(scope_node->parser, node), "%" PRIsVALUE, rb_obj_as_string(error)); + return string; } static VALUE -pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +pm_static_literal_concat(rb_iseq_t *iseq, const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, rb_encoding *implicit_regexp_encoding, rb_encoding *explicit_regexp_encoding, bool top) { VALUE current = Qnil; @@ -445,11 +381,28 @@ pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *sco switch (PM_NODE_TYPE(part)) { case PM_STRING_NODE: - string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); + if (implicit_regexp_encoding != NULL) { + if (top) { + string = parse_regexp_string_part(iseq, scope_node, part, &((const pm_string_node_t *) part)->unescaped, implicit_regexp_encoding, explicit_regexp_encoding); + } + else { + string = parse_string_encoded(part, &((const pm_string_node_t *) part)->unescaped, scope_node->encoding); + VALUE error = rb_reg_check_preprocess(string); + if (error != Qnil) parse_regexp_error(iseq, pm_node_line_number(scope_node->parser, part), "%" PRIsVALUE, rb_obj_as_string(error)); + } + } + else { + string = parse_string_encoded(part, &((const pm_string_node_t *) part)->unescaped, scope_node->encoding); + } break; case PM_INTERPOLATED_STRING_NODE: - string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); + string = pm_static_literal_concat(iseq, &((const pm_interpolated_string_node_t *) part)->parts, scope_node, implicit_regexp_encoding, explicit_regexp_encoding, false); break; + case PM_EMBEDDED_STATEMENTS_NODE: { + const pm_embedded_statements_node_t *cast = (const pm_embedded_statements_node_t *) part; + string = pm_static_literal_concat(iseq, &cast->statements->body, scope_node, implicit_regexp_encoding, explicit_regexp_encoding, false); + break; + } default: RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); return Qnil; @@ -543,21 +496,10 @@ parse_regexp_encoding(const pm_scope_node_t *scope_node, const pm_node_t *node) return rb_enc_get_from_index(ENCINDEX_Windows_31J); } else { - return scope_node->encoding; + return NULL; } } -/** Raise an error corresponding to the invalid regular expression. */ -static VALUE -parse_regexp_error(rb_iseq_t *iseq, int32_t line_number, const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - VALUE error = rb_syntax_error_append(Qnil, rb_iseq_path(iseq), line_number, -1, NULL, "%" PRIsVALUE, args); - va_end(args); - rb_exc_raise(error); -} - static VALUE parse_regexp(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, VALUE string) { @@ -581,22 +523,149 @@ parse_regexp(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t static inline VALUE parse_regexp_literal(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) { - VALUE string = rb_enc_str_new((const char *) pm_string_source(unescaped), pm_string_length(unescaped), parse_regexp_encoding(scope_node, node)); + rb_encoding *regexp_encoding = parse_regexp_encoding(scope_node, node); + if (regexp_encoding == NULL) regexp_encoding = scope_node->encoding; + + VALUE string = rb_enc_str_new((const char *) pm_string_source(unescaped), pm_string_length(unescaped), regexp_encoding); return parse_regexp(iseq, scope_node, node, string); } static inline VALUE parse_regexp_concat(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_node_list_t *parts) { - VALUE string = pm_static_literal_concat(parts, scope_node, false); - rb_enc_associate(string, parse_regexp_encoding(scope_node, node)); + rb_encoding *explicit_regexp_encoding = parse_regexp_encoding(scope_node, node); + rb_encoding *implicit_regexp_encoding = explicit_regexp_encoding != NULL ? explicit_regexp_encoding : scope_node->encoding; + + VALUE string = pm_static_literal_concat(iseq, parts, scope_node, implicit_regexp_encoding, explicit_regexp_encoding, false); return parse_regexp(iseq, scope_node, node, string); } +static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); + +static int +pm_interpolated_node_compile(rb_iseq_t *iseq, const pm_node_list_t *parts, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node, rb_encoding *implicit_regexp_encoding, rb_encoding *explicit_regexp_encoding) +{ + int stack_size = 0; + size_t parts_size = parts->size; + bool interpolated = false; + + if (parts_size > 0) { + VALUE current_string = Qnil; + pm_line_column_t current_location = *node_location; + + for (size_t index = 0; index < parts_size; index++) { + const pm_node_t *part = parts->nodes[index]; + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + const pm_string_node_t *string_node = (const pm_string_node_t *) part; + VALUE string_value; + + if (implicit_regexp_encoding == NULL) { + string_value = parse_string_encoded(part, &string_node->unescaped, scope_node->encoding); + } + else { + string_value = parse_regexp_string_part(iseq, scope_node, (const pm_node_t *) string_node, &string_node->unescaped, implicit_regexp_encoding, explicit_regexp_encoding); + } + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + if (index != 0) current_location = PM_NODE_END_LINE_COLUMN(scope_node->parser, part); + } + } + else { + interpolated = true; + + if ( + PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && + ((const pm_embedded_statements_node_t *) part)->statements != NULL && + ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && + PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE) + ) { + const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; + VALUE string_value; + + if (implicit_regexp_encoding == NULL) { + string_value = parse_string_encoded(part, &string_node->unescaped, scope_node->encoding); + } + else { + string_value = parse_regexp_string_part(iseq, scope_node, (const pm_node_t *) string_node, &string_node->unescaped, implicit_regexp_encoding, explicit_regexp_encoding); + } + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + current_location = PM_NODE_START_LINE_COLUMN(scope_node->parser, part); + } + } + else { + if (!RTEST(current_string)) { + rb_encoding *encoding; + + if (implicit_regexp_encoding != NULL) { + if (explicit_regexp_encoding != NULL) { + encoding = explicit_regexp_encoding; + } + else if (scope_node->parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { + encoding = rb_ascii8bit_encoding(); + } + else { + encoding = implicit_regexp_encoding; + } + } + else { + encoding = scope_node->encoding; + } + + current_string = rb_enc_str_new(NULL, 0, encoding); + } + + PUSH_INSN1(ret, current_location, putobject, rb_fstring(current_string)); + PM_COMPILE_NOT_POPPED(part); + + const pm_line_column_t current_location = PM_NODE_START_LINE_COLUMN(scope_node->parser, part); + PUSH_INSN(ret, current_location, dup); + PUSH_INSN1(ret, current_location, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE, NULL, FALSE)); + PUSH_INSN(ret, current_location, anytostring); + + current_string = Qnil; + stack_size += 2; + } + } + } + + if (RTEST(current_string)) { + current_string = rb_fstring(current_string); + + if (stack_size == 0 && interpolated) { + PUSH_INSN1(ret, current_location, putstring, current_string); + } + else { + PUSH_INSN1(ret, current_location, putobject, current_string); + } + + current_string = Qnil; + stack_size++; + } + } + else { + PUSH_INSN(ret, *node_location, putnil); + } + + return stack_size; +} + static void pm_compile_regexp_dynamic(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_list_t *parts, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) { - int length = pm_interpolated_node_compile(iseq, parts, node_location, ret, popped, scope_node); + rb_encoding *explicit_regexp_encoding = parse_regexp_encoding(scope_node, node); + rb_encoding *implicit_regexp_encoding = explicit_regexp_encoding != NULL ? explicit_regexp_encoding : scope_node->encoding; + + int length = pm_interpolated_node_compile(iseq, parts, node_location, ret, popped, scope_node, implicit_regexp_encoding, explicit_regexp_encoding); PUSH_INSN2(ret, *node_location, toregexp, INT2FIX(parse_regexp_flags(node) & 0xFF), INT2FIX(length)); } @@ -693,13 +762,13 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } case PM_INTERPOLATED_STRING_NODE: { - VALUE string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, false); + VALUE string = pm_static_literal_concat(iseq, &((const pm_interpolated_string_node_t *) node)->parts, scope_node, NULL, NULL, false); int line_number = pm_node_line_number(scope_node->parser, node); return pm_static_literal_string(iseq, string, line_number); } case PM_INTERPOLATED_SYMBOL_NODE: { const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; - VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + VALUE string = pm_static_literal_concat(iseq, &cast->parts, scope_node, NULL, NULL, true); return ID2SYM(rb_intern_str(string)); } @@ -1654,7 +1723,7 @@ pm_compile_index_operator_write_node(rb_iseq_t *iseq, const pm_index_operator_wr PUSH_SEND_R(ret, location, idAREF, INT2FIX(argc), NULL, INT2FIX(flag & ~(VM_CALL_ARGS_SPLAT_MUT | VM_CALL_KW_SPLAT_MUT)), keywords); PM_COMPILE_NOT_POPPED(node->value); - ID id_operator = pm_constant_id_lookup(scope_node, node->operator); + ID id_operator = pm_constant_id_lookup(scope_node, node->binary_operator); PUSH_SEND(ret, location, id_operator, INT2FIX(1)); if (!popped) { @@ -2780,6 +2849,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ scope->encoding = previous->encoding; scope->filepath_encoding = previous->filepath_encoding; scope->constants = previous->constants; + scope->coverage_enabled = previous->coverage_enabled; } switch (PM_NODE_TYPE(node)) { @@ -3218,7 +3288,7 @@ pm_compile_defined_expr0(rb_iseq_t *iseq, const pm_node_t *node, const pm_line_c } case PM_CONSTANT_PATH_NODE: { const pm_constant_path_node_t *cast = (const pm_constant_path_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) cast->child)->name)); + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); if (cast->parent != NULL) { if (!lfinish[1]) lfinish[1] = NEW_LABEL(location.line); @@ -3514,8 +3584,11 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl if (rest->expression != NULL) { RUBY_ASSERT(PM_NODE_TYPE_P(rest->expression, PM_REQUIRED_PARAMETER_NODE)); - pm_insert_local_index(((const pm_required_parameter_node_t *) rest->expression)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); - local_index++; + + if (!PM_NODE_FLAG_P(rest->expression, PM_PARAMETER_FLAGS_REPEATED_PARAMETER)) { + pm_insert_local_index(((const pm_required_parameter_node_t *) rest->expression)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); + local_index++; + } } } @@ -3523,8 +3596,10 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl const pm_node_t *right = node->rights.nodes[index]; if (PM_NODE_TYPE_P(right, PM_REQUIRED_PARAMETER_NODE)) { - pm_insert_local_index(((const pm_required_parameter_node_t *) right)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); - local_index++; + if (!PM_NODE_FLAG_P(right, PM_PARAMETER_FLAGS_REPEATED_PARAMETER)) { + pm_insert_local_index(((const pm_required_parameter_node_t *) right)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); + local_index++; + } } else { RUBY_ASSERT(PM_NODE_TYPE_P(right, PM_MULTI_TARGET_NODE)); @@ -3717,11 +3792,11 @@ pm_multi_target_state_update(pm_multi_target_state_t *state) previous = current; current = current->next; - free(previous); + xfree(previous); } } -static size_t +static void pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const parents, LINK_ANCHOR *const writes, LINK_ANCHOR *const cleanup, pm_scope_node_t *scope_node, pm_multi_target_state_t *state); /** @@ -3829,7 +3904,7 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons // for I::J in []; end // const pm_constant_path_target_node_t *cast = (const pm_constant_path_target_node_t *) node; - ID name = pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) cast->child)->name); + ID name = pm_constant_id_lookup(scope_node, cast->name); if (cast->parent != NULL) { pm_compile_node(iseq, cast->parent, parents, false, scope_node); @@ -3868,6 +3943,13 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons pm_compile_node(iseq, cast->receiver, parents, false, scope_node); + LABEL *safe_label = NULL; + if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { + safe_label = NEW_LABEL(location.line); + PUSH_INSN(parents, location, dup); + PUSH_INSNL(parents, location, branchnil, safe_label); + } + if (state != NULL) { PUSH_INSN1(writes, location, topn, INT2FIX(1)); pm_multi_target_state_push(state, (INSN *) LAST_ELEMENT(writes), 1); @@ -3878,7 +3960,9 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY)) flags |= VM_CALL_FCALL; PUSH_SEND_WITH_FLAG(writes, location, method_id, INT2FIX(1), INT2FIX(flags)); + if (safe_label != NULL && state == NULL) PUSH_LABEL(writes, safe_label); PUSH_INSN(writes, location, pop); + if (safe_label != NULL && state != NULL) PUSH_LABEL(writes, safe_label); if (state != NULL) { PUSH_INSN(cleanup, location, pop); @@ -3968,7 +4052,7 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons * on the stack that correspond to the parent expressions of the various * targets. */ -static size_t +static void pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const parents, LINK_ANCHOR *const writes, LINK_ANCHOR *const cleanup, pm_scope_node_t *scope_node, pm_multi_target_state_t *state) { const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); @@ -4008,26 +4092,28 @@ pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR // going through the targets because we will need to revisit them once // we know how many values are being pushed onto the stack. pm_multi_target_state_t target_state = { 0 }; - size_t base_position = state == NULL ? 0 : state->position; + if (state == NULL) state = &target_state; + + size_t base_position = state->position; size_t splat_position = has_rest ? 1 : 0; // Next, we'll iterate through all of the leading targets. for (size_t index = 0; index < lefts->size; index++) { const pm_node_t *target = lefts->nodes[index]; - target_state.position = lefts->size - index + splat_position + base_position; - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + state->position = lefts->size - index + splat_position + base_position; + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } // Next, we'll compile the rest target if there is one. if (has_rest) { const pm_node_t *target = ((const pm_splat_node_t *) rest)->expression; - target_state.position = 1 + rights->size + base_position; + state->position = 1 + rights->size + base_position; if (has_posts) { PUSH_INSN2(writes, location, expandarray, INT2FIX(rights->size), INT2FIX(3)); } - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } // Finally, we'll compile the trailing targets. @@ -4038,18 +4124,10 @@ pm_compile_multi_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR for (size_t index = 0; index < rights->size; index++) { const pm_node_t *target = rights->nodes[index]; - target_state.position = rights->size - index + base_position; - pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, &target_state); + state->position = rights->size - index + base_position; + pm_compile_target_node(iseq, target, parents, writes, cleanup, scope_node, state); } } - - // Now, we need to go back and modify the topn instructions in order to - // ensure they can correctly retrieve the parent expressions. - pm_multi_target_state_update(&target_state); - - if (state != NULL) state->stack_size += target_state.stack_size; - - return target_state.stack_size; } /** @@ -4359,7 +4437,7 @@ pm_constant_path_parts(const pm_node_t *node, const pm_scope_node_t *scope_node) } case PM_CONSTANT_PATH_NODE: { const pm_constant_path_node_t *cast = (const pm_constant_path_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) cast->child)->name)); + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); rb_ary_unshift(parts, name); if (cast->parent == NULL) { @@ -4397,7 +4475,7 @@ pm_compile_constant_path(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *co } case PM_CONSTANT_PATH_NODE: { const pm_constant_path_node_t *cast = (const pm_constant_path_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) cast->child)->name)); + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); if (cast->parent == NULL) { PUSH_INSN(body, location, pop); @@ -4475,7 +4553,496 @@ pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t * return dispatch; } -/* +/** + * Return the object that will be pushed onto the stack for the given node. + */ +static VALUE +pm_compile_shareable_constant_literal(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_node_t *scope_node) +{ + switch (PM_NODE_TYPE(node)) { + case PM_TRUE_NODE: + case PM_FALSE_NODE: + case PM_NIL_NODE: + case PM_SYMBOL_NODE: + case PM_REGULAR_EXPRESSION_NODE: + case PM_SOURCE_LINE_NODE: + case PM_INTEGER_NODE: + case PM_FLOAT_NODE: + case PM_RATIONAL_NODE: + case PM_IMAGINARY_NODE: + case PM_SOURCE_ENCODING_NODE: + return pm_static_literal_value(iseq, node, scope_node); + case PM_STRING_NODE: + return parse_static_literal_string(iseq, scope_node, node, &((const pm_string_node_t *) node)->unescaped); + case PM_SOURCE_FILE_NODE: + return pm_source_file_value((const pm_source_file_node_t *) node, scope_node); + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + VALUE result = rb_ary_new_capa(cast->elements.size); + + for (size_t index = 0; index < cast->elements.size; index++) { + VALUE element = pm_compile_shareable_constant_literal(iseq, cast->elements.nodes[index], scope_node); + if (element == Qundef) return Qundef; + + rb_ary_push(result, element); + } + + return rb_ractor_make_shareable(result); + } + case PM_HASH_NODE: { + const pm_hash_node_t *cast = (const pm_hash_node_t *) node; + VALUE result = rb_hash_new_capa(cast->elements.size); + + for (size_t index = 0; index < cast->elements.size; index++) { + const pm_node_t *element = cast->elements.nodes[index]; + if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE)) return Qundef; + + const pm_assoc_node_t *assoc = (const pm_assoc_node_t *) element; + + VALUE key = pm_compile_shareable_constant_literal(iseq, assoc->key, scope_node); + if (key == Qundef) return Qundef; + + VALUE value = pm_compile_shareable_constant_literal(iseq, assoc->value, scope_node); + if (value == Qundef) return Qundef; + + rb_hash_aset(result, key, value); + } + + return rb_ractor_make_shareable(result); + } + default: + return Qundef; + } +} + +/** + * Compile the instructions for pushing the value that will be written to a + * shared constant. + */ +static void +pm_compile_shareable_constant_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_flags_t shareability, VALUE path, LINK_ANCHOR *const ret, pm_scope_node_t *scope_node, bool top) +{ + VALUE literal = pm_compile_shareable_constant_literal(iseq, node, scope_node); + if (literal != Qundef) { + const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); + PUSH_INSN1(ret, location, putobject, literal); + return; + } + + const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(scope_node->parser, node); + switch (PM_NODE_TYPE(node)) { + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + + if (top) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + } + + for (size_t index = 0; index < cast->elements.size; index++) { + pm_compile_shareable_constant_value(iseq, cast->elements.nodes[index], shareability, path, ret, scope_node, false); + } + + PUSH_INSN1(ret, location, newarray, INT2FIX(cast->elements.size)); + + if (top) { + ID method_id = (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) ? rb_intern("make_shareable_copy") : rb_intern("make_shareable"); + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + return; + } + case PM_HASH_NODE: { + const pm_hash_node_t *cast = (const pm_hash_node_t *) node; + + if (top) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + } + + for (size_t index = 0; index < cast->elements.size; index++) { + const pm_node_t *element = cast->elements.nodes[index]; + + if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE)) { + COMPILE_ERROR(ERROR_ARGS "Ractor constant writes do not support **"); + } + + const pm_assoc_node_t *assoc = (const pm_assoc_node_t *) element; + pm_compile_shareable_constant_value(iseq, assoc->key, shareability, path, ret, scope_node, false); + pm_compile_shareable_constant_value(iseq, assoc->value, shareability, path, ret, scope_node, false); + } + + PUSH_INSN1(ret, location, newhash, INT2FIX(cast->elements.size * 2)); + + if (top) { + ID method_id = (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) ? rb_intern("make_shareable_copy") : rb_intern("make_shareable"); + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + return; + } + default: { + DECL_ANCHOR(value_seq); + INIT_ANCHOR(value_seq); + + pm_compile_node(iseq, node, value_seq, false, scope_node); + if (PM_NODE_TYPE_P(node, PM_INTERPOLATED_STRING_NODE)) { + PUSH_SEND_WITH_FLAG(value_seq, location, idUMinus, INT2FIX(0), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL) { + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + PUSH_INSN1(ret, location, putobject, path); + PUSH_SEND_WITH_FLAG(ret, location, rb_intern("ensure_shareable"), INT2FIX(2), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + else if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY) { + if (top) PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + if (top) PUSH_SEND_WITH_FLAG(ret, location, rb_intern("make_shareable_copy"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + else if (shareability & PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING) { + if (top) PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PUSH_SEQ(ret, value_seq); + if (top) PUSH_SEND_WITH_FLAG(ret, location, rb_intern("make_shareable"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + break; + } + } +} + +/** + * Compile a constant write node, either in the context of a ractor pragma or + * not. + */ +static void +pm_compile_constant_write_node(rb_iseq_t *iseq, const pm_constant_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + ID name_id = pm_constant_id_lookup(scope_node, node->name); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, rb_id2str(name_id), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, ID2SYM(name_id)); +} + +/** + * Compile a constant and write node, either in the context of a ractor pragma + * or not. + */ +static void +pm_compile_constant_and_write_node(rb_iseq_t *iseq, const pm_constant_and_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + LABEL *end_label = NEW_LABEL(location.line); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSNL(ret, location, branchunless, end_label); + if (!popped) PUSH_INSN(ret, location, pop); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, end_label); +} + +/** + * Compile a constant or write node, either in the context of a ractor pragma or + * not. + */ +static void +pm_compile_constant_or_write_node(rb_iseq_t *iseq, const pm_constant_or_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + + LABEL *set_label = NEW_LABEL(location.line); + LABEL *end_label = NEW_LABEL(location.line); + + PUSH_INSN(ret, location, putnil); + PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST), name, Qtrue); + PUSH_INSNL(ret, location, branchunless, set_label); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSNL(ret, location, branchif, end_label); + if (!popped) PUSH_INSN(ret, location, pop); + PUSH_LABEL(ret, set_label); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, end_label); +} + +/** + * Compile a constant operator write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_operator_write_node(rb_iseq_t *iseq, const pm_constant_operator_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, node->name)); + ID method_id = pm_constant_id_lookup(scope_node, node->binary_operator); + + pm_compile_constant_read(iseq, name, &node->name_loc, ret, scope_node); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, name, ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + if (!popped) PUSH_INSN(ret, location, dup); + + PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + PUSH_INSN1(ret, location, setconstant, name); +} + +/** + * Creates a string that is used in ractor error messages to describe the + * constant path being written. + */ +static VALUE +pm_constant_path_path(const pm_constant_path_node_t *node, const pm_scope_node_t *scope_node) +{ + VALUE parts = rb_ary_new(); + rb_ary_push(parts, rb_id2str(pm_constant_id_lookup(scope_node, node->name))); + + const pm_node_t *current = node->parent; + while (current != NULL && PM_NODE_TYPE_P(current, PM_CONSTANT_PATH_NODE)) { + const pm_constant_path_node_t *cast = (const pm_constant_path_node_t *) current; + rb_ary_unshift(parts, rb_id2str(pm_constant_id_lookup(scope_node, cast->name))); + current = cast->parent; + } + + if (current == NULL) { + rb_ary_unshift(parts, rb_id2str(idNULL)); + } + else if (PM_NODE_TYPE_P(current, PM_CONSTANT_READ_NODE)) { + rb_ary_unshift(parts, rb_id2str(pm_constant_id_lookup(scope_node, ((const pm_constant_read_node_t *) current)->name))); + } + else { + rb_ary_unshift(parts, rb_str_new_cstr("...")); + } + + return rb_ary_join(parts, rb_str_new_cstr("::")); +} + +/** + * Compile a constant path write node, either in the context of a ractor pragma + * or not. + */ +static void +pm_compile_constant_path_write_node(rb_iseq_t *iseq, const pm_constant_path_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + + if (target->parent) { + PM_COMPILE_NOT_POPPED((const pm_node_t *) target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (!popped) { + PUSH_INSN(ret, location, swap); + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + + PUSH_INSN(ret, location, swap); + PUSH_INSN1(ret, location, setconstant, name); +} + +/** + * Compile a constant path and write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_path_and_write_node(rb_iseq_t *iseq, const pm_constant_path_and_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + LABEL *lfin = NEW_LABEL(location.line); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSNL(ret, location, branchunless, lfin); + + if (!popped) PUSH_INSN(ret, location, pop); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + else { + PUSH_INSN1(ret, location, dupn, INT2FIX(2)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, lfin); + + if (!popped) PUSH_INSN(ret, location, swap); + PUSH_INSN(ret, location, pop); +} + +/** + * Compile a constant path or write node, either in the context of a ractor + * pragma or not. + */ +static void +pm_compile_constant_path_or_write_node(rb_iseq_t *iseq, const pm_constant_path_or_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + LABEL *lassign = NEW_LABEL(location.line); + LABEL *lfin = NEW_LABEL(location.line); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST_FROM), name, Qtrue); + PUSH_INSNL(ret, location, branchunless, lassign); + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (!popped) PUSH_INSN(ret, location, dup); + PUSH_INSNL(ret, location, branchif, lfin); + + if (!popped) PUSH_INSN(ret, location, pop); + PUSH_LABEL(ret, lassign); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + if (popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + } + else { + PUSH_INSN1(ret, location, dupn, INT2FIX(2)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); + PUSH_LABEL(ret, lfin); + + if (!popped) PUSH_INSN(ret, location, swap); + PUSH_INSN(ret, location, pop); +} + +/** + * Compile a constant path operator write node, either in the context of a + * ractor pragma or not. + */ +static void +pm_compile_constant_path_operator_write_node(rb_iseq_t *iseq, const pm_constant_path_operator_write_node_t *node, const pm_node_flags_t shareability, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + const pm_line_column_t location = *node_location; + const pm_constant_path_node_t *target = node->target; + + ID method_id = pm_constant_id_lookup(scope_node, node->binary_operator); + VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, target->name)); + + if (target->parent) { + PM_COMPILE_NOT_POPPED(target->parent); + } + else { + PUSH_INSN1(ret, location, putobject, rb_cObject); + } + + PUSH_INSN(ret, location, dup); + PUSH_INSN1(ret, location, putobject, Qtrue); + PUSH_INSN1(ret, location, getconstant, name); + + if (shareability != 0) { + pm_compile_shareable_constant_value(iseq, node->value, shareability, pm_constant_path_path(node->target, scope_node), ret, scope_node, true); + } + else { + PM_COMPILE_NOT_POPPED(node->value); + } + + PUSH_CALL(ret, location, method_id, INT2FIX(1)); + PUSH_INSN(ret, location, swap); + + if (!popped) { + PUSH_INSN1(ret, location, topn, INT2FIX(1)); + PUSH_INSN(ret, location, swap); + } + + PUSH_INSN1(ret, location, setconstant, name); +} + +/** * Compiles a prism node into instruction sequences. * * iseq - The current instruction sequence object (used for locals) @@ -4492,14 +5059,16 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, const pm_line_column_t location = PM_NODE_START_LINE_COLUMN(parser, node); int lineno = (int) location.line; - if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_NEWLINE) && ISEQ_COMPILE_DATA(iseq)->last_line != lineno) { - int event = RUBY_EVENT_LINE; + if (!PM_NODE_TYPE_P(node, PM_RETURN_NODE) || !PM_NODE_FLAG_P(node, PM_RETURN_NODE_FLAGS_REDUNDANT) || ((const pm_return_node_t *) node)->arguments != NULL) { + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_NEWLINE) && ISEQ_COMPILE_DATA(iseq)->last_line != lineno) { + int event = RUBY_EVENT_LINE; - ISEQ_COMPILE_DATA(iseq)->last_line = lineno; - if (ISEQ_COVERAGE(iseq) && ISEQ_LINE_COVERAGE(iseq)) { - event |= RUBY_EVENT_COVERAGE_LINE; + ISEQ_COMPILE_DATA(iseq)->last_line = lineno; + if (ISEQ_COVERAGE(iseq) && ISEQ_LINE_COVERAGE(iseq)) { + event |= RUBY_EVENT_COVERAGE_LINE; + } + PUSH_TRACE(ret, event); } - PUSH_TRACE(ret, event); } switch (PM_NODE_TYPE(node)) { @@ -4827,7 +5396,6 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } COMPILE_ERROR(ERROR_ARGS "Invalid break"); - rb_bug("Invalid break"); } return; } @@ -4963,7 +5531,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_SEND_WITH_FLAG(ret, location, id_read_name, INT2FIX(0), INT2FIX(flag)); PM_COMPILE_NOT_POPPED(cast->value); - ID id_operator = pm_constant_id_lookup(scope_node, cast->operator); + ID id_operator = pm_constant_id_lookup(scope_node, cast->binary_operator); PUSH_SEND(ret, location, id_operator, INT2FIX(1)); if (!popped) { @@ -5008,7 +5576,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (cast->predicate == NULL) { // Establish branch coverage for the case node. VALUE branches = Qfalse; - rb_code_location_t case_location; + rb_code_location_t case_location = { 0 }; int branch_id = 0; if (PM_BRANCH_COVERAGE_P(iseq)) { @@ -5095,7 +5663,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, else { // Establish branch coverage for the case node. VALUE branches = Qfalse; - rb_code_location_t case_location; + rb_code_location_t case_location = { 0 }; int branch_id = 0; if (PM_BRANCH_COVERAGE_P(iseq)) { @@ -5229,7 +5797,9 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN(ret, location, pop); // Establish branch coverage for the implicit else clause. - add_trace_branch_coverage(iseq, ret, &case_location, case_location.beg_pos.column, branch_id, "else", branches); + if (PM_BRANCH_COVERAGE_P(iseq)) { + add_trace_branch_coverage(iseq, ret, &case_location, case_location.beg_pos.column, branch_id, "else", branches); + } if (!popped) PUSH_INSN(ret, location, putnil); PUSH_INSNL(ret, location, jump, end_label); @@ -5465,7 +6035,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, getclassvariable, name, get_cvar_ic_value(iseq, name_id)); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -5559,154 +6129,28 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Foo::Bar &&= baz // ^^^^^^^^^^^^^^^^ const pm_constant_path_and_write_node_t *cast = (const pm_constant_path_and_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - - const pm_constant_read_node_t *child = (const pm_constant_read_node_t *) target->child; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, child->name)); - LABEL *lfin = NEW_LABEL(location.line); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - if (!popped) PUSH_INSN(ret, location, dup); - PUSH_INSNL(ret, location, branchunless, lfin); - - if (!popped) PUSH_INSN(ret, location, pop); - PM_COMPILE_NOT_POPPED(cast->value); - - if (popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - else { - PUSH_INSN1(ret, location, dupn, INT2FIX(2)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, lfin); - - if (!popped) PUSH_INSN(ret, location, swap); - PUSH_INSN(ret, location, pop); - + pm_compile_constant_path_and_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_OR_WRITE_NODE: { // Foo::Bar ||= baz // ^^^^^^^^^^^^^^^^ const pm_constant_path_or_write_node_t *cast = (const pm_constant_path_or_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - - const pm_constant_read_node_t *child = (const pm_constant_read_node_t *) target->child; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, child->name)); - - LABEL *lassign = NEW_LABEL(location.line); - LABEL *lfin = NEW_LABEL(location.line); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST_FROM), name, Qtrue); - PUSH_INSNL(ret, location, branchunless, lassign); - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - if (!popped) PUSH_INSN(ret, location, dup); - PUSH_INSNL(ret, location, branchif, lfin); - - if (!popped) PUSH_INSN(ret, location, pop); - PUSH_LABEL(ret, lassign); - PM_COMPILE_NOT_POPPED(cast->value); - - if (popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - else { - PUSH_INSN1(ret, location, dupn, INT2FIX(2)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, lfin); - - if (!popped) PUSH_INSN(ret, location, swap); - PUSH_INSN(ret, location, pop); - + pm_compile_constant_path_or_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_OPERATOR_WRITE_NODE: { // Foo::Bar += baz // ^^^^^^^^^^^^^^^ const pm_constant_path_operator_write_node_t *cast = (const pm_constant_path_operator_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); - - const pm_constant_read_node_t *child = (const pm_constant_read_node_t *) target->child; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, child->name)); - - if (target->parent) { - PM_COMPILE_NOT_POPPED(target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PUSH_INSN(ret, location, dup); - PUSH_INSN1(ret, location, putobject, Qtrue); - PUSH_INSN1(ret, location, getconstant, name); - - PM_COMPILE_NOT_POPPED(cast->value); - PUSH_CALL(ret, location, method_id, INT2FIX(1)); - PUSH_INSN(ret, location, swap); - - if (!popped) { - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - PUSH_INSN(ret, location, swap); - } - - PUSH_INSN1(ret, location, setconstant, name); + pm_compile_constant_path_operator_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_PATH_WRITE_NODE: { // Foo::Bar = 1 // ^^^^^^^^^^^^ const pm_constant_path_write_node_t *cast = (const pm_constant_path_write_node_t *) node; - const pm_constant_path_node_t *target = cast->target; - - const pm_constant_read_node_t *child = (const pm_constant_read_node_t *) target->child; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, child->name)); - - if (target->parent) { - PM_COMPILE_NOT_POPPED((const pm_node_t *) target->parent); - } - else { - PUSH_INSN1(ret, location, putobject, rb_cObject); - } - - PM_COMPILE_NOT_POPPED(cast->value); - - if (!popped) { - PUSH_INSN(ret, location, swap); - PUSH_INSN1(ret, location, topn, INT2FIX(1)); - } - - PUSH_INSN(ret, location, swap); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_path_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_READ_NODE: { @@ -5724,82 +6168,28 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Foo &&= bar // ^^^^^^^^^^^ const pm_constant_and_write_node_t *cast = (const pm_constant_and_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - LABEL *end_label = NEW_LABEL(location.line); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSNL(ret, location, branchunless, end_label); - if (!popped) PUSH_INSN(ret, location, pop); - - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, end_label); - + pm_compile_constant_and_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_OR_WRITE_NODE: { // Foo ||= bar // ^^^^^^^^^^^ const pm_constant_or_write_node_t *cast = (const pm_constant_or_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - LABEL *set_label = NEW_LABEL(location.line); - LABEL *end_label = NEW_LABEL(location.line); - - PUSH_INSN(ret, location, putnil); - PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_CONST), name, Qtrue); - PUSH_INSNL(ret, location, branchunless, set_label); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSNL(ret, location, branchif, end_label); - if (!popped) PUSH_INSN(ret, location, pop); - - PUSH_LABEL(ret, set_label); - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - PUSH_LABEL(ret, end_label); - + pm_compile_constant_or_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_OPERATOR_WRITE_NODE: { // Foo += bar // ^^^^^^^^^^ const pm_constant_operator_write_node_t *cast = (const pm_constant_operator_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); - - pm_compile_constant_read(iseq, name, &cast->name_loc, ret, scope_node); - PM_COMPILE_NOT_POPPED(cast->value); - - PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_operator_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_CONSTANT_WRITE_NODE: { // Foo = 1 // ^^^^^^^ const pm_constant_write_node_t *cast = (const pm_constant_write_node_t *) node; - VALUE name = ID2SYM(pm_constant_id_lookup(scope_node, cast->name)); - - PM_COMPILE_NOT_POPPED(cast->value); - if (!popped) PUSH_INSN(ret, location, dup); - - PUSH_INSN1(ret, location, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); - PUSH_INSN1(ret, location, setconstant, name); - + pm_compile_constant_write_node(iseq, cast, 0, &location, ret, popped, scope_node); return; } case PM_DEF_NODE: { @@ -6136,7 +6526,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN1(ret, location, getglobal, name); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -6334,7 +6724,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, getinstancevariable, name, get_ivar_ic_value(iseq, name_id)); PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); int flags = VM_CALL_ARGS_SIMPLE; PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(flags)); @@ -6470,7 +6860,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } else { const pm_interpolated_string_node_t *cast = (const pm_interpolated_string_node_t *) node; - int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, popped, scope_node); + int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, popped, scope_node, NULL, NULL); if (length > 1) PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); if (popped) PUSH_INSN(ret, location, pop); } @@ -6489,7 +6879,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } } else { - int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, popped, scope_node); + int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, popped, scope_node, NULL, NULL); if (length > 1) { PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); } @@ -6511,7 +6901,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN(ret, location, putself); - int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, false, scope_node); + int length = pm_interpolated_node_compile(iseq, &cast->parts, &location, ret, false, scope_node, NULL, NULL); if (length > 1) PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); PUSH_SEND_WITH_FLAG(ret, location, idBackquote, INT2NUM(1), INT2FIX(VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE)); @@ -6519,6 +6909,15 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } + case PM_IT_LOCAL_VARIABLE_READ_NODE: { + // -> { it } + // ^^ + if (!popped) { + PUSH_GETLOCAL(ret, location, scope_node->local_table_for_iseq_size, 0); + } + + return; + } case PM_KEYWORD_HASH_NODE: { // foo(bar: baz) // ^^^^^^^^ @@ -6584,7 +6983,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE_NOT_POPPED(cast->value); - ID method_id = pm_constant_id_lookup(scope_node, cast->operator); + ID method_id = pm_constant_id_lookup(scope_node, cast->binary_operator); PUSH_SEND_WITH_FLAG(ret, location, method_id, INT2NUM(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); if (!popped) PUSH_INSN(ret, location, dup); @@ -6622,9 +7021,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_LOCAL_VARIABLE_READ_NODE: { // foo // ^^^ - const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) node; - if (!popped) { + const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) node; pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, cast->name, cast->depth); PUSH_GETLOCAL(ret, location, index.index, index.level); } @@ -6885,18 +7283,22 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_multi_target_state_t state = { 0 }; state.position = popped ? 0 : 1; - size_t stack_size = pm_compile_multi_target_node(iseq, node, ret, writes, cleanup, scope_node, &state); + pm_compile_multi_target_node(iseq, node, ret, writes, cleanup, scope_node, &state); PM_COMPILE_NOT_POPPED(cast->value); if (!popped) PUSH_INSN(ret, location, dup); PUSH_SEQ(ret, writes); - if (!popped && stack_size >= 1) { + if (!popped && state.stack_size >= 1) { // Make sure the value on the right-hand side of the = operator is // being returned before we pop the parent expressions. - PUSH_INSN1(ret, location, setn, INT2FIX(stack_size)); + PUSH_INSN1(ret, location, setn, INT2FIX(state.stack_size)); } + // Now, we need to go back and modify the topn instructions in order to + // ensure they can correctly retrieve the parent expressions. + pm_multi_target_state_update(&state); + PUSH_SEQ(ret, cleanup); return; } @@ -7566,6 +7968,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } } + // If we have the `it` implicit local variable, we need to account for + // it in the local table size. + if (scope_node->parameters != NULL && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { + table_size++; + } + // Ensure there is enough room in the local table for any // parameters that have been repeated // ex: def underscore_parameters(_, _ = 1, _ = 2); _; end @@ -7714,6 +8122,11 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, body->param.flags.has_lead = true; } + if (scope_node->parameters != NULL && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { + ID local = rb_make_temporary_id(local_index); + local_table_for_iseq->ids[local_index++] = local; + } + // def foo(a, (b, *c, d), e = 1, *f, g, (h, *i, j), k:, l: 1, **m, &n) // ^^^^^ if (optionals_list && optionals_list->size) { @@ -8074,18 +8487,6 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, body->param.flags.has_lead = true; } - // Fill in the it variable, if it exists - if (scope_node->parameters && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { - const uint8_t param_name[] = { '0', 'i', 't' }; - pm_constant_id_t constant_id = pm_constant_pool_find(&parser->constant_pool, param_name, 3); - RUBY_ASSERT(constant_id && "parser should have inserted 0it for 'it' local"); - - ID local = rb_make_temporary_id(local_index); - local_table_for_iseq->ids[local_index] = local; - st_insert(index_lookup_table, (st_data_t) constant_id, (st_data_t) local_index); - local_index++; - } - //********END OF STEP 3********** //********STEP 4********** @@ -8344,7 +8745,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } - if (PM_NODE_TYPE_P(scope_node->ast_node, PM_CLASS_NODE)) { + if (PM_NODE_TYPE_P(scope_node->ast_node, PM_CLASS_NODE) || PM_NODE_TYPE_P(scope_node->ast_node, PM_MODULE_NODE)) { const pm_line_column_t end_location = PM_NODE_END_LINE_COLUMN(scope_node->parser, scope_node->ast_node); ADD_TRACE(ret, RUBY_EVENT_END); ISEQ_COMPILE_DATA(iseq)->last_line = end_location.line; @@ -8368,7 +8769,38 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_SHAREABLE_CONSTANT_NODE: { // A value that is being written to a constant that is being marked as // shared depending on the current lexical context. - PM_COMPILE(((const pm_shareable_constant_node_t *) node)->write); + const pm_shareable_constant_node_t *cast = (const pm_shareable_constant_node_t *) node; + + switch (PM_NODE_TYPE(cast->write)) { + case PM_CONSTANT_WRITE_NODE: + pm_compile_constant_write_node(iseq, (const pm_constant_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_AND_WRITE_NODE: + pm_compile_constant_and_write_node(iseq, (const pm_constant_and_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_OR_WRITE_NODE: + pm_compile_constant_or_write_node(iseq, (const pm_constant_or_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_OPERATOR_WRITE_NODE: + pm_compile_constant_operator_write_node(iseq, (const pm_constant_operator_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_WRITE_NODE: + pm_compile_constant_path_write_node(iseq, (const pm_constant_path_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_AND_WRITE_NODE: + pm_compile_constant_path_and_write_node(iseq, (const pm_constant_path_and_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_OR_WRITE_NODE: + pm_compile_constant_path_or_write_node(iseq, (const pm_constant_path_or_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + case PM_CONSTANT_PATH_OPERATOR_WRITE_NODE: + pm_compile_constant_path_operator_write_node(iseq, (const pm_constant_path_operator_write_node_t *) cast->write, cast->base.flags, &location, ret, popped, scope_node); + break; + default: + rb_bug("Unexpected node type for shareable constant write: %s", pm_node_type_to_str(PM_NODE_TYPE(cast->write))); + break; + } + return; } case PM_SINGLETON_CLASS_NODE: { @@ -8727,6 +9159,368 @@ pm_parse_result_free(pm_parse_result_t *result) pm_options_free(&result->options); } +/** An error that is going to be formatted into the output. */ +typedef struct { + /** A pointer to the diagnostic that was generated during parsing. */ + pm_diagnostic_t *error; + + /** The start line of the diagnostic message. */ + int32_t line; + + /** The column start of the diagnostic message. */ + uint32_t column_start; + + /** The column end of the diagnostic message. */ + uint32_t column_end; +} pm_parse_error_t; + +/** The format that will be used to format the errors into the output. */ +typedef struct { + /** The prefix that will be used for line numbers. */ + const char *number_prefix; + + /** The prefix that will be used for blank lines. */ + const char *blank_prefix; + + /** The divider that will be used between sections of source code. */ + const char *divider; + + /** The length of the blank prefix. */ + size_t blank_prefix_length; + + /** The length of the divider. */ + size_t divider_length; +} pm_parse_error_format_t; + +#define PM_COLOR_GRAY "\033[38;5;102m" +#define PM_COLOR_RED "\033[1;31m" +#define PM_COLOR_RESET "\033[m" +#define PM_ERROR_TRUNCATE 30 + +static inline pm_parse_error_t * +pm_parse_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { + pm_parse_error_t *errors = xcalloc(error_list->size, sizeof(pm_parse_error_t)); + if (errors == NULL) return NULL; + + int32_t start_line = parser->start_line; + for (pm_diagnostic_t *error = (pm_diagnostic_t *) error_list->head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { + pm_line_column_t start = pm_newline_list_line_column(newline_list, error->location.start, start_line); + pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.end, start_line); + + // We're going to insert this error into the array in sorted order. We + // do this by finding the first error that has a line number greater + // than the current error and then inserting the current error before + // that one. + size_t index = 0; + while ( + (index < error_list->size) && + (errors[index].error != NULL) && + ( + (errors[index].line < start.line) || + ((errors[index].line == start.line) && (errors[index].column_start < start.column)) + ) + ) index++; + + // Now we're going to shift all of the errors after this one down one + // index to make room for the new error. + if (index + 1 < error_list->size) { + memmove(&errors[index + 1], &errors[index], sizeof(pm_parse_error_t) * (error_list->size - index - 1)); + } + + // Finally, we'll insert the error into the array. + uint32_t column_end; + if (start.line == end.line) { + column_end = end.column; + } else { + column_end = (uint32_t) (newline_list->offsets[start.line - start_line + 1] - newline_list->offsets[start.line - start_line] - 1); + } + + // Ensure we have at least one column of error. + if (start.column == column_end) column_end++; + + errors[index] = (pm_parse_error_t) { + .error = error, + .line = start.line, + .column_start = start.column, + .column_end = column_end + }; + } + + return errors; +} + +static inline void +pm_parse_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, uint32_t column_start, uint32_t column_end, pm_buffer_t *buffer) { + int32_t line_delta = line - parser->start_line; + assert(line_delta >= 0); + + size_t index = (size_t) line_delta; + assert(index < newline_list->size); + + const uint8_t *start = &parser->start[newline_list->offsets[index]]; + const uint8_t *end; + + if (index >= newline_list->size - 1) { + end = parser->end; + } else { + end = &parser->start[newline_list->offsets[index + 1]]; + } + + pm_buffer_append_format(buffer, number_prefix, line); + + // Here we determine if we should truncate the end of the line. + bool truncate_end = false; + if ((column_end != 0) && ((end - (start + column_end)) >= PM_ERROR_TRUNCATE)) { + end = start + column_end + PM_ERROR_TRUNCATE; + truncate_end = true; + } + + // Here we determine if we should truncate the start of the line. + if (column_start >= PM_ERROR_TRUNCATE) { + pm_buffer_append_string(buffer, "... ", 4); + start += column_start; + } + + pm_buffer_append_string(buffer, (const char *) start, (size_t) (end - start)); + + if (truncate_end) { + pm_buffer_append_string(buffer, " ...\n", 5); + } else if (end == parser->end && end[-1] != '\n') { + pm_buffer_append_string(buffer, "\n", 1); + } +} + +/** + * Format the errors on the parser into the given buffer. + */ +static void +pm_parse_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { + assert(error_list->size != 0); + + // First, we're going to sort all of the errors by line number using an + // insertion sort into a newly allocated array. + const int32_t start_line = parser->start_line; + const pm_newline_list_t *newline_list = &parser->newline_list; + + pm_parse_error_t *errors = pm_parse_errors_format_sort(parser, error_list, newline_list); + if (errors == NULL) return; + + // Now we're going to determine how we're going to format line numbers and + // blank lines based on the maximum number of digits in the line numbers + // that are going to be displaid. + pm_parse_error_format_t error_format; + int32_t first_line_number = errors[0].line; + int32_t last_line_number = errors[error_list->size - 1].line; + + // If we have a maximum line number that is negative, then we're going to + // use the absolute value for comparison but multiple by 10 to additionally + // have a column for the negative sign. + if (first_line_number < 0) first_line_number = (-first_line_number) * 10; + if (last_line_number < 0) last_line_number = (-last_line_number) * 10; + int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; + + if (max_line_number < 10) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%1" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%1" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~\n" + }; + } + } else if (max_line_number < 100) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%2" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%2" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~\n" + }; + } + } else if (max_line_number < 1000) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%3" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%3" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~\n" + }; + } + } else if (max_line_number < 10000) { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%4" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%4" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~~\n" + }; + } + } else { + if (colorize) { + error_format = (pm_parse_error_format_t) { + .number_prefix = PM_COLOR_GRAY "%5" PRIi32 " | " PM_COLOR_RESET, + .blank_prefix = PM_COLOR_GRAY " | " PM_COLOR_RESET, + .divider = PM_COLOR_GRAY " ~~~~~~~~" PM_COLOR_RESET "\n" + }; + } else { + error_format = (pm_parse_error_format_t) { + .number_prefix = "%5" PRIi32 " | ", + .blank_prefix = " | ", + .divider = " ~~~~~~~~\n" + }; + } + } + + error_format.blank_prefix_length = strlen(error_format.blank_prefix); + error_format.divider_length = strlen(error_format.divider); + + // Now we're going to iterate through every error in our error list and + // display it. While we're iterating, we will display some padding lines of + // the source before the error to give some context. We'll be careful not to + // display the same line twice in case the errors are close enough in the + // source. + int32_t last_line = parser->start_line - 1; + uint32_t last_column_start = 0; + const pm_encoding_t *encoding = parser->encoding; + + for (size_t index = 0; index < error_list->size; index++) { + pm_parse_error_t *error = &errors[index]; + + // Here we determine how many lines of padding of the source to display, + // based on the difference from the last line that was displaid. + if (error->line - last_line > 1) { + if (error->line - last_line > 2) { + if ((index != 0) && (error->line - last_line > 3)) { + pm_buffer_append_string(buffer, error_format.divider, error_format.divider_length); + } + + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 2, 0, 0, buffer); + } + + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line - 1, 0, 0, buffer); + } + + // If this is the first error or we're on a new line, then we'll display + // the line that has the error in it. + if ((index == 0) || (error->line != last_line)) { + if (colorize) { + pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); + } else { + pm_buffer_append_string(buffer, "> ", 2); + } + + last_column_start = error->column_start; + + // Find the maximum column end of all the errors on this line. + uint32_t column_end = error->column_end; + for (size_t next_index = index + 1; next_index < error_list->size; next_index++) { + if (errors[next_index].line != error->line) break; + if (errors[next_index].column_end > column_end) column_end = errors[next_index].column_end; + } + + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, error->column_start, column_end, buffer); + } + + const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; + if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); + + // Now we'll display the actual error message. We'll do this by first + // putting the prefix to the line, then a bunch of blank spaces + // depending on the column, then as many carets as we need to display + // the width of the error, then the error message itself. + // + // Note that this doesn't take into account the width of the actual + // character when displaid in the terminal. For some east-asian + // languages or emoji, this means it can be thrown off pretty badly. We + // will need to solve this eventually. + pm_buffer_append_string(buffer, " ", 2); + pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); + + size_t column = 0; + if (last_column_start >= PM_ERROR_TRUNCATE) { + pm_buffer_append_string(buffer, " ", 4); + column = last_column_start; + } + + while (column < error->column_start) { + pm_buffer_append_byte(buffer, ' '); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + } + + if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RED, 7); + pm_buffer_append_byte(buffer, '^'); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + + while (column < error->column_end) { + pm_buffer_append_byte(buffer, '~'); + + size_t char_width = encoding->char_width(start + column, parser->end - (start + column)); + column += (char_width == 0 ? 1 : char_width); + } + + if (colorize) pm_buffer_append_string(buffer, PM_COLOR_RESET, 3); + + if (inline_messages) { + pm_buffer_append_byte(buffer, ' '); + assert(error->error != NULL); + + const char *message = error->error->message; + pm_buffer_append_string(buffer, message, strlen(message)); + } + + pm_buffer_append_byte(buffer, '\n'); + + // Here we determine how many lines of padding to display after the + // error, depending on where the next error is in source. + last_line = error->line; + int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; + + if (next_line - last_line > 1) { + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, 0, 0, buffer); + } + + if (next_line - last_line > 1) { + pm_buffer_append_string(buffer, " ", 2); + pm_parse_errors_format_line(parser, newline_list, error_format.number_prefix, ++last_line, 0, 0, buffer); + } + } + + // Finally, we'll free the array of errors that we allocated. + xfree(errors); +} + +#undef PM_ERROR_TRUNCATE +#undef PM_COLOR_GRAY +#undef PM_COLOR_RED +#undef PM_COLOR_RESET + /** * Check if the given source slice is valid UTF-8. The location represents the * location of the error, but the slice of the source will include the content @@ -8798,7 +9592,7 @@ pm_parse_process_error(const pm_parse_result_t *result) pm_list_node_t *list_node = (pm_list_node_t *) error; pm_list_t error_list = { .size = 1, .head = list_node, .tail = list_node }; - pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); + pm_parse_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); } VALUE value = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); @@ -8828,7 +9622,7 @@ pm_parse_process_error(const pm_parse_result_t *result) ); if (valid_utf8) { - pm_parser_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); + pm_parse_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); } else { for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { @@ -8837,7 +9631,8 @@ pm_parse_process_error(const pm_parse_result_t *result) } } - VALUE error = rb_exc_new(rb_eSyntaxError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + VALUE message = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), result->node.encoding); + VALUE error = rb_exc_new_str(rb_eSyntaxError, message); rb_encoding *filepath_encoding = result->node.filepath_encoding != NULL ? result->node.filepath_encoding : rb_utf8_encoding(); VALUE path = rb_enc_str_new((const char *) pm_string_source(filepath), pm_string_length(filepath), filepath_encoding); @@ -8862,10 +9657,16 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // freed regardless of whether or we return an error. pm_scope_node_t *scope_node = &result->node; rb_encoding *filepath_encoding = scope_node->filepath_encoding; + int coverage_enabled = scope_node->coverage_enabled; pm_scope_node_init(node, scope_node, NULL); scope_node->filepath_encoding = filepath_encoding; + scope_node->encoding = rb_enc_find(parser->encoding->name); + if (!scope_node->encoding) rb_bug("Encoding not found %s!", parser->encoding->name); + + scope_node->coverage_enabled = coverage_enabled; + // Emit all of the various warnings from the parse. const pm_diagnostic_t *warning; const char *warning_filepath = (const char *) pm_string_source(&parser->filepath); @@ -8874,10 +9675,10 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) int line = pm_location_line_number(parser, &warning->location); if (warning->level == PM_WARNING_LEVEL_VERBOSE) { - rb_compile_warning(warning_filepath, line, "%s", warning->message); + rb_enc_compile_warning(scope_node->encoding, warning_filepath, line, "%s", warning->message); } else { - rb_compile_warn(warning_filepath, line, "%s", warning->message); + rb_enc_compile_warn(scope_node->encoding, warning_filepath, line, "%s", warning->message); } } @@ -8892,9 +9693,6 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // Now set up the constant pool and intern all of the various constants into // their corresponding IDs. - scope_node->encoding = rb_enc_find(parser->encoding->name); - if (!scope_node->encoding) rb_bug("Encoding not found %s!", parser->encoding->name); - scope_node->parser = parser; scope_node->constants = calloc(parser->constant_pool.size, sizeof(ID)); @@ -9018,6 +9816,7 @@ pm_load_file(pm_parse_result_t *result, VALUE filepath, bool load_error) VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath) { + result->node.filepath_encoding = rb_enc_get(filepath); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); diff --git a/prism_compile.h b/prism_compile.h index 0f82782ec0..8df82c5c7c 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -36,14 +36,16 @@ typedef struct pm_scope_node { */ rb_encoding *filepath_encoding; - // The size of the local table - // on the iseq which includes - // locals and hidden variables + // The size of the local table on the iseq which includes locals and hidden + // variables. int local_table_for_iseq_size; ID *constants; st_table *index_lookup_table; + // The current coverage setting, passed down through the various scopes. + int coverage_enabled; + /** * This will only be set on the top-level scope node. It will contain all of * the instructions pertaining to BEGIN{} nodes. @@ -1804,17 +1804,17 @@ ractor_select_internal(rb_execution_context_t *ec, VALUE self, VALUE ractors, VA int state; EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG() == TAG_NONE)) { + if ((state = EC_EXEC_TAG()) == TAG_NONE) { result = ractor_selector__wait(selector, do_receive, do_yield, yield_value, move); } - else { + EC_POP_TAG(); + if (state != TAG_NONE) { // ensure ractor_selector_clear(selector); // jump EC_JUMP_TAG(ec, state); } - EC_POP_TAG(); RB_GC_GUARD(ractors); return result; @@ -519,6 +519,7 @@ extern VALUE rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_fram extern VALUE rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); extern VALUE rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); extern VALUE rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr); +extern VALUE rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr, VALUE fmt); extern VALUE rb_vm_splat_array(VALUE flag, VALUE array); extern bool rb_simple_iseq_p(const rb_iseq_t *iseq); extern bool rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v); @@ -691,6 +691,10 @@ module RubyVM::RJIT # :nodoc: all Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_opt_newarray_min) } end + def C.rb_vm_opt_newarray_pack + Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_opt_newarray_pack) } + end + def C.rb_vm_set_ivar_id Primitive.cexpr! %q{ SIZET2NUM((size_t)rb_vm_set_ivar_id) } end @@ -2060,7 +2060,7 @@ static VALUE process_script(ruby_cmdline_options_t *opt) { rb_ast_t *ast; - VALUE vast; + VALUE ast_value; VALUE parser = rb_parser_new(); const unsigned int dump = opt->dump; @@ -2080,7 +2080,7 @@ process_script(ruby_cmdline_options_t *opt) ruby_set_script_name(progname); rb_parser_set_options(parser, opt->do_print, opt->do_loop, opt->do_line, opt->do_split); - vast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1); + ast_value = rb_parser_compile_string(parser, opt->script, opt->e_script, 1); } else { VALUE f; @@ -2088,14 +2088,14 @@ process_script(ruby_cmdline_options_t *opt) f = open_load_file(opt->script_name, &xflag); opt->xflag = xflag != 0; rb_parser_set_context(parser, 0, f == rb_stdin); - vast = load_file(parser, opt->script_name, f, 1, opt); + ast_value = load_file(parser, opt->script_name, f, 1, opt); } - ast = rb_ruby_ast_data_get(vast); + ast = rb_ruby_ast_data_get(ast_value); if (!ast->body.root) { rb_ast_dispose(ast); return Qnil; } - return vast; + return ast_value; } /** @@ -2155,6 +2155,7 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) pm_options_command_line_set(options, command_line); prism_opt_init(opt); + result->node.coverage_enabled = 0; error = pm_parse_string(result, opt->e_script, rb_str_new2("-e")); } else { @@ -2239,7 +2240,7 @@ process_options_global_setup(const ruby_cmdline_options_t *opt, const rb_iseq_t static VALUE process_options(int argc, char **argv, ruby_cmdline_options_t *opt) { - VALUE vast = Qnil; + VALUE ast_value = Qnil; struct { rb_ast_t *ast; pm_parse_result_t prism; @@ -2474,8 +2475,8 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt) } if (!(*rb_ruby_prism_ptr())) { - vast = process_script(opt); - if (!(result.ast = rb_ruby_ast_data_get(vast))) return Qfalse; + ast_value = process_script(opt); + if (!(result.ast = rb_ruby_ast_data_get(ast_value))) return Qfalse; } else { prism_script(opt, &result.prism); @@ -2557,7 +2558,7 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt) } else { rb_ast_t *ast = result.ast; - iseq = rb_iseq_new_main(vast, opt->script_name, path, parent, optimize); + iseq = rb_iseq_new_main(ast_value, opt->script_name, path, parent, optimize); rb_ast_dispose(ast); } } @@ -2608,7 +2609,7 @@ load_file_internal(VALUE argp_v) ruby_cmdline_options_t *opt = argp->opt; VALUE f = argp->f; int line_start = 1; - VALUE vast = Qnil; + VALUE ast_value = Qnil; rb_encoding *enc; ID set_encoding; @@ -2709,7 +2710,7 @@ load_file_internal(VALUE argp_v) return rb_parser_compile_string_path(parser, orig_fname, f, line_start); } rb_funcall(f, set_encoding, 2, rb_enc_from_encoding(enc), rb_str_new_cstr("-")); - vast = rb_parser_compile_file_path(parser, orig_fname, f, line_start); + ast_value = rb_parser_compile_file_path(parser, orig_fname, f, line_start); rb_funcall(f, set_encoding, 1, rb_parser_encoding(parser)); if (script && rb_parser_end_seen_p(parser)) { /* @@ -2727,7 +2728,7 @@ load_file_internal(VALUE argp_v) rb_define_global_const("DATA", f); argp->f = Qnil; } - return vast; + return ast_value; } /* disabling O_NONBLOCK, and returns 0 on success, otherwise errno */ @@ -2859,9 +2860,9 @@ rb_load_file(const char *fname) void * rb_load_file_str(VALUE fname_v) { - VALUE vast; - vast = rb_parser_load_file(rb_parser_new(), fname_v); - return (void *)rb_ruby_ast_data_get(vast); + VALUE ast_value; + ast_value = rb_parser_load_file(rb_parser_new(), fname_v); + return (void *)rb_ruby_ast_data_get(ast_value); } VALUE diff --git a/ruby_parser.c b/ruby_parser.c index 40b7e18aca..1dcdfd8e79 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -32,6 +32,8 @@ #include "vm_core.h" #include "symbol.h" +#define parser_encoding const void + static int is_ascii_string2(VALUE str) { @@ -41,9 +43,9 @@ is_ascii_string2(VALUE str) RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0) static VALUE syntax_error_append(VALUE exc, VALUE file, int line, int column, - void *enc, const char *fmt, va_list args) + parser_encoding *enc, const char *fmt, va_list args) { - return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args); + return rb_syntax_error_append(exc, file, line, column, enc, fmt, args); } static int @@ -59,9 +61,9 @@ dvar_defined(ID id, const void *p) } static int -is_usascii_enc(void *enc) +is_usascii_enc(parser_encoding *enc) { - return rb_is_usascii_enc((rb_encoding *)enc); + return rb_is_usascii_enc(enc); } static int @@ -83,21 +85,21 @@ is_notop_id2(ID id) } static VALUE -enc_str_new(const char *ptr, long len, void *enc) +enc_str_new(const char *ptr, long len, parser_encoding *enc) { - return rb_enc_str_new(ptr, len, (rb_encoding *)enc); + return rb_enc_str_new(ptr, len, enc); } static int -enc_isalnum(OnigCodePoint c, void *enc) +enc_isalnum(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isalnum(c, (rb_encoding *)enc); + return rb_enc_isalnum(c, enc); } static int -enc_precise_mbclen(const char *p, const char *e, void *enc) +enc_precise_mbclen(const char *p, const char *e, parser_encoding *enc) { - return rb_enc_precise_mbclen(p, e, (rb_encoding *)enc); + return rb_enc_precise_mbclen(p, e, enc); } static int @@ -113,87 +115,93 @@ mbclen_charfound_len(int len) } static const char * -enc_name(void *enc) +enc_name(parser_encoding *enc) { - return rb_enc_name((rb_encoding *)enc); + return rb_enc_name(enc); } static char * -enc_prev_char(const char *s, const char *p, const char *e, void *enc) +enc_prev_char(const char *s, const char *p, const char *e, parser_encoding *enc) { - return rb_enc_prev_char(s, p, e, (rb_encoding *)enc); + return rb_enc_prev_char(s, p, e, enc); } -static void * +static parser_encoding * enc_get(VALUE obj) { - return (void *)rb_enc_get(obj); + return rb_enc_get(obj); } static int -enc_asciicompat(void *enc) +enc_asciicompat(parser_encoding *enc) { - return rb_enc_asciicompat((rb_encoding *)enc); + return rb_enc_asciicompat(enc); } -static void * +static parser_encoding * utf8_encoding(void) { - return (void *)rb_utf8_encoding(); + return rb_utf8_encoding(); } static VALUE -enc_associate(VALUE obj, void *enc) +enc_associate(VALUE obj, parser_encoding *enc) { - return rb_enc_associate(obj, (rb_encoding *)enc); + return rb_enc_associate(obj, enc); } -static void * +static parser_encoding * ascii8bit_encoding(void) { - return (void *)rb_ascii8bit_encoding(); + return rb_ascii8bit_encoding(); } static int -enc_codelen(int c, void *enc) +enc_codelen(int c, parser_encoding *enc) { - return rb_enc_codelen(c, (rb_encoding *)enc); + return rb_enc_codelen(c, enc); } static int -enc_mbcput(unsigned int c, void *buf, void *enc) +enc_mbcput(unsigned int c, void *buf, parser_encoding *enc) { - return rb_enc_mbcput(c, buf, (rb_encoding *)enc); + return rb_enc_mbcput(c, buf, enc); } -static void * +static int +enc_mbclen(const char *p, const char *e, parser_encoding *enc) +{ + return rb_enc_mbclen(p, e, enc); +} + +static parser_encoding * enc_from_index(int idx) { - return (void *)rb_enc_from_index(idx); + return rb_enc_from_index(idx); } static int -enc_isspace(OnigCodePoint c, void *enc) +enc_isspace(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isspace(c, (rb_encoding *)enc); + return rb_enc_isspace(c, enc); } static ID -intern3(const char *name, long len, void *enc) +intern3(const char *name, long len, parser_encoding *enc) { - return rb_intern3(name, len, (rb_encoding *)enc); + return rb_intern3(name, len, enc); } -static void * +static parser_encoding * usascii_encoding(void) { - return (void *)rb_usascii_encoding(); + return rb_usascii_encoding(); } static int -enc_symname_type(const char *name, long len, void *enc, unsigned int allowed_attrset) +enc_symname_type(const char *name, long len, parser_encoding *enc, unsigned int allowed_attrset) { - return rb_enc_symname_type(name, len, (rb_encoding *)enc, allowed_attrset); + return rb_enc_symname_type(name, len, enc, allowed_attrset); } typedef struct { @@ -214,7 +222,7 @@ reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end, long len = name_end - name; const char *s = (const char *)name; - return rb_reg_named_capture_assign_iter_impl(p, s, len, (void *)enc, &arg->succ_block, loc); + return rb_reg_named_capture_assign_iter_impl(p, s, len, enc, &arg->succ_block, loc); } static NODE * @@ -292,14 +300,6 @@ arg_error(void) return rb_eArgError; } -static rb_ast_t * -ast_new(node_buffer_t *nb) -{ - rb_ast_t *ast = ruby_xcalloc(1, sizeof(rb_ast_t)); - ast->node_buffer = nb; - return ast; -} - static VALUE static_id2sym(ID id) { @@ -307,25 +307,25 @@ static_id2sym(ID id) } static long -str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr) +str_coderange_scan_restartable(const char *s, const char *e, parser_encoding *enc, int *cr) { - return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr); + return rb_str_coderange_scan_restartable(s, e, enc, cr); } static int -enc_mbminlen(void *enc) +enc_mbminlen(parser_encoding *enc) { - return rb_enc_mbminlen((rb_encoding *)enc); + return rb_enc_mbminlen(enc); } static bool -enc_isascii(OnigCodePoint c, void *enc) +enc_isascii(OnigCodePoint c, parser_encoding *enc) { - return rb_enc_isascii(c, (rb_encoding *)enc); + return rb_enc_isascii(c, enc); } static OnigCodePoint -enc_mbc_to_codepoint(const char *p, const char *e, void *enc) +enc_mbc_to_codepoint(const char *p, const char *e, parser_encoding *enc) { const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p); const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e); @@ -348,8 +348,6 @@ static const rb_parser_config_t rb_global_parser_config = { .nonempty_memcpy = nonempty_memcpy, .xmalloc_mul_add = rb_xmalloc_mul_add, - .ast_new = ast_new, - .compile_callback = rb_suppress_tracing, .reg_named_capture_assign = reg_named_capture_assign, @@ -421,6 +419,7 @@ static const rb_parser_config_t rb_global_parser_config = { .ascii8bit_encoding = ascii8bit_encoding, .enc_codelen = enc_codelen, .enc_mbcput = enc_mbcput, + .enc_mbclen = enc_mbclen, .enc_find_index = rb_enc_find_index, .enc_from_index = enc_from_index, .enc_isspace = enc_isspace, @@ -528,6 +527,7 @@ parser_free(void *ptr) { struct ruby_parser *parser = (struct ruby_parser*)ptr; rb_ruby_parser_free(parser->parser_params); + xfree(parser); } static size_t @@ -632,8 +632,8 @@ rb_parser_keep_tokens(VALUE vparser) rb_ruby_parser_keep_tokens(parser->parser_params); } -VALUE -rb_parser_lex_get_str(struct lex_pointer_string *ptr_str) +rb_parser_string_t * +rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str) { char *beg, *end, *start; long len; @@ -643,20 +643,20 @@ rb_parser_lex_get_str(struct lex_pointer_string *ptr_str) len = RSTRING_LEN(s); start = beg; if (ptr_str->ptr) { - if (len == ptr_str->ptr) return Qnil; + if (len == ptr_str->ptr) return 0; beg += ptr_str->ptr; len -= ptr_str->ptr; } end = memchr(beg, '\n', len); if (end) len = ++end - beg; ptr_str->ptr += len; - return rb_str_subseq(s, beg - start, len); + return rb_str_to_parser_string(p, rb_str_subseq(s, beg - start, len)); } -static VALUE +static rb_parser_string_t * lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count) { - return rb_parser_lex_get_str((struct lex_pointer_string *)input); + return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input); } static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines); @@ -664,7 +664,19 @@ static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines); static rb_ast_t* parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line) { - rb_ast_t *ast = rb_parser_compile(p, gets, fname, input, line); + rb_ast_t *ast; + const char *ptr = 0; + long len = 0; + rb_encoding *enc = 0; + + if (!NIL_P(fname)) { + StringValueCStr(fname); + ptr = RSTRING_PTR(fname); + len = RSTRING_LEN(fname); + enc = rb_enc_get(fname); + } + + ast = rb_parser_compile(p, gets, ptr, len, enc, input, line); parser_aset_script_lines_for(fname, ast->body.script_lines); return ast; } @@ -706,15 +718,16 @@ parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int li VALUE rb_io_gets_internal(VALUE io); -static VALUE +static rb_parser_string_t * lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count) { VALUE io = (VALUE)input; - - return rb_io_gets_internal(io); + VALUE line = rb_io_gets_internal(io); + if (NIL_P(line)) return 0; + return rb_str_to_parser_string(p, line); } -static VALUE +static rb_parser_string_t * lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index) { VALUE array = (VALUE)data; @@ -724,8 +737,11 @@ lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index) if (!rb_enc_asciicompat(rb_enc_get(str))) { rb_raise(rb_eArgError, "invalid source encoding"); } + return rb_str_to_parser_string(p, str); + } + else { + return 0; } - return str; } static rb_ast_t* @@ -774,78 +790,72 @@ static const rb_data_type_t ast_data_type = { static VALUE ast_alloc(void) { - rb_ast_t *ast; - VALUE vast = TypedData_Make_Struct(0, rb_ast_t, &ast_data_type, ast); -#ifdef UNIVERSAL_PARSER - ast = (rb_ast_t *)DATA_PTR(vast); - ast->config = &rb_global_parser_config; -#endif - return vast; + return TypedData_Wrap_Struct(0, &ast_data_type, NULL); } VALUE rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start) { struct ruby_parser *parser; - VALUE vast = ast_alloc(); + VALUE ast_value = ast_alloc(); TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - DATA_PTR(vast) = parser_compile_file_path(parser, fname, file, start); + DATA_PTR(ast_value) = parser_compile_file_path(parser, fname, file, start); RB_GC_GUARD(vparser); - return vast; + return ast_value; } VALUE rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start) { struct ruby_parser *parser; - VALUE vast = ast_alloc(); + VALUE ast_value = ast_alloc(); TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - DATA_PTR(vast) = parser_compile_array(parser, fname, array, start); + DATA_PTR(ast_value) = parser_compile_array(parser, fname, array, start); RB_GC_GUARD(vparser); - return vast; + return ast_value; } VALUE rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start) { struct ruby_parser *parser; - VALUE vast = ast_alloc(); + VALUE ast_value = ast_alloc(); TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - DATA_PTR(vast) = parser_compile_generic(parser, lex_gets, fname, input, start); + DATA_PTR(ast_value) = parser_compile_generic(parser, lex_gets, fname, input, start); RB_GC_GUARD(vparser); - return vast; + return ast_value; } VALUE rb_parser_compile_string(VALUE vparser, const char *f, VALUE s, int line) { struct ruby_parser *parser; - VALUE vast = ast_alloc(); + VALUE ast_value = ast_alloc(); TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - DATA_PTR(vast) = parser_compile_string(parser, f, s, line); + DATA_PTR(ast_value) = parser_compile_string(parser, f, s, line); RB_GC_GUARD(vparser); - return vast; + return ast_value; } VALUE rb_parser_compile_string_path(VALUE vparser, VALUE f, VALUE s, int line) { struct ruby_parser *parser; - VALUE vast = ast_alloc(); + VALUE ast_value = ast_alloc(); TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - DATA_PTR(vast) = parser_compile_string_path(parser, f, s, line); + DATA_PTR(ast_value) = parser_compile_string_path(parser, f, s, line); RB_GC_GUARD(vparser); - return vast; + return ast_value; } VALUE @@ -1132,8 +1142,11 @@ parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines) VALUE rb_ruby_ast_new(const NODE *const root) { - VALUE vast = ast_alloc(); - rb_ast_t *ast = DATA_PTR(vast); + rb_ast_t *ast; + VALUE ast_value = TypedData_Make_Struct(0, rb_ast_t, &ast_data_type, ast); +#ifdef UNIVERSAL_PARSER + ast->config = &rb_global_parser_config; +#endif ast->body = (rb_ast_body_t){ .root = root, .frozen_string_literal = -1, @@ -1141,14 +1154,14 @@ rb_ruby_ast_new(const NODE *const root) .script_lines = NULL, .line_count = 0, }; - return vast; + return ast_value; } rb_ast_t * -rb_ruby_ast_data_get(VALUE vast) +rb_ruby_ast_data_get(VALUE ast_value) { rb_ast_t *ast; - if (NIL_P(vast)) return NULL; - TypedData_Get_Struct(vast, rb_ast_t, &ast_data_type, ast); + if (NIL_P(ast_value)) return NULL; + TypedData_Get_Struct(ast_value, rb_ast_t, &ast_data_type, ast); return ast; } diff --git a/rubyparser.h b/rubyparser.h index 813ad321e2..fc8be9633a 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -9,7 +9,7 @@ #ifdef UNIVERSAL_PARSER -#define rb_encoding void +#define rb_encoding const void #define OnigCodePoint unsigned int #include "parser_st.h" #ifndef RUBY_RUBY_H @@ -1255,8 +1255,6 @@ typedef struct rb_parser_config_struct { void *(*nonempty_memcpy)(void *dest, const void *src, size_t t, size_t n); void *(*xmalloc_mul_add)(size_t x, size_t y, size_t z); - rb_ast_t *(*ast_new)(node_buffer_t *nb); - // VALUE rb_suppress_tracing(VALUE (*func)(VALUE), VALUE arg); VALUE (*compile_callback)(VALUE (*func)(VALUE), VALUE arg); NODE *(*reg_named_capture_assign)(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc); @@ -1340,6 +1338,7 @@ typedef struct rb_parser_config_struct { rb_encoding *(*ascii8bit_encoding)(void); int (*enc_codelen)(int c, rb_encoding *enc); int (*enc_mbcput)(unsigned int c, void *buf, rb_encoding *enc); + int (*enc_mbclen)(const char *p, const char *e, rb_encoding *enc); int (*enc_find_index)(const char *name); rb_encoding *(*enc_from_index)(int idx); int (*enc_isspace)(OnigCodePoint c, rb_encoding *enc); @@ -118,6 +118,7 @@ redblack_value(redblack_node_t * node) return (rb_shape_t *)((uintptr_t)node->value & (((uintptr_t)-1) - 1)); } +#ifdef HAVE_MMAP static redblack_id_t redblack_id_for(redblack_node_t * node) { @@ -292,6 +293,7 @@ redblack_insert(redblack_node_t * tree, ID key, rb_shape_t * value) return root; } } +#endif rb_shape_tree_t *rb_shape_tree_ptr = NULL; diff --git a/spec/bundler/bundler/compact_index_client/parser_spec.rb b/spec/bundler/bundler/compact_index_client/parser_spec.rb new file mode 100644 index 0000000000..1d1d41a5a8 --- /dev/null +++ b/spec/bundler/bundler/compact_index_client/parser_spec.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true + +require "bundler/compact_index_client" +require "bundler/compact_index_client/parser" + +TestCompactIndexClient = Struct.new(:names, :versions, :info_data) do + # Requiring the checksum to match the input data helps ensure + # that we are parsing the correct checksum from the versions file + def info(name, checksum) + info_data.dig(name, checksum) + end + + def set_info_data(name, value) + info_data[name] = value + end +end + +RSpec.describe Bundler::CompactIndexClient::Parser do + subject(:parser) { described_class.new(compact_index) } + + let(:compact_index) { TestCompactIndexClient.new(names, versions, info_data) } + let(:names) { "a\nb\nc\n" } + let(:versions) { <<~VERSIONS } + a 1.0.0,1.0.1,1.1.0 aaa111 + b 2.0.0,2.0.0-java bbb222 + c 3.0.0,3.0.3,3.3.3 ccc333 + c -3.0.3 ccc333yanked + VERSIONS + let(:info_data) do + { + "a" => { "aaa111" => a_info }, + "b" => { "bbb222" => b_info }, + "c" => { "ccc333yanked" => c_info }, + } + end + let(:a_info) { <<~INFO } + 1.0.0 |checksum:aaa1,ruby:>= 3.0.0,rubygems:>= 3.2.3 + 1.0.1 |checksum:aaa2,ruby:>= 3.0.0,rubygems:>= 3.2.3 + 1.1.0 |checksum:aaa3,ruby:>= 3.0.0,rubygems:>= 3.2.3 + INFO + let(:b_info) { <<~INFO } + 2.0.0 a:~> 1.0&<= 3.0|checksum:bbb1 + 2.0.0-java a:~> 1.0&<= 3.0|checksum:bbb2 + INFO + let(:c_info) { <<~INFO } + 3.0.0 a:= 1.0.0,b:~> 2.0|checksum:ccc1,ruby:>= 2.7.0,rubygems:>= 3.0.0 + 3.3.3 a:>= 1.1.0,b:~> 2.0|checksum:ccc3,ruby:>= 3.0.0,rubygems:>= 3.2.3 + INFO + + describe "#available?" do + it "returns true versions are available" do + expect(parser).to be_available + end + + it "returns false when versions are not available" do + compact_index.versions = nil + expect(parser).not_to be_available + end + end + + describe "#names" do + it "returns the names" do + expect(parser.names).to eq(%w[a b c]) + end + + it "returns an empty array when names is empty" do + compact_index.names = "" + expect(parser.names).to eq([]) + end + + it "returns an empty array when names is not readable" do + compact_index.names = nil + expect(parser.names).to eq([]) + end + end + + describe "#versions" do + it "returns the versions" do + expect(parser.versions).to eq( + "a" => [ + ["a", "1.0.0"], + ["a", "1.0.1"], + ["a", "1.1.0"], + ], + "b" => [ + ["b", "2.0.0"], + ["b", "2.0.0", "java"], + ], + "c" => [ + ["c", "3.0.0"], + ["c", "3.3.3"], + ], + ) + end + + it "returns an empty hash when versions is empty" do + compact_index.versions = "" + expect(parser.versions).to eq({}) + end + + it "returns an empty hash when versions is not readable" do + compact_index.versions = nil + expect(parser.versions).to eq({}) + end + end + + describe "#info" do + it "returns the info for example gem 'a' which has no deps" do + expect(parser.info("a")).to eq( + [ + [ + "a", + "1.0.0", + nil, + [], + [ + ["checksum", ["aaa1"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + [ + "a", + "1.0.1", + nil, + [], + [ + ["checksum", ["aaa2"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + [ + "a", + "1.1.0", + nil, + [], + [ + ["checksum", ["aaa3"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + ] + ) + end + + it "returns the info for example gem 'b' which has platform and compound deps" do + expect(parser.info("b")).to eq( + [ + [ + "b", + "2.0.0", + nil, + [ + ["a", ["~> 1.0", "<= 3.0"]], + ], + [ + ["checksum", ["bbb1"]], + ], + ], + [ + "b", + "2.0.0", + "java", + [ + ["a", ["~> 1.0", "<= 3.0"]], + ], + [ + ["checksum", ["bbb2"]], + ], + ], + ] + ) + end + + it "returns the info for example gem 'c' which has deps and yanked version (requires use of correct info checksum)" do + expect(parser.info("c")).to eq( + [ + [ + "c", + "3.0.0", + nil, + [ + ["a", ["= 1.0.0"]], + ["b", ["~> 2.0"]], + ], + [ + ["checksum", ["ccc1"]], + ["ruby", [">= 2.7.0"]], + ["rubygems", [">= 3.0.0"]], + ], + ], + [ + "c", + "3.3.3", + nil, + [ + ["a", [">= 1.1.0"]], + ["b", ["~> 2.0"]], + ], + [ + ["checksum", ["ccc3"]], + ["ruby", [">= 3.0.0"]], + ["rubygems", [">= 3.2.3"]], + ], + ], + ] + ) + end + + it "returns an empty array when the info is empty" do + compact_index.set_info_data("a", {}) + expect(parser.info("a")).to eq([]) + end + + it "returns an empty array when the info is not readable" do + expect(parser.info("d")).to eq([]) + end + end +end diff --git a/spec/bundler/bundler/compact_index_client/updater_spec.rb b/spec/bundler/bundler/compact_index_client/updater_spec.rb index 6eed88ca9e..87a73d993f 100644 --- a/spec/bundler/bundler/compact_index_client/updater_spec.rb +++ b/spec/bundler/bundler/compact_index_client/updater_spec.rb @@ -119,23 +119,7 @@ RSpec.describe Bundler::CompactIndexClient::Updater do context "without an etag file" do let(:headers) do - { - "Range" => "bytes=2-", - # This MD5 feature should be deleted after sufficient time has passed since release. - # From then on, requests that still don't have a saved etag will be made without this header. - "If-None-Match" => %("#{Digest::MD5.hexdigest(local_body)}"), - } - end - - it "saves only the etag_path if generated etag matches" do - expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) - allow(response).to receive(:is_a?).with(Gem::Net::HTTPPartialContent) { false } - allow(response).to receive(:is_a?).with(Gem::Net::HTTPNotModified) { true } - - updater.update(remote_path, local_path, etag_path) - - expect(local_path.read).to eq("abc") - expect(%("#{etag_path.read}")).to eq(headers["If-None-Match"]) + { "Range" => "bytes=2-" } end it "appends the file" do diff --git a/spec/bundler/bundler/fetcher/compact_index_spec.rb b/spec/bundler/bundler/fetcher/compact_index_spec.rb index a988171f34..a7bb61a900 100644 --- a/spec/bundler/bundler/fetcher/compact_index_spec.rb +++ b/spec/bundler/bundler/fetcher/compact_index_spec.rb @@ -4,13 +4,15 @@ require "bundler/compact_index_client" RSpec.describe Bundler::Fetcher::CompactIndex do - let(:downloader) { double(:downloader) } + let(:response) { double(:response) } + let(:downloader) { double(:downloader, fetch: response) } let(:display_uri) { Gem::URI("http://sampleuri.com") } let(:remote) { double(:remote, cache_slug: "lsjdf", uri: display_uri) } let(:gem_remote_fetcher) { nil } let(:compact_index) { described_class.new(downloader, remote, display_uri, gem_remote_fetcher) } before do + allow(response).to receive(:is_a?).with(Gem::Net::HTTPNotModified).and_return(true) allow(compact_index).to receive(:log_specs) {} end @@ -34,7 +36,7 @@ RSpec.describe Bundler::Fetcher::CompactIndex do describe "#available?" do before do allow(compact_index).to receive(:compact_index_client). - and_return(double(:compact_index_client, update_and_parse_checksums!: true)) + and_return(double(:compact_index_client, available?: true)) end it "returns true" do diff --git a/spec/bundler/bundler/source/git/git_proxy_spec.rb b/spec/bundler/bundler/source/git/git_proxy_spec.rb index 1450316d59..f7c883eed4 100644 --- a/spec/bundler/bundler/source/git/git_proxy_spec.rb +++ b/spec/bundler/bundler/source/git/git_proxy_spec.rb @@ -197,4 +197,18 @@ RSpec.describe Bundler::Source::Git::GitProxy do expect(Pathname.new(bundled_app("canary"))).not_to exist end + + context "URI is HTTP" do + let(:uri) { "http://github.com/rubygems/rubygems.git" } + let(:without_depth_arguments) { ["clone", "--bare", "--no-hardlinks", "--quiet", "--no-tags", "--single-branch"] } + let(:fail_clone_result) { double(Process::Status, success?: false) } + + it "retries without --depth when git url is http and fails" do + allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.14.0") + allow(git_proxy).to receive(:capture).with([*base_clone_args, "--", uri, path.to_s], nil).and_return(["", "dumb http transport does not support shallow capabilities", fail_clone_result]) + expect(git_proxy).to receive(:capture).with([*without_depth_arguments, "--", uri, path.to_s], nil).and_return(["", "", clone_result]) + + subject.checkout + end + end end diff --git a/spec/bundler/commands/cache_spec.rb b/spec/bundler/commands/cache_spec.rb index 70e2c84961..37d8b3ac1a 100644 --- a/spec/bundler/commands/cache_spec.rb +++ b/spec/bundler/commands/cache_spec.rb @@ -386,6 +386,66 @@ RSpec.describe "bundle install with gem sources" do expect(the_bundle).to include_gems "rack 1.0.0" end + it "uses cached gems for secondary sources when cache_all_platforms configured" do + build_repo4 do + build_gem "foo", "1.0.0" do |s| + s.platform = "x86_64-linux" + end + + build_gem "foo", "1.0.0" do |s| + s.platform = "arm64-darwin" + end + end + + gemfile <<~G + source "https://gems.repo2" + + source "https://gems.repo4" do + gem "foo" + end + G + + lockfile <<~L + GEM + remote: https://gems.repo2/ + specs: + + GEM + remote: https://gems.repo4/ + specs: + foo (1.0.0-x86_64-linux) + foo (1.0.0-arm64-darwin) + + PLATFORMS + arm64-darwin + ruby + x86_64-linux + + DEPENDENCIES + foo + + BUNDLED WITH + #{Bundler::VERSION} + L + + simulate_platform "x86_64-linux" do + bundle "config set cache_all_platforms true" + bundle "config set path vendor/bundle" + bundle :cache, artifice: "compact_index", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + + build_repo4 do + # simulate removal of all remote gems + end + + # delete compact index cache + FileUtils.rm_rf home(".bundle/cache/compact_index") + + bundle "install", artifice: "compact_index", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + + expect(the_bundle).to include_gems "foo 1.0.0 x86_64-linux" + end + end + it "does not reinstall already-installed gems" do install_gemfile <<-G source "#{file_uri_for(gem_repo1)}" diff --git a/spec/bundler/commands/exec_spec.rb b/spec/bundler/commands/exec_spec.rb index d59b690d2f..9f5f12739a 100644 --- a/spec/bundler/commands/exec_spec.rb +++ b/spec/bundler/commands/exec_spec.rb @@ -885,7 +885,7 @@ RSpec.describe "bundle exec" do let(:exit_code) { Bundler::GemNotFound.new.status_code } let(:expected) { "" } let(:expected_err) { <<-EOS.strip } -Could not find gem 'rack (= 2)' in cached gems or installed locally. +Could not find gem 'rack (= 2)' in locally installed gems. The source contains the following gems matching 'rack': * rack-0.9.1 @@ -915,7 +915,7 @@ Run `bundle install` to install missing gems. let(:exit_code) { Bundler::GemNotFound.new.status_code } let(:expected) { "" } let(:expected_err) { <<-EOS.strip } -Could not find gem 'rack (= 2)' in cached gems or installed locally. +Could not find gem 'rack (= 2)' in locally installed gems. The source contains the following gems matching 'rack': * rack-1.0.0 diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index f0c9aaea8e..edc5887d7b 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1024,6 +1024,29 @@ RSpec.describe "bundle install with gem sources" do end end + describe "when gemspecs are unreadable", :permissions do + let(:gemspec_path) { vendored_gems("specifications/rack-1.0.0.gemspec") } + + before do + gemfile <<~G + source "#{file_uri_for(gem_repo1)}" + gem 'rack' + G + bundle "config path vendor/bundle" + bundle :install + expect(out).to include("Bundle complete!") + expect(err).to be_empty + + FileUtils.chmod("-r", gemspec_path) + end + + it "shows a good error" do + bundle :install, raise_on_error: false + expect(err).to include(gemspec_path.to_s) + expect(err).to include("grant read permissions") + end + end + context "after installing with --standalone" do before do install_gemfile <<-G @@ -1384,4 +1407,33 @@ RSpec.describe "bundle install with gem sources" do expect(bundled_app(".bundle/config")).not_to exist end end + + context "when bundler installation is corrupt" do + before do + system_gems "bundler-9.99.8" + + replace_version_file("9.99.9", dir: system_gem_path("gems/bundler-9.99.8")) + end + + it "shows a proper error" do + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo1)}/ + specs: + + PLATFORMS + #{lockfile_platforms} + + DEPENDENCIES + + BUNDLED WITH + 9.99.8 + L + + install_gemfile "source \"#{file_uri_for(gem_repo1)}\"", env: { "BUNDLER_VERSION" => "9.99.8" }, raise_on_error: false + + expect(err).not_to include("ERROR REPORT TEMPLATE") + expect(err).to include("The running version of Bundler (9.99.9) does not match the version of the specification installed for it (9.99.8)") + end + end end diff --git a/spec/bundler/commands/lock_spec.rb b/spec/bundler/commands/lock_spec.rb index f6793d393b..b0d6fa9134 100644 --- a/spec/bundler/commands/lock_spec.rb +++ b/spec/bundler/commands/lock_spec.rb @@ -138,7 +138,7 @@ RSpec.describe "bundle lock" do it "does not fetch remote specs when using the --local option" do bundle "lock --update --local", raise_on_error: false - expect(err).to match(/cached gems or installed locally/) + expect(err).to match(/locally installed gems/) end it "does not fetch remote checksums with --local" do @@ -252,6 +252,128 @@ RSpec.describe "bundle lock" do expect(read_lockfile).to eq(remove_checksums_from_lockfile(@lockfile, "(2.3.2)", "(#{rake_version})")) end + it "updates specific gems using --update, even if that requires unlocking other top level gems" do + build_repo4 do + build_gem "prism", "0.15.1" + build_gem "prism", "0.24.0" + + build_gem "ruby-lsp", "0.12.0" do |s| + s.add_dependency "prism", "< 0.24.0" + end + + build_gem "ruby-lsp", "0.16.1" do |s| + s.add_dependency "prism", ">= 0.24.0" + end + + build_gem "tapioca", "0.11.10" do |s| + s.add_dependency "prism", "< 0.24.0" + end + + build_gem "tapioca", "0.13.1" do |s| + s.add_dependency "prism", ">= 0.24.0" + end + end + + gemfile <<~G + source "#{file_uri_for(gem_repo4)}" + + gem "tapioca" + gem "ruby-lsp" + G + + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo4)} + specs: + prism (0.15.1) + ruby-lsp (0.12.0) + prism (< 0.24.0) + tapioca (0.11.10) + prism (< 0.24.0) + + PLATFORMS + #{lockfile_platforms} + + DEPENDENCIES + ruby-lsp + tapioca + + BUNDLED WITH + #{Bundler::VERSION} + L + + bundle "lock --update tapioca --verbose" + + expect(lockfile).to include("tapioca (0.13.1)") + end + + it "updates specific gems using --update, even if that requires unlocking other top level gems, but only as few as possible" do + build_repo4 do + build_gem "prism", "0.15.1" + build_gem "prism", "0.24.0" + + build_gem "ruby-lsp", "0.12.0" do |s| + s.add_dependency "prism", "< 0.24.0" + end + + build_gem "ruby-lsp", "0.16.1" do |s| + s.add_dependency "prism", ">= 0.24.0" + end + + build_gem "tapioca", "0.11.10" do |s| + s.add_dependency "prism", "< 0.24.0" + end + + build_gem "tapioca", "0.13.1" do |s| + s.add_dependency "prism", ">= 0.24.0" + end + + build_gem "other-prism-dependent", "1.0.0" do |s| + s.add_dependency "prism", ">= 0.15.1" + end + + build_gem "other-prism-dependent", "1.1.0" do |s| + s.add_dependency "prism", ">= 0.15.1" + end + end + + gemfile <<~G + source "#{file_uri_for(gem_repo4)}" + + gem "tapioca" + gem "ruby-lsp" + gem "other-prism-dependent" + G + + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo4)} + specs: + other-prism-dependent (1.0.0) + prism (>= 0.15.1) + prism (0.15.1) + ruby-lsp (0.12.0) + prism (< 0.24.0) + tapioca (0.11.10) + prism (< 0.24.0) + + PLATFORMS + #{lockfile_platforms} + + DEPENDENCIES + ruby-lsp + tapioca + + BUNDLED WITH + #{Bundler::VERSION} + L + + bundle "lock --update tapioca" + + expect(lockfile).to include("tapioca (0.13.1)") + expect(lockfile).to include("other-prism-dependent (1.0.0)") + end + it "preserves unknown checksum algorithms" do lockfile @lockfile.gsub(/(sha256=[a-f0-9]+)$/, "constant=true,\\1,xyz=123") @@ -1227,7 +1349,7 @@ RSpec.describe "bundle lock" do Because rails >= 7.0.4 depends on railties = 7.0.4 and rails < 7.0.4 depends on railties = 7.0.3.1, railties = 7.0.3.1 OR = 7.0.4 is required. - So, because railties = 7.0.3.1 OR = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally, + So, because railties = 7.0.3.1 OR = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally, version solving has failed. ERR end @@ -1338,7 +1460,7 @@ RSpec.describe "bundle lock" do Thus, rails >= 7.0.2.3, < 7.0.4 cannot be used. And because rails >= 7.0.4 depends on activemodel = 7.0.4, rails >= 7.0.2.3 requires activemodel = 7.0.4. - So, because activemodel = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally + So, because activemodel = 7.0.4 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally and Gemfile depends on rails >= 7.0.2.3, version solving has failed. ERR diff --git a/spec/bundler/commands/newgem_spec.rb b/spec/bundler/commands/newgem_spec.rb index 199340b131..53d0f9d6eb 100644 --- a/spec/bundler/commands/newgem_spec.rb +++ b/spec/bundler/commands/newgem_spec.rb @@ -36,22 +36,25 @@ RSpec.describe "bundle gem" do sys_exec("git config --global user.name 'Bundler User'") sys_exec("git config --global user.email user@example.com") sys_exec("git config --global github.user bundleuser") + + global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__LINTER" => "false", + "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__CHANGELOG" => "false" end describe "git repo initialization" do - it "generates a gem skeleton with a .git folder", :readline do + it "generates a gem skeleton with a .git folder" do bundle "gem #{gem_name}" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).to exist end - it "generates a gem skeleton with a .git folder when passing --git", :readline do + it "generates a gem skeleton with a .git folder when passing --git" do bundle "gem #{gem_name} --git" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).to exist end - it "generates a gem skeleton without a .git folder when passing --no-git", :readline do + it "generates a gem skeleton without a .git folder when passing --no-git" do bundle "gem #{gem_name} --no-git" gem_skeleton_assertions expect(bundled_app("#{gem_name}/.git")).not_to exist @@ -62,7 +65,9 @@ RSpec.describe "bundle gem" do Dir.mkdir(bundled_app("path with spaces")) end - it "properly initializes git repo", :readline do + it "properly initializes git repo" do + skip "path with spaces needs special handling on Windows" if Gem.win_platform? + bundle "gem #{gem_name}", dir: bundled_app("path with spaces") expect(bundled_app("path with spaces/#{gem_name}/.git")).to exist end @@ -131,7 +136,7 @@ RSpec.describe "bundle gem" do before do bundle "gem #{gem_name} --changelog" end - it "generates a gem skeleton with a CHANGELOG", :readline do + it "generates a gem skeleton with a CHANGELOG" do gem_skeleton_assertions expect(bundled_app("#{gem_name}/CHANGELOG.md")).to exist end @@ -141,7 +146,7 @@ RSpec.describe "bundle gem" do before do bundle "gem #{gem_name} --no-changelog" end - it "generates a gem skeleton without a CHANGELOG", :readline do + it "generates a gem skeleton without a CHANGELOG" do gem_skeleton_assertions expect(bundled_app("#{gem_name}/CHANGELOG.md")).to_not exist end @@ -150,6 +155,7 @@ RSpec.describe "bundle gem" do shared_examples_for "--rubocop flag" do context "is deprecated", bundler: "< 3" do before do + global_config "BUNDLE_GEM__LINTER" => nil bundle "gem #{gem_name} --rubocop" end @@ -303,49 +309,49 @@ RSpec.describe "bundle gem" do end end - it "has no rubocop offenses when using --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=minitest, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=minitest, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=minitest --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=rspec, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=rspec, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=rspec --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=c, --test=test-unit, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=c, --test=test-unit, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --ext=c --test=test-unit --linter=rubocop" bundle_exec_rubocop expect(last_command).to be_success end - it "has no standard offenses when using --linter=standard flag", :readline do + it "has no standard offenses when using --linter=standard flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? bundle "gem #{gem_name} --linter=standard" bundle_exec_standardrb expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -354,7 +360,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=minitest, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=minitest, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -363,7 +369,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=rspec, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=rspec, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -372,7 +378,7 @@ RSpec.describe "bundle gem" do expect(last_command).to be_success end - it "has no rubocop offenses when using --ext=rust, --test=test-unit, and --linter=rubocop flag", :readline do + it "has no rubocop offenses when using --ext=rust, --test=test-unit, and --linter=rubocop flag" do skip "ruby_core has an 'ast.rb' file that gets in the middle and breaks this spec" if ruby_core? skip "RubyGems incompatible with Rust builder" if ::Gem::Version.new("3.3.11") > ::Gem.rubygems_version @@ -399,7 +405,7 @@ RSpec.describe "bundle gem" do end end - context "README.md", :readline do + context "README.md" do context "git config github.user present" do before do bundle "gem #{gem_name}" @@ -424,12 +430,12 @@ RSpec.describe "bundle gem" do end end - it "creates a new git repository", :readline do + it "creates a new git repository" do bundle "gem #{gem_name}" expect(bundled_app("#{gem_name}/.git")).to exist end - context "when git is not available", :readline do + context "when git is not available" do # This spec cannot have `git` available in the test env before do load_paths = [lib_dir, spec_dir] @@ -451,7 +457,7 @@ RSpec.describe "bundle gem" do end end - it "generates a valid gemspec", :readline, :ruby_repo do + it "generates a valid gemspec", :ruby_repo do bundle "gem newgem --bin" prepare_gemspec(bundled_app("newgem", "newgem.gemspec")) @@ -464,7 +470,7 @@ RSpec.describe "bundle gem" do expect(last_command.stdboth).not_to include("ERROR") end - context "gem naming with relative paths", :readline do + context "gem naming with relative paths" do it "resolves ." do create_temporary_dir("tmp") @@ -557,8 +563,12 @@ RSpec.describe "bundle gem" do expect(bundled_app("#{gem_name}/bin/setup")).to exist expect(bundled_app("#{gem_name}/bin/console")).to exist - expect(bundled_app("#{gem_name}/bin/setup")).to be_executable - expect(bundled_app("#{gem_name}/bin/console")).to be_executable + + unless Gem.win_platform? + expect(bundled_app("#{gem_name}/bin/setup")).to be_executable + expect(bundled_app("#{gem_name}/bin/console")).to be_executable + end + expect(bundled_app("#{gem_name}/bin/setup").read).to start_with("#!") expect(bundled_app("#{gem_name}/bin/console").read).to start_with("#!") end @@ -870,7 +880,7 @@ RSpec.describe "bundle gem" do end end - context "gem.test set to rspec and --test with no arguments", :hint_text do + context "gem.test set to rspec and --test with no arguments" do before do bundle "config set gem.test rspec" bundle "gem #{gem_name} --test" @@ -887,10 +897,12 @@ RSpec.describe "bundle gem" do end end - context "gem.test setting set to false and --test with no arguments", :hint_text do + context "gem.test setting set to false and --test with no arguments", :readline do before do bundle "config set gem.test false" - bundle "gem #{gem_name} --test" + bundle "gem #{gem_name} --test" do |input, _, _| + input.puts + end end it "asks to generate test files" do @@ -904,9 +916,12 @@ RSpec.describe "bundle gem" do it_behaves_like "test framework is absent" end - context "gem.test setting not set and --test with no arguments", :hint_text do + context "gem.test setting not set and --test with no arguments", :readline do before do - bundle "gem #{gem_name} --test" + global_config "BUNDLE_GEM__TEST" => nil + bundle "gem #{gem_name} --test" do |input, _, _| + input.puts + end end it "asks to generate test files" do @@ -1009,7 +1024,7 @@ RSpec.describe "bundle gem" do end end - context "gem.ci set to github and --ci with no arguments", :hint_text do + context "gem.ci set to github and --ci with no arguments" do before do bundle "config set gem.ci github" bundle "gem #{gem_name} --ci" @@ -1024,10 +1039,12 @@ RSpec.describe "bundle gem" do end end - context "gem.ci setting set to false and --ci with no arguments", :hint_text do + context "gem.ci setting set to false and --ci with no arguments", :readline do before do bundle "config set gem.ci false" - bundle "gem #{gem_name} --ci" + bundle "gem #{gem_name} --ci" do |input, _, _| + input.puts "github" + end end it "asks to setup CI" do @@ -1039,9 +1056,12 @@ RSpec.describe "bundle gem" do end end - context "gem.ci setting not set and --ci with no arguments", :hint_text do + context "gem.ci setting not set and --ci with no arguments", :readline do before do - bundle "gem #{gem_name} --ci" + global_config "BUNDLE_GEM__CI" => nil + bundle "gem #{gem_name} --ci" do |input, _, _| + input.puts "github" + end end it "asks to setup CI" do @@ -1111,6 +1131,7 @@ RSpec.describe "bundle gem" do context "gem.rubocop setting set to true", bundler: "< 3" do before do + global_config "BUNDLE_GEM__LINTER" => nil bundle "config set gem.rubocop true" bundle "gem #{gem_name}" end @@ -1130,7 +1151,7 @@ RSpec.describe "bundle gem" do end end - context "gem.linter set to rubocop and --linter with no arguments", :hint_text do + context "gem.linter set to rubocop and --linter with no arguments" do before do bundle "config set gem.linter rubocop" bundle "gem #{gem_name} --linter" @@ -1145,10 +1166,12 @@ RSpec.describe "bundle gem" do end end - context "gem.linter setting set to false and --linter with no arguments", :hint_text do + context "gem.linter setting set to false and --linter with no arguments", :readline do before do bundle "config set gem.linter false" - bundle "gem #{gem_name} --linter" + bundle "gem #{gem_name} --linter" do |input, _, _| + input.puts "rubocop" + end end it "asks to setup a linter" do @@ -1160,9 +1183,12 @@ RSpec.describe "bundle gem" do end end - context "gem.linter setting not set and --linter with no arguments", :hint_text do + context "gem.linter setting not set and --linter with no arguments", :readline do before do - bundle "gem #{gem_name} --linter" + global_config "BUNDLE_GEM__LINTER" => nil + bundle "gem #{gem_name} --linter" do |input, _, _| + input.puts "rubocop" + end end it "asks to setup a linter" do @@ -1185,7 +1211,7 @@ RSpec.describe "bundle gem" do end end - context "testing --mit and --coc options against bundle config settings", :readline do + context "testing --mit and --coc options against bundle config settings" do let(:gem_name) { "test-gem" } let(:require_path) { "test/gem" } @@ -1288,7 +1314,7 @@ RSpec.describe "bundle gem" do end end - context "testing --github-username option against git and bundle config settings", :readline do + context "testing --github-username option against git and bundle config settings" do context "without git config set" do before do sys_exec("git config --global --unset github.user") @@ -1325,7 +1351,7 @@ RSpec.describe "bundle gem" do end end - context "testing github_username bundle config against git config settings", :readline do + context "testing github_username bundle config against git config settings" do context "without git config set" do before do sys_exec("git config --global --unset github.user") @@ -1339,7 +1365,7 @@ RSpec.describe "bundle gem" do end end - context "gem naming with underscore", :readline do + context "gem naming with underscore" do let(:gem_name) { "test_gem" } let(:require_path) { "test_gem" } @@ -1485,7 +1511,7 @@ RSpec.describe "bundle gem" do end end - context "gem naming with dashed", :readline do + context "gem naming with dashed" do let(:gem_name) { "test-gem" } let(:require_path) { "test/gem" } @@ -1506,7 +1532,7 @@ RSpec.describe "bundle gem" do end describe "uncommon gem names" do - it "can deal with two dashes", :readline do + it "can deal with two dashes" do bundle "gem a--a" expect(bundled_app("a--a/a--a.gemspec")).to exist @@ -1536,7 +1562,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - describe "#ensure_safe_gem_name", :readline do + describe "#ensure_safe_gem_name" do before do bundle "gem #{subject}", raise_on_error: false end @@ -1564,7 +1590,7 @@ Usage: "bundle gem NAME [OPTIONS]" context "on first run", :readline do it "asks about test framework" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__COC" => "false" + global_config "BUNDLE_GEM__TEST" => nil bundle "gem foobar" do |input, _, _| input.puts "rspec" @@ -1587,7 +1613,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CI service" do - global_config "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__CI" => nil bundle "gem foobar" do |input, _, _| input.puts "github" @@ -1597,7 +1623,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about MIT license" do - global_config "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__MIT" => nil bundle "config list" @@ -1609,7 +1635,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CoC" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false" + global_config "BUNDLE_GEM__COC" => nil bundle "gem foobar" do |input, _, _| input.puts "yes" @@ -1619,8 +1645,7 @@ Usage: "bundle gem NAME [OPTIONS]" end it "asks about CHANGELOG" do - global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__LINTER" => "false", - "BUNDLE_GEM__COC" => "false" + global_config "BUNDLE_GEM__CHANGELOG" => nil bundle "gem foobar" do |input, _, _| input.puts "yes" @@ -1630,7 +1655,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - context "on conflicts with a previously created file", :readline do + context "on conflicts with a previously created file" do it "should fail gracefully" do FileUtils.touch(bundled_app("conflict-foobar")) bundle "gem conflict-foobar", raise_on_error: false @@ -1639,7 +1664,7 @@ Usage: "bundle gem NAME [OPTIONS]" end end - context "on conflicts with a previously created directory", :readline do + context "on conflicts with a previously created directory" do it "should succeed" do FileUtils.mkdir_p(bundled_app("conflict-foobar/Gemfile")) bundle "gem conflict-foobar" diff --git a/spec/bundler/commands/post_bundle_message_spec.rb b/spec/bundler/commands/post_bundle_message_spec.rb index 07fd5a79e9..3c7fd3486d 100644 --- a/spec/bundler/commands/post_bundle_message_spec.rb +++ b/spec/bundler/commands/post_bundle_message_spec.rb @@ -120,7 +120,7 @@ RSpec.describe "post bundle message" do gem "not-a-gem", :group => :development G expect(err).to include <<-EOS.strip -Could not find gem 'not-a-gem' in rubygems repository #{file_uri_for(gem_repo1)}/, cached gems or installed locally. +Could not find gem 'not-a-gem' in rubygems repository #{file_uri_for(gem_repo1)}/ or installed locally. EOS end diff --git a/spec/bundler/commands/update_spec.rb b/spec/bundler/commands/update_spec.rb index cfb86ebb54..8565e27ebf 100644 --- a/spec/bundler/commands/update_spec.rb +++ b/spec/bundler/commands/update_spec.rb @@ -1954,6 +1954,52 @@ RSpec.describe "bundle update conservative" do end end + context "when Gemfile dependencies have changed" do + before do + build_repo4 do + build_gem "nokogiri", "1.16.4" do |s| + s.platform = "arm64-darwin" + end + + build_gem "nokogiri", "1.16.4" do |s| + s.platform = "x86_64-linux" + end + + build_gem "prism", "0.25.0" + end + + gemfile <<~G + source "#{file_uri_for(gem_repo4)}" + gem "nokogiri", ">=1.16.4" + gem "prism", ">=0.25.0" + G + + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo4)}/ + specs: + nokogiri (1.16.4-arm64-darwin) + nokogiri (1.16.4-x86_64-linux) + + PLATFORMS + arm64-darwin + x86_64-linux + + DEPENDENCIES + nokogiri (>= 1.16.4) + + BUNDLED WITH + #{Bundler::VERSION} + L + end + + it "still works" do + simulate_platform "arm64-darwin-23" do + bundle "update" + end + end + end + context "error handling" do before do gemfile "source \"#{file_uri_for(gem_repo1)}\"" diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index 45ee7b44d1..24cf30eadb 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -929,7 +929,7 @@ RSpec.describe "bundle install with git sources" do gem "has_submodule" end G - expect(err).to match(%r{submodule >= 0 could not be found in rubygems repository #{file_uri_for(gem_repo1)}/, cached gems or installed locally}) + expect(err).to match(%r{submodule >= 0 could not be found in rubygems repository #{file_uri_for(gem_repo1)}/ or installed locally}) expect(the_bundle).not_to include_gems "has_submodule 1.0" end diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb index a5ba76f4d9..daee8a2744 100644 --- a/spec/bundler/install/gemfile/sources_spec.rb +++ b/spec/bundler/install/gemfile/sources_spec.rb @@ -520,7 +520,7 @@ RSpec.describe "bundle install with gems on multiple sources" do it "fails" do bundle :install, artifice: "compact_index", raise_on_error: false - expect(err).to include("Could not find gem 'private_gem_1' in rubygems repository https://gem.repo2/, cached gems or installed locally.") + expect(err).to include("Could not find gem 'private_gem_1' in rubygems repository https://gem.repo2/ or installed locally.") end end @@ -611,7 +611,7 @@ RSpec.describe "bundle install with gems on multiple sources" do Could not find compatible versions Because every version of depends_on_rack depends on rack >= 0 - and rack >= 0 could not be found in rubygems repository https://gem.repo2/, cached gems or installed locally, + and rack >= 0 could not be found in rubygems repository https://gem.repo2/ or installed locally, depends_on_rack cannot be used. So, because Gemfile depends on depends_on_rack >= 0, version solving has failed. diff --git a/spec/bundler/install/gemfile/specific_platform_spec.rb b/spec/bundler/install/gemfile/specific_platform_spec.rb index 5f1b034bfc..1a33053dc6 100644 --- a/spec/bundler/install/gemfile/specific_platform_spec.rb +++ b/spec/bundler/install/gemfile/specific_platform_spec.rb @@ -395,7 +395,7 @@ RSpec.describe "bundle install with specific platforms" do G error_message = <<~ERROR.strip - Could not find gem 'sorbet-static (= 0.5.6433)' with platform 'arm64-darwin-21' in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gem 'sorbet-static (= 0.5.6433)' with platform 'arm64-darwin-21' in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.6433)': * sorbet-static-0.5.6433-universal-darwin-20 @@ -434,7 +434,7 @@ RSpec.describe "bundle install with specific platforms" do Could not find compatible versions Because every version of sorbet depends on sorbet-static = 0.5.6433 - and sorbet-static = 0.5.6433 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally for any resolution platforms (arm64-darwin-21), + and sorbet-static = 0.5.6433 could not be found in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally for any resolution platforms (arm64-darwin-21), sorbet cannot be used. So, because Gemfile depends on sorbet = 0.5.6433, version solving has failed. @@ -473,7 +473,7 @@ RSpec.describe "bundle install with specific platforms" do bundle "lock", raise_on_error: false, env: { "BUNDLE_FORCE_RUBY_PLATFORM" => "true" } expect(err).to include <<~ERROR.rstrip - Could not find gem 'sorbet-static (= 0.5.9889)' with platform 'ruby' in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gem 'sorbet-static (= 0.5.9889)' with platform 'ruby' in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.9889)': * sorbet-static-0.5.9889-#{Gem::Platform.local} diff --git a/spec/bundler/install/gems/flex_spec.rb b/spec/bundler/install/gems/flex_spec.rb index 8ef3984975..5e0c88fc95 100644 --- a/spec/bundler/install/gems/flex_spec.rb +++ b/spec/bundler/install/gems/flex_spec.rb @@ -197,7 +197,7 @@ RSpec.describe "bundle flex_install" do Could not find compatible versions Because rack-obama >= 2.0 depends on rack = 1.2 - and rack = 1.2 could not be found in rubygems repository #{file_uri_for(gem_repo2)}/, cached gems or installed locally, + and rack = 1.2 could not be found in rubygems repository #{file_uri_for(gem_repo2)}/ or installed locally, rack-obama >= 2.0 cannot be used. So, because Gemfile depends on rack-obama = 2.0, version solving has failed. diff --git a/spec/bundler/install/gems/resolving_spec.rb b/spec/bundler/install/gems/resolving_spec.rb index c5f9c4a3d3..b54674898d 100644 --- a/spec/bundler/install/gems/resolving_spec.rb +++ b/spec/bundler/install/gems/resolving_spec.rb @@ -434,7 +434,7 @@ RSpec.describe "bundle install with install-time dependencies" do end nice_error = <<~E.strip - Could not find gems matching 'sorbet-static (= 0.5.10554)' valid for all resolution platforms (arm64-darwin-21, aarch64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gems matching 'sorbet-static (= 0.5.10554)' valid for all resolution platforms (arm64-darwin-21, aarch64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static (= 0.5.10554)': * sorbet-static-0.5.10554-universal-darwin-21 @@ -490,7 +490,7 @@ RSpec.describe "bundle install with install-time dependencies" do it "raises a proper error" do nice_error = <<~E.strip - Could not find gems matching 'sorbet-static' valid for all resolution platforms (arm-linux, x86_64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/, cached gems or installed locally. + Could not find gems matching 'sorbet-static' valid for all resolution platforms (arm-linux, x86_64-linux) in rubygems repository #{file_uri_for(gem_repo4)}/ or installed locally. The source contains the following gems matching 'sorbet-static': * sorbet-static-0.5.10696-x86_64-linux diff --git a/spec/bundler/install/yanked_spec.rb b/spec/bundler/install/yanked_spec.rb index 7408c24327..5aeabd2f23 100644 --- a/spec/bundler/install/yanked_spec.rb +++ b/spec/bundler/install/yanked_spec.rb @@ -188,7 +188,7 @@ RSpec.context "when using gem before installing" do bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1 in locally installed gems") expect(err).to_not include("Your bundle is locked to rack (0.9.1) from") expect(err).to_not include("If you haven't changed sources, that means the author of rack (0.9.1) has removed it.") expect(err).to_not include("You'll need to update your bundle to a different version of rack (0.9.1) that hasn't been removed in order to install.") @@ -197,7 +197,7 @@ RSpec.context "when using gem before installing" do lockfile lockfile.gsub(/PLATFORMS\n #{lockfile_platforms}/m, "PLATFORMS\n #{lockfile_platforms("ruby")}") bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1 in locally installed gems") end it "does not suggest the author has yanked the gem when using more than one gem, but shows all gems that couldn't be found in the source" do @@ -224,7 +224,7 @@ RSpec.context "when using gem before installing" do bundle :list, raise_on_error: false - expect(err).to include("Could not find rack-0.9.1, rack_middleware-1.0 in cached gems or installed locally") + expect(err).to include("Could not find rack-0.9.1, rack_middleware-1.0 in locally installed gems") expect(err).to include("Install missing gems with `bundle install`.") expect(err).to_not include("Your bundle is locked to rack (0.9.1) from") expect(err).to_not include("If you haven't changed sources, that means the author of rack (0.9.1) has removed it.") diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index 939b68a0bb..e7577d38b4 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -618,7 +618,12 @@ RSpec.describe "major deprecations" do pending "fails with a helpful message", bundler: "3" end - describe "deprecating rubocop", :readline do + describe "deprecating rubocop" do + before do + global_config "BUNDLE_GEM__MIT" => "false", "BUNDLE_GEM__TEST" => "false", "BUNDLE_GEM__COC" => "false", + "BUNDLE_GEM__CI" => "false", "BUNDLE_GEM__CHANGELOG" => "false" + end + context "bundle gem --rubocop" do before do bundle "gem my_new_gem --rubocop", raise_on_error: false diff --git a/spec/bundler/runtime/require_spec.rb b/spec/bundler/runtime/require_spec.rb index 76271a5593..e630e902c9 100644 --- a/spec/bundler/runtime/require_spec.rb +++ b/spec/bundler/runtime/require_spec.rb @@ -430,6 +430,30 @@ RSpec.describe "Bundler.require" do expect(out).to eq("WIN") end + + it "does not extract gemspecs from application cache packages" do + gemfile <<-G + source "#{file_uri_for(gem_repo1)}" + gem "rack" + G + + bundle :cache + + path = cached_gem("rack-1.0.0") + + run <<-R + File.open("#{path}", "w") do |f| + f.write "broken package" + end + R + + run <<-R + Bundler.require + puts "WIN" + R + + expect(out).to eq("WIN") + end end RSpec.describe "Bundler.require with platform specific dependencies" do diff --git a/spec/bundler/runtime/setup_spec.rb b/spec/bundler/runtime/setup_spec.rb index 0344d24223..8b8988063c 100644 --- a/spec/bundler/runtime/setup_spec.rb +++ b/spec/bundler/runtime/setup_spec.rb @@ -767,6 +767,18 @@ end expect(err).to be_empty end + it "can require rubygems without warnings, when using a local cache" do + install_gemfile <<-G + source "#{file_uri_for(gem_repo1)}" + gem "rack" + G + + bundle "package" + bundle %(exec ruby -w -e "require 'rubygems'") + + expect(err).to be_empty + end + context "when the user has `MANPATH` set", :man do before { ENV["MANPATH"] = "/foo#{File::PATH_SEPARATOR}" } diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index ab2dafb0b9..8f646b9358 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -518,7 +518,6 @@ module Spec if options[:rubygems_version] @spec.rubygems_version = options[:rubygems_version] - def @spec.mark_version; end def @spec.validate(*); end end diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb index 1ad9cc78ca..dbb8b8152b 100644 --- a/spec/bundler/support/helpers.rb +++ b/spec/bundler/support/helpers.rb @@ -226,12 +226,20 @@ module Spec end def config(config = nil, path = bundled_app(".bundle/config")) - return Psych.load_file(path) unless config + current = File.exist?(path) ? Psych.load_file(path) : {} + return current unless config + + current = {} if current.empty? + FileUtils.mkdir_p(File.dirname(path)) - File.open(path, "w") do |f| - f.puts config.to_yaml + + new_config = current.merge(config).compact + + File.open(path, "w+") do |f| + f.puts new_config.to_yaml end - config + + new_config end def global_config(config = nil) diff --git a/spec/prism.mspec b/spec/prism.mspec deleted file mode 100644 index c13b58b1cd..0000000000 --- a/spec/prism.mspec +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -# This is turned off because when we run with --parser=prism we explicitly turn -# off experimental warnings to make sure the output is consistent. -MSpec.register(:exclude, "Warning.[] returns default values for categories :deprecated and :experimental") - -## Language -MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes") -MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes and 'key: value' syntax used") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation /o") -MSpec.register(:exclude, "Regexps with encoding modifiers preserves EUC-JP as /e encoding through interpolation") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation and /o") -MSpec.register(:exclude, "Regexps with encoding modifiers preserves Windows-31J as /s encoding through interpolation") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation") -MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation and /o") -MSpec.register(:exclude, "Regexps with encoding modifiers preserves UTF-8 as /u encoding through interpolation") -MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time when Symbol with invalid bytes") - -## Core -MSpec.register(:exclude, "TracePoint#path equals \"(eval at __FILE__:__LINE__)\" inside an eval for :end event") - -## Library -MSpec.register(:exclude, "Coverage.result returns the correct results when eval coverage is disabled") -MSpec.register(:exclude, "Socket.gethostbyaddr using an IPv6 address with an explicit address family raises SocketError when the address is not supported by the family") diff --git a/spec/ruby/core/binding/dup_spec.rb b/spec/ruby/core/binding/dup_spec.rb index 55fac6e333..4eff66bd9a 100644 --- a/spec/ruby/core/binding/dup_spec.rb +++ b/spec/ruby/core/binding/dup_spec.rb @@ -10,4 +10,21 @@ describe "Binding#dup" do bind.frozen?.should == true bind.dup.frozen?.should == false end + + it "retains original binding variables but the list is distinct" do + bind1 = binding + eval "a = 1", bind1 + + bind2 = bind1.dup + eval("a = 2", bind2) + eval("a", bind1).should == 2 + eval("a", bind2).should == 2 + + eval("b = 2", bind2) + -> { eval("b", bind1) }.should raise_error(NameError) + eval("b", bind2).should == 2 + + bind1.local_variables.sort.should == [:a, :bind1, :bind2] + bind2.local_variables.sort.should == [:a, :b, :bind1, :bind2] + end end diff --git a/spec/ruby/core/binding/irb_spec.rb b/spec/ruby/core/binding/irb_spec.rb index 25521f0dd7..2607c7ef33 100644 --- a/spec/ruby/core/binding/irb_spec.rb +++ b/spec/ruby/core/binding/irb_spec.rb @@ -10,7 +10,7 @@ describe "Binding#irb" do IO.popen([envs, *ruby_exe, irb_fixture, chdir: dir], "r+") do |pipe| pipe.puts "a ** 2" pipe.puts "exit" - pipe.readlines.map(&:chomp) + pipe.readlines.map(&:chomp).reject(&:empty?) end end diff --git a/spec/ruby/core/enumerator/next_values_spec.rb b/spec/ruby/core/enumerator/next_values_spec.rb index 201b5d323f..2202700c58 100644 --- a/spec/ruby/core/enumerator/next_values_spec.rb +++ b/spec/ruby/core/enumerator/next_values_spec.rb @@ -11,6 +11,7 @@ describe "Enumerator#next_values" do yield :e1, :e2, :e3 yield nil yield + yield [:f1, :f2] end @e = o.to_enum @@ -48,8 +49,13 @@ describe "Enumerator#next_values" do @e.next_values.should == [] end - it "raises StopIteration if called on a finished enumerator" do + it "returns an array of array if yield is called with an array" do 7.times { @e.next } + @e.next_values.should == [[:f1, :f2]] + end + + it "raises StopIteration if called on a finished enumerator" do + 8.times { @e.next } -> { @e.next_values }.should raise_error(StopIteration) end end diff --git a/spec/ruby/core/enumerator/peek_values_spec.rb b/spec/ruby/core/enumerator/peek_values_spec.rb index 7865546515..8b84fc8afc 100644 --- a/spec/ruby/core/enumerator/peek_values_spec.rb +++ b/spec/ruby/core/enumerator/peek_values_spec.rb @@ -11,6 +11,7 @@ describe "Enumerator#peek_values" do yield :e1, :e2, :e3 yield nil yield + yield [:f1, :f2] end @e = o.to_enum @@ -50,8 +51,13 @@ describe "Enumerator#peek_values" do @e.peek_values.should == [] end - it "raises StopIteration if called on a finished enumerator" do + it "returns an array of array if yield is called with an array" do 7.times { @e.next } + @e.peek_values.should == [[:f1, :f2]] + end + + it "raises StopIteration if called on a finished enumerator" do + 8.times { @e.next } -> { @e.peek_values }.should raise_error(StopIteration) end end diff --git a/spec/ruby/core/io/pread_spec.rb b/spec/ruby/core/io/pread_spec.rb index 28afc80e5c..6d93b432c2 100644 --- a/spec/ruby/core/io/pread_spec.rb +++ b/spec/ruby/core/io/pread_spec.rb @@ -22,7 +22,7 @@ guard -> { platform_is_not :windows or ruby_version_is "3.3" } do it "accepts a length, an offset, and an output buffer" do buffer = +"foo" - @file.pread(3, 4, buffer) + @file.pread(3, 4, buffer).should.equal?(buffer) buffer.should == "567" end @@ -38,6 +38,13 @@ guard -> { platform_is_not :windows or ruby_version_is "3.3" } do buffer.should == "12345" end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @file.pread(10, 0, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + it "does not advance the file pointer" do @file.pread(4, 0).should == "1234" @file.read.should == "1234567890" diff --git a/spec/ruby/core/io/read_spec.rb b/spec/ruby/core/io/read_spec.rb index eb3652e692..8741d9f017 100644 --- a/spec/ruby/core/io/read_spec.rb +++ b/spec/ruby/core/io/read_spec.rb @@ -376,6 +376,21 @@ describe "IO#read" do buf.should == @contents[0..4] end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.read(10, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + + # https://bugs.ruby-lang.org/issues/20416 + it "does not preserve the encoding of the given buffer when max length is not provided" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.read(nil, buffer) + + buffer.encoding.should_not == Encoding::ISO_8859_1 + end + it "returns the given buffer" do buf = +"" diff --git a/spec/ruby/core/io/readpartial_spec.rb b/spec/ruby/core/io/readpartial_spec.rb index 0060beb545..0852f20b2d 100644 --- a/spec/ruby/core/io/readpartial_spec.rb +++ b/spec/ruby/core/io/readpartial_spec.rb @@ -62,7 +62,7 @@ describe "IO#readpartial" do buffer = +"existing content" @wr.write("hello world") @wr.close - @rd.readpartial(11, buffer) + @rd.readpartial(11, buffer).should.equal?(buffer) buffer.should == "hello world" end @@ -106,6 +106,7 @@ describe "IO#readpartial" do @wr.write("abc") @wr.close @rd.readpartial(10, buffer) + buffer.encoding.should == Encoding::ISO_8859_1 end end diff --git a/spec/ruby/core/io/sysread_spec.rb b/spec/ruby/core/io/sysread_spec.rb index 003bb9eb94..8851214283 100644 --- a/spec/ruby/core/io/sysread_spec.rb +++ b/spec/ruby/core/io/sysread_spec.rb @@ -97,7 +97,7 @@ describe "IO#sysread on a file" do it "discards the existing buffer content upon successful read" do buffer = +"existing content" - @file.sysread(11, buffer) + @file.sysread(11, buffer).should.equal?(buffer) buffer.should == "01234567890" end @@ -107,6 +107,13 @@ describe "IO#sysread on a file" do -> { @file.sysread(1, buffer) }.should raise_error(EOFError) buffer.should be_empty end + + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + string = @file.sysread(10, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end end describe "IO#sysread" do diff --git a/spec/ruby/core/module/include_spec.rb b/spec/ruby/core/module/include_spec.rb index c073bc31ca..78f6b41031 100644 --- a/spec/ruby/core/module/include_spec.rb +++ b/spec/ruby/core/module/include_spec.rb @@ -47,6 +47,34 @@ describe "Module#include" do -> { ModuleSpecs::SubclassSpec.include(ModuleSpecs::Subclass.new) }.should_not raise_error(TypeError) end + ruby_version_is ""..."3.2" do + it "raises ArgumentError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.include(refinement) }.should raise_error(ArgumentError, "refinement module is not allowed") + end + end + + ruby_version_is "3.2" do + it "raises a TypeError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.include(refinement) }.should raise_error(TypeError, "Cannot include refinement") + end + end + it "imports constants to modules and classes" do ModuleSpecs::A.constants.should include(:CONSTANT_A) ModuleSpecs::B.constants.should include(:CONSTANT_A, :CONSTANT_B) diff --git a/spec/ruby/core/module/prepend_spec.rb b/spec/ruby/core/module/prepend_spec.rb index c90fa9700e..b40d12f0de 100644 --- a/spec/ruby/core/module/prepend_spec.rb +++ b/spec/ruby/core/module/prepend_spec.rb @@ -75,6 +75,26 @@ describe "Module#prepend" do foo.call.should == 'm' end + it "updates the optimized method when a prepended module is updated" do + out = ruby_exe(<<~RUBY) + module M; end + class Integer + prepend M + end + l = -> { 1 + 2 } + p l.call + M.module_eval do + def +(o) + $called = true + super(o) + end + end + p l.call + p $called + RUBY + out.should == "3\n3\ntrue\n" + end + it "updates the method when there is a base included method and the prepended module overrides it" do base_module = Module.new do def foo @@ -415,6 +435,34 @@ describe "Module#prepend" do -> { ModuleSpecs::SubclassSpec.prepend(ModuleSpecs::Subclass.new) }.should_not raise_error(TypeError) end + ruby_version_is ""..."3.2" do + it "raises ArgumentError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.prepend(refinement) }.should raise_error(ArgumentError, "refinement module is not allowed") + end + end + + ruby_version_is "3.2" do + it "raises a TypeError when the argument is a refinement" do + refinement = nil + + Module.new do + refine String do + refinement = self + end + end + + -> { ModuleSpecs::Basic.prepend(refinement) }.should raise_error(TypeError, "Cannot prepend refinement") + end + end + it "imports constants" do m1 = Module.new m1::MY_CONSTANT = 1 diff --git a/spec/ruby/core/string/index_spec.rb b/spec/ruby/core/string/index_spec.rb index be79708045..835263a2cd 100644 --- a/spec/ruby/core/string/index_spec.rb +++ b/spec/ruby/core/string/index_spec.rb @@ -231,6 +231,17 @@ describe "String#index with Regexp" do $~.should == nil end + ruby_bug "#20421", ""..."3.3" do + it "always clear $~" do + "a".index(/a/) + $~.should_not == nil + + string = "blablabla" + string.index(/bla/, string.length + 1) + $~.should == nil + end + end + it "starts the search at the given offset" do "blablabla".index(/.{0}/, 5).should == 5 "blablabla".index(/.{1}/, 5).should == 5 diff --git a/spec/ruby/core/warning/element_reference_spec.rb b/spec/ruby/core/warning/element_reference_spec.rb index 8cb4018c20..c0ed37ef13 100644 --- a/spec/ruby/core/warning/element_reference_spec.rb +++ b/spec/ruby/core/warning/element_reference_spec.rb @@ -2,6 +2,10 @@ require_relative '../../spec_helper' describe "Warning.[]" do it "returns default values for categories :deprecated and :experimental" do + # If any warning options were set on the Ruby that will be executed, then + # it's possible this test will fail. In this case we will skip this test. + skip if ruby_exe.any? { |opt| opt.start_with?("-W") } + ruby_exe('p [Warning[:deprecated], Warning[:experimental]]').chomp.should == "[false, true]" ruby_exe('p [Warning[:deprecated], Warning[:experimental]]', options: "-w").chomp.should == "[true, true]" end diff --git a/spec/ruby/core/warning/performance_warning_spec.rb b/spec/ruby/core/warning/performance_warning_spec.rb new file mode 100644 index 0000000000..ab0badcd3d --- /dev/null +++ b/spec/ruby/core/warning/performance_warning_spec.rb @@ -0,0 +1,28 @@ +require_relative '../../spec_helper' + + +describe "Performance warnings" do + guard -> { ruby_version_is("3.4") || RUBY_ENGINE == "truffleruby" } do + # Optimising Integer, Float or Symbol methods is kind of implementation detail + # but multiple implementations do so. So it seems reasonable to have a test case + # for at least one such common method. + # See https://bugs.ruby-lang.org/issues/20429 + context "when redefined optimised methods" do + it "emits performance warning for redefining Integer#+" do + code = <<~CODE + Warning[:performance] = true + + class Integer + ORIG_METHOD = instance_method(:+) + + def +(...) + ORIG_METHOD.bind(self).call(...) + end + end + CODE + + ruby_exe(code, args: "2>&1").should.include?("warning: Redefining 'Integer#+' disables interpreter and JIT optimizations") + end + end + end +end diff --git a/spec/ruby/language/break_spec.rb b/spec/ruby/language/break_spec.rb index e725e77e80..7e5b6fb328 100644 --- a/spec/ruby/language/break_spec.rb +++ b/spec/ruby/language/break_spec.rb @@ -252,6 +252,25 @@ describe "Break inside a while loop" do end end +describe "The break statement in a method" do + it "is invalid and raises a SyntaxError" do + -> { + eval("def m; break; end") + }.should raise_error(SyntaxError) + end +end + +describe "The break statement in a module literal" do + it "is invalid and raises a SyntaxError" do + code = <<~RUBY + module BreakSpecs:ModuleWithBreak + break + end + RUBY + + -> { eval(code) }.should raise_error(SyntaxError) + end +end # TODO: Rewrite all the specs from here to the end of the file in the style # above. diff --git a/spec/ruby/language/case_spec.rb b/spec/ruby/language/case_spec.rb index 3262f09dd5..d289eca037 100644 --- a/spec/ruby/language/case_spec.rb +++ b/spec/ruby/language/case_spec.rb @@ -416,17 +416,34 @@ describe "The 'case'-construct" do self.test(true).should == true end - it "warns if there are identical when clauses" do - -> { - eval <<~RUBY - case 1 - when 2 - :foo - when 2 - :bar - end - RUBY - }.should complain(/warning: duplicated .when' clause with line \d+ is ignored/, verbose: true) + ruby_version_is ""..."3.4" do + it "warns if there are identical when clauses" do + -> { + eval <<~RUBY + case 1 + when 2 + :foo + when 2 + :bar + end + RUBY + }.should complain(/warning: duplicated .when' clause with line \d+ is ignored/, verbose: true) + end + end + + ruby_version_is "3.4" do + it "warns if there are identical when clauses" do + -> { + eval <<~RUBY + case 1 + when 2 + :foo + when 2 + :bar + end + RUBY + }.should complain(/warning: 'when' clause on line \d+ duplicates 'when' clause on line \d+ and is ignored/, verbose: true) + end end end diff --git a/spec/ruby/language/execution_spec.rb b/spec/ruby/language/execution_spec.rb index ef1de38899..51bcde62e8 100644 --- a/spec/ruby/language/execution_spec.rb +++ b/spec/ruby/language/execution_spec.rb @@ -38,7 +38,7 @@ describe "``" do 2.times do runner.instance_exec do - `test #{:command}` + `test #{:command}` # rubocop:disable Lint/LiteralInInterpolation end end @@ -84,7 +84,7 @@ describe "%x" do 2.times do runner.instance_exec do - %x{test #{:command}} + %x{test #{:command}} # rubocop:disable Lint/LiteralInInterpolation end end diff --git a/spec/ruby/language/hash_spec.rb b/spec/ruby/language/hash_spec.rb index a7631fb0d6..068ac0f39c 100644 --- a/spec/ruby/language/hash_spec.rb +++ b/spec/ruby/language/hash_spec.rb @@ -86,6 +86,30 @@ describe "Hash literal" do -> { eval("{:a ==> 1}") }.should raise_error(SyntaxError) end + it "recognizes '!' at the end of the key" do + eval("{:a! =>1}").should == {:"a!" => 1} + eval("{:a! => 1}").should == {:"a!" => 1} + + eval("{a!:1}").should == {:"a!" => 1} + eval("{a!: 1}").should == {:"a!" => 1} + end + + it "raises a SyntaxError if there is no space between `!` and `=>`" do + -> { eval("{:a!=> 1}") }.should raise_error(SyntaxError) + end + + it "recognizes '?' at the end of the key" do + eval("{:a? =>1}").should == {:"a?" => 1} + eval("{:a? => 1}").should == {:"a?" => 1} + + eval("{a?:1}").should == {:"a?" => 1} + eval("{a?: 1}").should == {:"a?" => 1} + end + + it "raises a SyntaxError if there is no space between `?` and `=>`" do + -> { eval("{:a?=> 1}") }.should raise_error(SyntaxError) + end + it "constructs a new hash with the given elements" do {foo: 123}.should == {foo: 123} h = {rbx: :cool, specs: 'fail_sometimes'} @@ -271,6 +295,14 @@ describe "The ** operator" do a.new.foo(1).should == {bar: "baz", val: 1} end + + it "raises a SyntaxError when the hash key ends with `!`" do + -> { eval("{a!:}") }.should raise_error(SyntaxError, /identifier a! is not valid to get/) + end + + it "raises a SyntaxError when the hash key ends with `?`" do + -> { eval("{a?:}") }.should raise_error(SyntaxError, /identifier a\? is not valid to get/) + end end end end diff --git a/spec/ruby/language/keyword_arguments_spec.rb b/spec/ruby/language/keyword_arguments_spec.rb index ffb5b1fab0..8668799d26 100644 --- a/spec/ruby/language/keyword_arguments_spec.rb +++ b/spec/ruby/language/keyword_arguments_spec.rb @@ -395,4 +395,32 @@ describe "Keyword arguments" do end end end + + context "in define_method(name, &proc)" do + # This tests that a free-standing proc used in define_method and converted to ruby2_keywords adopts that logic. + # See jruby/jruby#8119 for a case where aggressive JIT optimization broke later ruby2_keywords changes. + it "works with ruby2_keywords" do + m = Class.new do + def bar(a, foo: nil) + [a, foo] + end + + # define_method and ruby2_keywords using send to avoid peephole optimizations + def self.setup + pr = make_proc + send :define_method, :foo, &pr + send :ruby2_keywords, :foo + end + + # create proc in isolated method to force jit compilation on some implementations + def self.make_proc + proc { |a, *args| bar(a, *args) } + end + end + + m.setup + + m.new.foo(1, foo:2).should == [1, 2] + end + end end diff --git a/spec/ruby/language/regexp/back-references_spec.rb b/spec/ruby/language/regexp/back-references_spec.rb index 26750c20c5..627c8daace 100644 --- a/spec/ruby/language/regexp/back-references_spec.rb +++ b/spec/ruby/language/regexp/back-references_spec.rb @@ -22,6 +22,15 @@ describe "Regexps with back-references" do $10.should == "0" end + it "returns nil for numbered variable with too large index" do + -> { + eval(<<~CODE).should == nil + "a" =~ /(.)/ + eval('$4294967296') + CODE + }.should complain(/warning: ('|`)\$4294967296' is too big for a number variable, always nil/) + end + it "will not clobber capture variables across threads" do cap1, cap2, cap3 = nil "foo" =~ /(o+)/ diff --git a/spec/ruby/language/retry_spec.rb b/spec/ruby/language/retry_spec.rb index ee5377946f..669d5f0ff5 100644 --- a/spec/ruby/language/retry_spec.rb +++ b/spec/ruby/language/retry_spec.rb @@ -31,8 +31,11 @@ describe "The retry statement" do results.should == [1, 2, 3, 1, 2, 4, 5, 6, 4, 5] end - it "raises a SyntaxError when used outside of a begin statement" do + it "raises a SyntaxError when used outside of a rescue statement" do -> { eval 'retry' }.should raise_error(SyntaxError) + -> { eval 'begin; retry; end' }.should raise_error(SyntaxError) + -> { eval 'def m; retry; end' }.should raise_error(SyntaxError) + -> { eval 'module RetrySpecs; retry; end' }.should raise_error(SyntaxError) end end diff --git a/spec/ruby/language/yield_spec.rb b/spec/ruby/language/yield_spec.rb index 5283517636..e125cf8e73 100644 --- a/spec/ruby/language/yield_spec.rb +++ b/spec/ruby/language/yield_spec.rb @@ -206,3 +206,15 @@ describe "Using yield in non-lambda block" do -> { eval(code) }.should raise_error(SyntaxError, /Invalid yield/) end end + +describe "Using yield in a module literal" do + it 'raises a SyntaxError' do + code = <<~RUBY + module YieldSpecs::ModuleWithYield + yield + end + RUBY + + -> { eval(code) }.should raise_error(SyntaxError, /Invalid yield/) + end +end diff --git a/spec/ruby/library/time/to_date_spec.rb b/spec/ruby/library/date/time/to_date_spec.rb index baeafe0847..f9132da289 100644 --- a/spec/ruby/library/time/to_date_spec.rb +++ b/spec/ruby/library/date/time/to_date_spec.rb @@ -1,5 +1,5 @@ -require_relative '../../spec_helper' +require_relative '../../../spec_helper' require 'time' describe "Time#to_date" do diff --git a/spec/ruby/library/time/to_datetime_spec.rb b/spec/ruby/library/datetime/time/to_datetime_spec.rb index 9c44f38e5c..1125dbe851 100644 --- a/spec/ruby/library/time/to_datetime_spec.rb +++ b/spec/ruby/library/datetime/time/to_datetime_spec.rb @@ -1,4 +1,4 @@ -require_relative '../../spec_helper' +require_relative '../../../spec_helper' require 'time' require 'date' date_version = defined?(Date::VERSION) ? Date::VERSION : '3.1.0' diff --git a/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb b/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb index 17c846054d..df42c116fb 100644 --- a/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_nonblock_spec.rb @@ -52,9 +52,19 @@ describe "Socket::BasicSocket#recv_nonblock" do @s2.send("data", 0, @s1.getsockname) IO.select([@s1], nil, nil, 2) - buf = +"foo" - @s1.recv_nonblock(5, 0, buf) - buf.should == "data" + buffer = +"foo" + @s1.recv_nonblock(5, 0, buffer).should.equal?(buffer) + buffer.should == "data" + end + + it "preserves the encoding of the given buffer" do + @s1.bind(Socket.pack_sockaddr_in(0, ip_address)) + @s2.send("data", 0, @s1.getsockname) + IO.select([@s1], nil, nil, 2) + + buffer = ''.encode(Encoding::ISO_8859_1) + @s1.recv_nonblock(5, 0, buffer) + buffer.encoding.should == Encoding::ISO_8859_1 end it "does not block if there's no data available" do diff --git a/spec/ruby/library/socket/basicsocket/recv_spec.rb b/spec/ruby/library/socket/basicsocket/recv_spec.rb index 9fe8c52f9a..e82a357d3d 100644 --- a/spec/ruby/library/socket/basicsocket/recv_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_spec.rb @@ -100,13 +100,29 @@ describe "BasicSocket#recv" do socket.write("data") client = @server.accept - buf = +"foo" + buffer = +"foo" begin - client.recv(4, 0, buf) + client.recv(4, 0, buffer).should.equal?(buffer) ensure client.close end - buf.should == "data" + buffer.should == "data" + + socket.close + end + + it "preserves the encoding of the given buffer" do + socket = TCPSocket.new('127.0.0.1', @port) + socket.write("data") + + client = @server.accept + buffer = ''.encode(Encoding::ISO_8859_1) + begin + client.recv(4, 0, buffer) + ensure + client.close + end + buffer.encoding.should == Encoding::ISO_8859_1 socket.close end diff --git a/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb b/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb index 94f58ac49f..5596f91bb8 100644 --- a/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb +++ b/spec/ruby/library/socket/socket/recvfrom_nonblock_spec.rb @@ -52,6 +52,27 @@ describe 'Socket#recvfrom_nonblock' do end end + it "allows an output buffer as third argument" do + @client.write('hello') + + IO.select([@server]) + buffer = +'' + message, = @server.recvfrom_nonblock(5, 0, buffer) + + message.should.equal?(buffer) + buffer.should == 'hello' + end + + it "preserves the encoding of the given buffer" do + @client.write('hello') + + IO.select([@server]) + buffer = ''.encode(Encoding::ISO_8859_1) + @server.recvfrom_nonblock(5, 0, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + describe 'the returned data' do it 'is the same as the sent data' do 5.times do diff --git a/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb b/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb index 650a061221..b804099589 100644 --- a/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb +++ b/spec/ruby/library/socket/udpsocket/recvfrom_nonblock_spec.rb @@ -58,6 +58,15 @@ describe 'UDPSocket#recvfrom_nonblock' do buffer.should == 'h' end + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + IO.select([@server]) + message, = @server.recvfrom_nonblock(1, 0, buffer) + + message.should.equal?(buffer) + buffer.encoding.should == Encoding::ISO_8859_1 + end + describe 'the returned Array' do before do IO.select([@server]) diff --git a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb index fedf74bb2f..d849fdc302 100644 --- a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb +++ b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb @@ -31,6 +31,29 @@ with_feature :unix_socket do sock.close end + it "allows an output buffer as third argument" do + buffer = +'' + + @client.send("foobar", 0) + sock = @server.accept + message, = sock.recvfrom(6, 0, buffer) + sock.close + + message.should.equal?(buffer) + buffer.should == "foobar" + end + + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + + @client.send("foobar", 0) + sock = @server.accept + sock.recvfrom(6, 0, buffer) + sock.close + + buffer.encoding.should == Encoding::ISO_8859_1 + end + it "uses different message options" do @client.send("foobar", Socket::MSG_PEEK) sock = @server.accept diff --git a/spec/ruby/library/stringio/shared/read.rb b/spec/ruby/library/stringio/shared/read.rb index e3840786d9..8ef6ec2734 100644 --- a/spec/ruby/library/stringio/shared/read.rb +++ b/spec/ruby/library/stringio/shared/read.rb @@ -11,10 +11,28 @@ describe :stringio_read, shared: true do end it "reads length bytes and writes them to the buffer String" do - @io.send(@method, 7, buffer = +"") + @io.send(@method, 7, buffer = +"").should.equal?(buffer) buffer.should == "example" end + ruby_version_is ""..."3.4" do + it "does not preserve the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.send(@method, 7, buffer) + + buffer.encoding.should_not == Encoding::ISO_8859_1 + end + end + + ruby_version_is "3.4" do + it "preserves the encoding of the given buffer" do + buffer = ''.encode(Encoding::ISO_8859_1) + @io.send(@method, 7, buffer) + + buffer.encoding.should == Encoding::ISO_8859_1 + end + end + it "tries to convert the passed buffer Object to a String using #to_str" do obj = mock("to_str") obj.should_receive(:to_str).and_return(buffer = +"") diff --git a/spec/ruby/optional/capi/class_spec.rb b/spec/ruby/optional/capi/class_spec.rb index d0a9913570..a231245ebe 100644 --- a/spec/ruby/optional/capi/class_spec.rb +++ b/spec/ruby/optional/capi/class_spec.rb @@ -487,4 +487,16 @@ describe "C-API Class function" do @s.rb_class_real(0).should == 0 end end + + describe "rb_class_get_superclass" do + it "returns parent class for a provided class" do + a = Class.new + @s.rb_class_get_superclass(Class.new(a)).should == a + end + + it "returns false when there is no parent class" do + @s.rb_class_get_superclass(BasicObject).should == false + @s.rb_class_get_superclass(Module.new).should == false + end + end end diff --git a/spec/ruby/optional/capi/ext/class_spec.c b/spec/ruby/optional/capi/ext/class_spec.c index f376534924..c13f02ecf2 100644 --- a/spec/ruby/optional/capi/ext/class_spec.c +++ b/spec/ruby/optional/capi/ext/class_spec.c @@ -79,6 +79,10 @@ static VALUE class_spec_rb_class_real(VALUE self, VALUE object) { } } +static VALUE class_spec_rb_class_get_superclass(VALUE self, VALUE klass) { + return rb_class_get_superclass(klass); +} + static VALUE class_spec_rb_class_superclass(VALUE self, VALUE klass) { return rb_class_superclass(klass); } @@ -160,6 +164,7 @@ void Init_class_spec(void) { rb_define_method(cls, "rb_class_new_instance_kw", class_spec_rb_class_new_instance_kw, 2); #endif rb_define_method(cls, "rb_class_real", class_spec_rb_class_real, 1); + rb_define_method(cls, "rb_class_get_superclass", class_spec_rb_class_get_superclass, 1); rb_define_method(cls, "rb_class_superclass", class_spec_rb_class_superclass, 1); rb_define_method(cls, "rb_cvar_defined", class_spec_cvar_defined, 2); rb_define_method(cls, "rb_cvar_get", class_spec_cvar_get, 2); diff --git a/spec/ruby/optional/capi/ext/gc_spec.c b/spec/ruby/optional/capi/ext/gc_spec.c index 1392bc6ee6..2637ad27ac 100644 --- a/spec/ruby/optional/capi/ext/gc_spec.c +++ b/spec/ruby/optional/capi/ext/gc_spec.c @@ -16,6 +16,8 @@ VALUE registered_after_rb_global_variable_bignum; VALUE registered_after_rb_global_variable_float; VALUE rb_gc_register_address_outside_init; +VALUE rb_gc_register_mark_object_not_referenced_float; + static VALUE registered_tagged_address(VALUE self) { return registered_tagged_value; } @@ -90,6 +92,10 @@ static VALUE gc_spec_rb_gc_register_mark_object(VALUE self, VALUE obj) { return Qnil; } +static VALUE gc_spec_rb_gc_register_mark_object_not_referenced_float(VALUE self) { + return rb_gc_register_mark_object_not_referenced_float; +} + void Init_gc_spec(void) { VALUE cls = rb_define_class("CApiGCSpecs", rb_cObject); @@ -115,6 +121,9 @@ void Init_gc_spec(void) { registered_after_rb_global_variable_float = DBL2NUM(6.28); rb_global_variable(®istered_after_rb_global_variable_float); + rb_gc_register_mark_object_not_referenced_float = DBL2NUM(1.61); + rb_gc_register_mark_object(rb_gc_register_mark_object_not_referenced_float); + rb_define_method(cls, "registered_tagged_address", registered_tagged_address, 0); rb_define_method(cls, "registered_reference_address", registered_reference_address, 0); rb_define_method(cls, "registered_before_rb_gc_register_address", get_registered_before_rb_gc_register_address, 0); @@ -131,6 +140,7 @@ void Init_gc_spec(void) { rb_define_method(cls, "rb_gc", gc_spec_rb_gc, 0); rb_define_method(cls, "rb_gc_adjust_memory_usage", gc_spec_rb_gc_adjust_memory_usage, 1); rb_define_method(cls, "rb_gc_register_mark_object", gc_spec_rb_gc_register_mark_object, 1); + rb_define_method(cls, "rb_gc_register_mark_object_not_referenced_float", gc_spec_rb_gc_register_mark_object_not_referenced_float, 0); rb_define_method(cls, "rb_gc_latest_gc_info", gc_spec_rb_gc_latest_gc_info, 1); } diff --git a/spec/ruby/optional/capi/ext/io_spec.c b/spec/ruby/optional/capi/ext/io_spec.c index bcd3940e34..1a73331386 100644 --- a/spec/ruby/optional/capi/ext/io_spec.c +++ b/spec/ruby/optional/capi/ext/io_spec.c @@ -157,7 +157,7 @@ VALUE io_spec_rb_io_wait_readable(VALUE self, VALUE io, VALUE read_p) { return ret ? Qtrue : Qfalse; #else - UNREACHABLE; + UNREACHABLE_RETURN(Qnil); #endif } @@ -174,6 +174,10 @@ VALUE io_spec_rb_io_maybe_wait_writable(VALUE self, VALUE error, VALUE io, VALUE #endif #ifdef RUBY_VERSION_IS_3_1 +#ifdef SET_NON_BLOCKING_FAILS_ALWAYS +NORETURN(VALUE io_spec_rb_io_maybe_wait_readable(VALUE self, VALUE error, VALUE io, VALUE timeout, VALUE read_p)); +#endif + VALUE io_spec_rb_io_maybe_wait_readable(VALUE self, VALUE error, VALUE io, VALUE timeout, VALUE read_p) { int fd = io_spec_get_fd(io); #ifndef SET_NON_BLOCKING_FAILS_ALWAYS @@ -209,7 +213,7 @@ VALUE io_spec_rb_io_maybe_wait_readable(VALUE self, VALUE error, VALUE io, VALUE return INT2NUM(ret); #else - UNREACHABLE; + UNREACHABLE_RETURN(Qnil); #endif } #endif diff --git a/spec/ruby/optional/capi/ext/object_spec.c b/spec/ruby/optional/capi/ext/object_spec.c index 8aa98cc5ce..aa60662e1e 100644 --- a/spec/ruby/optional/capi/ext/object_spec.c +++ b/spec/ruby/optional/capi/ext/object_spec.c @@ -390,22 +390,22 @@ static VALUE speced_allocator(VALUE klass) { return instance; } -static VALUE define_alloc_func(VALUE self, VALUE klass) { +static VALUE object_spec_rb_define_alloc_func(VALUE self, VALUE klass) { rb_define_alloc_func(klass, speced_allocator); return Qnil; } -static VALUE undef_alloc_func(VALUE self, VALUE klass) { +static VALUE object_spec_rb_undef_alloc_func(VALUE self, VALUE klass) { rb_undef_alloc_func(klass); return Qnil; } -static VALUE speced_allocator_p(VALUE self, VALUE klass) { +static VALUE object_spec_speced_allocator_p(VALUE self, VALUE klass) { rb_alloc_func_t allocator = rb_get_alloc_func(klass); return (allocator == speced_allocator) ? Qtrue : Qfalse; } -static VALUE custom_alloc_func_p(VALUE self, VALUE klass) { +static VALUE object_spec_custom_alloc_func_p(VALUE self, VALUE klass) { rb_alloc_func_t allocator = rb_get_alloc_func(klass); return allocator ? Qtrue : Qfalse; } @@ -485,10 +485,10 @@ void Init_object_spec(void) { rb_define_method(cls, "rb_ivar_defined", object_spec_rb_ivar_defined, 2); rb_define_method(cls, "rb_copy_generic_ivar", object_spec_rb_copy_generic_ivar, 2); rb_define_method(cls, "rb_free_generic_ivar", object_spec_rb_free_generic_ivar, 1); - rb_define_method(cls, "rb_define_alloc_func", define_alloc_func, 1); - rb_define_method(cls, "rb_undef_alloc_func", undef_alloc_func, 1); - rb_define_method(cls, "speced_allocator?", speced_allocator_p, 1); - rb_define_method(cls, "custom_alloc_func?", custom_alloc_func_p, 1); + rb_define_method(cls, "rb_define_alloc_func", object_spec_rb_define_alloc_func, 1); + rb_define_method(cls, "rb_undef_alloc_func", object_spec_rb_undef_alloc_func, 1); + rb_define_method(cls, "speced_allocator?", object_spec_speced_allocator_p, 1); + rb_define_method(cls, "custom_alloc_func?", object_spec_custom_alloc_func_p, 1); rb_define_method(cls, "not_implemented_method", rb_f_notimplement, -1); rb_define_method(cls, "rb_ivar_foreach", object_spec_rb_ivar_foreach, 1); } diff --git a/spec/ruby/optional/capi/gc_spec.rb b/spec/ruby/optional/capi/gc_spec.rb index aaced56483..d9661328ab 100644 --- a/spec/ruby/optional/capi/gc_spec.rb +++ b/spec/ruby/optional/capi/gc_spec.rb @@ -108,6 +108,10 @@ describe "CApiGCSpecs" do it "can be called with an object" do @f.rb_gc_register_mark_object(Object.new).should be_nil end + + it "keeps the value alive even if the value is not referenced by any Ruby object" do + @f.rb_gc_register_mark_object_not_referenced_float.should == 1.61 + end end describe "rb_gc_latest_gc_info" do diff --git a/spec/ruby/shared/kernel/at_exit.rb b/spec/ruby/shared/kernel/at_exit.rb index 16d41cb01c..29db79bb39 100644 --- a/spec/ruby/shared/kernel/at_exit.rb +++ b/spec/ruby/shared/kernel/at_exit.rb @@ -30,6 +30,12 @@ describe :kernel_at_exit, shared: true do result.lines.should.include?("The exception matches: true (message=foo)\n") end + it "gives access to an exception raised in a previous handler" do + code = "#{@method} { print '$!.message = ' + $!.message }; #{@method} { raise 'foo' }" + result = ruby_exe(code, args: "2>&1", exit_status: 1) + result.lines.should.include?("$!.message = foo") + end + it "both exceptions in a handler and in the main script are printed" do code = "#{@method} { raise 'at_exit_error' }; raise 'main_script_error'" result = ruby_exe(code, args: "2>&1", exit_status: 1) @@ -311,7 +311,7 @@ # - {Comparing}[rdoc-ref:String@Methods+for+Comparing] # - {Modifying a String}[rdoc-ref:String@Methods+for+Modifying+a+String] # - {Converting to New String}[rdoc-ref:String@Methods+for+Converting+to+New+String] -# - {Converting to Non-String}[rdoc-ref:String@Methods+for+Converting+to+Non--5CString] +# - {Converting to Non-String}[rdoc-ref:String@Methods+for+Converting+to+Non-String] # - {Iterating}[rdoc-ref:String@Methods+for+Iterating] # # === Methods for Creating a +String+ diff --git a/template/Makefile.in b/template/Makefile.in index d9a3cbc065..813a727cf9 100644 --- a/template/Makefile.in +++ b/template/Makefile.in @@ -663,3 +663,28 @@ yes-test-leaked-globals: yes-test-leaked-globals-precheck PLATFORM=$(hdrdir)/ruby/$(PLATFORM_DIR).h $(srcdir)/configure.ac \ $(COMMONOBJS) $(LIBRUBY_FOR_LEAKED_GLOBALS:yes=$(LIBRUBY_SO)) $(ACTIONS_ENDGROUP) + +test-syntax-suggest-precheck: $(TEST_RUNNABLE)-test-syntax-suggest-precheck +no-test-syntax-suggest-precheck: +yes-test-syntax-suggest-precheck: main + +test-syntax-suggest-prepare: $(TEST_RUNNABLE)-test-syntax-suggest-prepare +no-test-syntax-suggest-prepare: no-test-syntax-suggest-precheck +yes-test-syntax-suggest-prepare: yes-test-syntax-suggest-precheck + $(ACTIONS_GROUP) + $(XRUBY) -C "$(srcdir)" bin/gem install --no-document \ + --install-dir .bundle --conservative "rspec:~> 3" + $(ACTIONS_ENDGROUP) + +RSPECOPTS = +SYNTAX_SUGGEST_SPECS = +PREPARE_SYNTAX_SUGGEST = $(TEST_RUNNABLE)-test-syntax-suggest-prepare +test-syntax-suggest: $(TEST_RUNNABLE)-test-syntax-suggest +yes-test-syntax-suggest: $(PREPARE_SYNTAX_SUGGEST) + $(ACTIONS_GROUP) + $(XRUBY) -C $(srcdir) -Ispec/syntax_suggest:spec/lib .bundle/bin/rspec \ + --require rspec/expectations \ + --require spec_helper --require formatter_overrides --require spec_coverage \ + $(RSPECOPTS) spec/syntax_suggest/$(SYNTAX_SUGGEST_SPECS) + $(ACTIONS_ENDGROUP) +no-test-syntax-suggest: diff --git a/template/prelude.c.tmpl b/template/prelude.c.tmpl index 386511f20c..e17a75da79 100644 --- a/template/prelude.c.tmpl +++ b/template/prelude.c.tmpl @@ -142,16 +142,16 @@ COMPILER_WARNING_POP #define PRELUDE_CODE(n) rb_utf8_str_new_static(prelude_code##n.L0, sizeof(prelude_code##n)) static VALUE -prelude_vast(VALUE name, VALUE code, int line) +prelude_ast_value(VALUE name, VALUE code, int line) { rb_ast_t *ast; - VALUE vast = rb_parser_compile_string_path(rb_parser_new(), name, code, line); - ast = rb_ruby_ast_data_get(vast); + VALUE ast_value = rb_parser_compile_string_path(rb_parser_new(), name, code, line); + ast = rb_ruby_ast_data_get(ast_value); if (!ast || !ast->body.root) { if (ast) rb_ast_dispose(ast); rb_exc_raise(rb_errinfo()); } - return vast; + return ast_value; } % end @@ -159,21 +159,21 @@ prelude_vast(VALUE name, VALUE code, int line) #define PRELUDE_VAST(n, name_str, start_line) \ (((sizeof(prelude_name<%='##'%><%=%>n) - prefix_len - 2) == namelen) && \ (strncmp(prelude_name<%='##'%><%=%>n + prefix_len, feature_name, namelen) == 0) ? \ - prelude_vast((name_str) = PRELUDE_NAME(n), PRELUDE_CODE(n), start_line) : Qnil) + prelude_ast_value((name_str) = PRELUDE_NAME(n), PRELUDE_CODE(n), start_line) : Qnil) VALUE -rb_builtin_vast(const char *feature_name, VALUE *name_str) +rb_builtin_ast_value(const char *feature_name, VALUE *name_str) { const size_t prefix_len = rb_strlen_lit("<internal:"); size_t namelen = strlen(feature_name); - VALUE vast = Qnil; + VALUE ast_value = Qnil; % @preludes.each_value do |i, prelude, lines, sub, start_line| % if sub - if (!NIL_P(vast = PRELUDE_VAST(<%=i%><%=%>, *name_str, <%=start_line%>))) return vast; + if (!NIL_P(ast_value = PRELUDE_VAST(<%=i%><%=%>, *name_str, <%=start_line%>))) return ast_value; % end % end - return vast; + return ast_value; } % end @@ -199,9 +199,9 @@ prelude_eval(VALUE code, VALUE name, int line) }; rb_ast_t *ast; - VALUE vast = prelude_vast(name, code, line); - ast = rb_ruby_ast_data_get(vast); - rb_iseq_eval(rb_iseq_new_with_opt(vast, name, name, Qnil, line, + VALUE ast_value = prelude_ast_value(name, code, line); + ast = rb_ruby_ast_data_get(ast_value); + rb_iseq_eval(rb_iseq_new_with_opt(ast_value, name, name, Qnil, line, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil)); rb_ast_dispose(ast); diff --git a/test/-ext-/thread/test_instrumentation_api.rb b/test/-ext-/thread/test_instrumentation_api.rb index 9a3b67fa10..663e41be53 100644 --- a/test/-ext-/thread/test_instrumentation_api.rb +++ b/test/-ext-/thread/test_instrumentation_api.rb @@ -54,7 +54,7 @@ class TestThreadInstrumentation < Test::Unit::TestCase thread&.join end - def test_muti_thread_timeline + def test_multi_thread_timeline threads = nil full_timeline = record do threads = threaded_cpu_bound_work(1.0) @@ -68,7 +68,7 @@ class TestThreadInstrumentation < Test::Unit::TestCase threads.each do |thread| timeline = timeline_for(thread, full_timeline) assert_consistent_timeline(timeline) - assert timeline.count(:suspended) > 1, "Expected threads to yield suspended at least once: #{timeline.inspect}" + assert_operator timeline.count(:suspended), :>=, 1, "Expected threads to yield suspended at least once: #{timeline.inspect}" end timeline = timeline_for(Thread.current, full_timeline) diff --git a/test/.excludes-prism/TestAssignment.rb b/test/.excludes-prism/TestAssignment.rb deleted file mode 100644 index 23a1b4c5e7..0000000000 --- a/test/.excludes-prism/TestAssignment.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_massign_order, "https://github.com/ruby/prism/issues/2370") diff --git a/test/.excludes-prism/TestCall.rb b/test/.excludes-prism/TestCall.rb deleted file mode 100644 index 969e32ea5a..0000000000 --- a/test/.excludes-prism/TestCall.rb +++ /dev/null @@ -1,3 +0,0 @@ -exclude(:test_call_op_asgn_keywords, "https://github.com/ruby/prism/issues/2438") -exclude(:test_call_op_asgn_keywords_mutable, "https://github.com/ruby/prism/issues/2438") -exclude(:test_kwsplat_block_order_op_asgn, "https://github.com/ruby/prism/issues/2438") diff --git a/test/.excludes-prism/TestCoverage.rb b/test/.excludes-prism/TestCoverage.rb deleted file mode 100644 index 20f9972f89..0000000000 --- a/test/.excludes-prism/TestCoverage.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_eval, "unknown") diff --git a/test/.excludes-prism/TestIRB/RubyLexTest.rb b/test/.excludes-prism/TestIRB/RubyLexTest.rb deleted file mode 100644 index 2274ae62cf..0000000000 --- a/test/.excludes-prism/TestIRB/RubyLexTest.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_code_block_open_with_should_continue, "symbol encoding") diff --git a/test/.excludes-prism/TestISeq.rb b/test/.excludes-prism/TestISeq.rb index 3768d8fc05..499ae2e53d 100644 --- a/test/.excludes-prism/TestISeq.rb +++ b/test/.excludes-prism/TestISeq.rb @@ -1,3 +1,2 @@ -exclude(:test_each_child, "https://github.com/ruby/prism/issues/2660") -exclude(:test_syntax_error_message, "Assertion checks against specific error format") -exclude(:test_trace_points, "https://github.com/ruby/prism/issues/2660") +exclude(:test_each_child, "https://bugs.ruby-lang.org/issues/20479") +exclude(:test_trace_points, "https://bugs.ruby-lang.org/issues/20479") diff --git a/test/.excludes-prism/TestM17N.rb b/test/.excludes-prism/TestM17N.rb index 15ed504fc9..7f8c44d02a 100644 --- a/test/.excludes-prism/TestM17N.rb +++ b/test/.excludes-prism/TestM17N.rb @@ -1,7 +1 @@ -exclude(:test_dynamic_eucjp_regexp, "https://github.com/ruby/prism/issues/2664") -exclude(:test_dynamic_sjis_regexp, "https://github.com/ruby/prism/issues/2664") -exclude(:test_dynamic_utf8_regexp, "https://github.com/ruby/prism/issues/2664") -exclude(:test_regexp_ascii, "https://github.com/ruby/prism/issues/2664") -exclude(:test_regexp_embed, "https://github.com/ruby/prism/issues/2664") -exclude(:test_regexp_usascii, "unknown") -exclude(:test_string_mixed_unicode, "unknown") +exclude(:test_regexp_usascii, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestMixedUnicodeEscape.rb b/test/.excludes-prism/TestMixedUnicodeEscape.rb index 09e3cc168b..7bf964ebf1 100644 --- a/test/.excludes-prism/TestMixedUnicodeEscape.rb +++ b/test/.excludes-prism/TestMixedUnicodeEscape.rb @@ -1 +1 @@ -exclude(:test_basic, "unknown") +exclude(:test_basic, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb deleted file mode 100644 index 83af593b15..0000000000 --- a/test/.excludes-prism/TestParse.rb +++ /dev/null @@ -1,28 +0,0 @@ -exclude(:test_dynamic_constant_assignment, "unknown") -exclude(:test_else_without_rescue, "unknown") -exclude(:test_error_def_in_argument, "unknown") -exclude(:test_float, "unknown") -exclude(:test_global_variable, "unknown") -exclude(:test_here_document, "unknown") -exclude(:test_heredoc_unterminated_interpolation, "unknown") -exclude(:test_invalid_char, "unknown") -exclude(:test_location_of_invalid_token, "unknown") -exclude(:test_op_asgn1_with_block, "unknown") -exclude(:test_parse_string, "unknown") -exclude(:test_percent, "unknown") -exclude(:test_question, "unknown") -exclude(:test_shareable_constant_value_ignored, "unknown") -exclude(:test_shareable_constant_value_nested, "ractor support") -exclude(:test_shareable_constant_value_nonliteral, "ractor support") -exclude(:test_shareable_constant_value_simple, "ractor support") -exclude(:test_shareable_constant_value_unfrozen, "ractor support") -exclude(:test_shareable_constant_value_unshareable_literal, "ractor support") -exclude(:test_string, "unknown") -exclude(:test_truncated_source_line, "unknown") -exclude(:test_unassignable, "unknown") -exclude(:test_unexpected_eof, "unknown") -exclude(:test_unexpected_token_after_numeric, "unknown") -exclude(:test_unterminated_regexp_error, "unknown") -exclude(:test_unused_variable, "missing warning") -exclude(:test_void_value_in_rhs, "unknown") -exclude(:test_words, "unknown") diff --git a/test/.excludes-prism/TestPatternMatching.rb b/test/.excludes-prism/TestPatternMatching.rb deleted file mode 100644 index cfd0c6bed9..0000000000 --- a/test/.excludes-prism/TestPatternMatching.rb +++ /dev/null @@ -1,2 +0,0 @@ -exclude(:test_hash_pattern, "useless literal warning missing") -exclude(:test_invalid_syntax, "[a:] is disallowed") diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb deleted file mode 100644 index 68ad1414a9..0000000000 --- a/test/.excludes-prism/TestRegexp.rb +++ /dev/null @@ -1,6 +0,0 @@ -exclude(:test_invalid_escape_error, "unknown") -exclude(:test_invalid_fragment, "https://github.com/ruby/prism/issues/2664") -exclude(:test_unescape, "unknown") -exclude(:test_unicode_age_14_0, "https://github.com/ruby/prism/issues/2664") -exclude(:test_unicode_age_15_0, "https://github.com/ruby/prism/issues/2664") -exclude(:test_unicode_age, "https://github.com/ruby/prism/issues/2664") diff --git a/test/.excludes-prism/TestRequire.rb b/test/.excludes-prism/TestRequire.rb deleted file mode 100644 index a7f66c5d80..0000000000 --- a/test/.excludes-prism/TestRequire.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_require_nonascii_path_shift_jis, "encoding") diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 926f0c5a5e..853f23a3b9 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,5 +1 @@ -exclude(:test_debug_frozen_string_in_array_literal, "unknown") -exclude(:test_debug_frozen_string, "unknown") -exclude(:test_dregexp, "https://github.com/ruby/prism/issues/2664") -exclude(:test_hash_value_omission, "unknown") -exclude(:test_string, "https://github.com/ruby/prism/issues/2331") +exclude(:test_dregexp, "https://bugs.ruby-lang.org/issues/20504") diff --git a/test/.excludes-prism/TestRubyOptimization.rb b/test/.excludes-prism/TestRubyOptimization.rb deleted file mode 100644 index df22ca4f71..0000000000 --- a/test/.excludes-prism/TestRubyOptimization.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_peephole_string_literal_range, "unknown") diff --git a/test/.excludes-prism/TestRubyVM.rb b/test/.excludes-prism/TestRubyVM.rb deleted file mode 100644 index 6d4c3ca6fe..0000000000 --- a/test/.excludes-prism/TestRubyVM.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_keep_script_lines, "unknown") diff --git a/test/.excludes-prism/TestSetTraceFunc.rb b/test/.excludes-prism/TestSetTraceFunc.rb deleted file mode 100644 index 036faef650..0000000000 --- a/test/.excludes-prism/TestSetTraceFunc.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_return, "unknown") diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 61051e2f37..ed4ae44318 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -1,29 +1,3 @@ -exclude(:test__END___cr, "unknown") -exclude(:test_anonymous_block_forwarding, "unknown") -exclude(:test_anonymous_keyword_rest_forwarding, "unknown") -exclude(:test_anonymous_rest_forwarding, "unknown") -exclude(:test_argument_forwarding_with_super, "unknown") -exclude(:test_argument_forwarding, "unknown") -exclude(:test_brace_after_literal_argument, "unknown") -exclude(:test_dedented_heredoc_concatenation, "unknown") -exclude(:test_dedented_heredoc_continued_line, "unknown") -exclude(:test_dedented_heredoc_invalid_identifer, "unknown") -exclude(:test_duplicated_when, "unknown") -exclude(:test_error_message_encoding, "unknown") -exclude(:test_heredoc_cr, "unknown") -exclude(:test_heredoc_no_terminator, "unknown") -exclude(:test_invalid_encoding_symbol, "unknown") -exclude(:test_it, "https://github.com/ruby/prism/issues/2323") -exclude(:test_keyword_invalid_name, "unknown") -exclude(:test_keyword_self_reference, "unknown") -exclude(:test_keywords_specified_and_not_accepted, "unknown") -exclude(:test_methoddef_endless_command, "unknown") -exclude(:test_numbered_parameter, "unknown") -exclude(:test_optional_self_reference, "unknown") -exclude(:test_safe_call_in_massign_lhs, "unknown") -exclude(:test_syntax_error_at_newline, "unknown") -exclude(:test_unexpected_fraction, "unknown") -exclude(:test_unterminated_heredoc_cr, "unknown") -exclude(:test_unterminated_heredoc, "unknown") -exclude(:test_warn_balanced, "unknown") -exclude(:test_warn_unreachable, "unknown") +exclude(:test_dedented_heredoc_continued_line, "https://bugs.ruby-lang.org/issues/20503") +exclude(:test_optional_self_reference, "https://bugs.ruby-lang.org/issues/20478") +exclude(:test_keyword_self_reference, "https://bugs.ruby-lang.org/issues/20478") diff --git a/test/.excludes-prism/TestUnicodeEscape.rb b/test/.excludes-prism/TestUnicodeEscape.rb deleted file mode 100644 index add4911bc2..0000000000 --- a/test/.excludes-prism/TestUnicodeEscape.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_fail, "unknown") diff --git a/test/error_highlight/test_error_highlight.rb b/test/error_highlight/test_error_highlight.rb index 2095970af1..69aef31339 100644 --- a/test/error_highlight/test_error_highlight.rb +++ b/test/error_highlight/test_error_highlight.rb @@ -1,6 +1,7 @@ require "test/unit" require "error_highlight" +require "did_you_mean" require "tempfile" class ErrorHighlightTest < Test::Unit::TestCase diff --git a/test/irb/command/test_custom_command.rb b/test/irb/command/test_custom_command.rb index 6642d2b160..3a3ad11d5a 100644 --- a/test/irb/command/test_custom_command.rb +++ b/test/irb/command/test_custom_command.rb @@ -5,7 +5,7 @@ require_relative "../helper" module TestIRB class CustomCommandIntegrationTest < TestIRB::IntegrationTestCase - def test_command_regsitration_can_happen_after_irb_require + def test_command_registration_can_happen_after_irb_require write_ruby <<~RUBY require "irb" require "irb/command" @@ -15,7 +15,6 @@ module TestIRB description 'print_command' def execute(*) puts "Hello from PrintCommand" - nil end end @@ -25,14 +24,14 @@ module TestIRB RUBY output = run_ruby_file do - type "print!\n" + type "print!" type "exit" end assert_include(output, "Hello from PrintCommand") end - def test_command_regsitration_accepts_string_too + def test_command_registration_accepts_string_too write_ruby <<~RUBY require "irb/command" @@ -41,7 +40,6 @@ module TestIRB description 'print_command' def execute(*) puts "Hello from PrintCommand" - nil end end @@ -51,14 +49,14 @@ module TestIRB RUBY output = run_ruby_file do - type "print!\n" + type "print!" type "exit" end assert_include(output, "Hello from PrintCommand") end - def test_arguments_propogation + def test_arguments_propagation write_ruby <<~RUBY require "irb/command" @@ -69,7 +67,6 @@ module TestIRB $nth_execution ||= 0 puts "\#{$nth_execution} arg=\#{arg.inspect}" $nth_execution += 1 - nil end end @@ -79,9 +76,9 @@ module TestIRB RUBY output = run_ruby_file do - type "print_arg\n" + type "print_arg" type "print_arg \n" - type "print_arg a r g\n" + type "print_arg a r g" type "print_arg a r g \n" type "exit" end @@ -103,7 +100,6 @@ module TestIRB $nth_execution ||= 1 puts "\#{$nth_execution} FooBar executed" $nth_execution += 1 - nil end end @@ -123,5 +119,31 @@ module TestIRB assert_include(output, "2 FooBar executed") assert_include(output, "foobar_description") end + + def test_no_meta_command_also_works + write_ruby <<~RUBY + require "irb/command" + + class NoMetaCommand < IRB::Command::Base + def execute(*) + puts "This command does not override meta attributes" + end + end + + IRB::Command.register(:no_meta, NoMetaCommand) + + binding.irb + RUBY + + output = run_ruby_file do + type "no_meta" + type "help no_meta" + type "exit" + end + + assert_include(output, "This command does not override meta attributes") + assert_include(output, "No description provided.") + assert_not_include(output, "Maybe IRB bug") + end end end diff --git a/test/irb/command/test_help.rb b/test/irb/command/test_help.rb index df3753dae7..b34832b022 100644 --- a/test/irb/command/test_help.rb +++ b/test/irb/command/test_help.rb @@ -69,7 +69,7 @@ module TestIRB type "exit" end - assert_match(/Helper methods\s+conf\s+Returns the current context/, out) + assert_match(/Helper methods\s+conf\s+Returns the current IRB context/, out) end end end diff --git a/test/irb/helper.rb b/test/irb/helper.rb index 1614b42adb..acaf6277f3 100644 --- a/test/irb/helper.rb +++ b/test/irb/helper.rb @@ -121,7 +121,9 @@ module TestIRB @envs["XDG_CONFIG_HOME"] ||= tmp_dir @envs["IRBRC"] = nil unless @envs.key?("IRBRC") - PTY.spawn(@envs.merge("TERM" => "dumb"), *cmd) do |read, write, pid| + envs_for_spawn = @envs.merge('TERM' => 'dumb', 'TEST_IRB_FORCE_INTERACTIVE' => 'true') + + PTY.spawn(envs_for_spawn, *cmd) do |read, write, pid| Timeout.timeout(TIMEOUT_SEC) do while line = safe_gets(read) lines << line @@ -196,7 +198,7 @@ module TestIRB end def write_ruby(program) - @ruby_file = Tempfile.create(%w{irb- .rb}) + @ruby_file = Tempfile.create(%w{irbtest- .rb}) @tmpfiles << @ruby_file @ruby_file.write(program) @ruby_file.close diff --git a/test/irb/test_debugger_integration.rb b/test/irb/test_debugger_integration.rb index eca40c5702..8b1bddea17 100644 --- a/test/irb/test_debugger_integration.rb +++ b/test/irb/test_debugger_integration.rb @@ -67,6 +67,22 @@ module TestIRB assert_match(/IRB is already running with a debug session/, output) end + def test_debug_command_can_only_be_called_from_binding_irb + write_ruby <<~'ruby' + require "irb" + # trick test framework + puts "binding.irb" + IRB.start + ruby + + output = run_ruby_file do + type "debug" + type "exit" + end + + assert_include(output, "Debugging commands are only available when IRB is started with binding.irb") + end + def test_next write_ruby <<~'ruby' binding.irb diff --git a/test/irb/test_init.rb b/test/irb/test_init.rb index f11d7398c8..3207c2898b 100644 --- a/test/irb/test_init.rb +++ b/test/irb/test_init.rb @@ -268,15 +268,95 @@ module TestIRB end end + class ConfigValidationTest < TestCase + def setup + @original_home = ENV["HOME"] + @original_irbrc = ENV["IRBRC"] + # To prevent the test from using the user's .irbrc file + ENV["HOME"] = @home = Dir.mktmpdir + IRB.instance_variable_set(:@existing_rc_name_generators, nil) + super + end + + def teardown + super + ENV["IRBRC"] = @original_irbrc + ENV["HOME"] = @original_home + File.unlink(@irbrc) + Dir.rmdir(@home) + end + + def test_irb_name_converts_non_string_values_to_string + assert_no_irb_validation_error(<<~'RUBY') + IRB.conf[:IRB_NAME] = :foo + RUBY + + assert_equal "foo", IRB.conf[:IRB_NAME] + end + + def test_irb_rc_name_only_takes_callable_objects + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:IRB_RC] should be a callable object. Got :foo.") + IRB.conf[:IRB_RC] = :foo + RUBY + end + + def test_back_trace_limit_only_accepts_integers + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got \"foo\".") + IRB.conf[:BACK_TRACE_LIMIT] = "foo" + RUBY + end + + def test_prompt_only_accepts_hash + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:PROMPT] should be a Hash. Got \"foo\".") + IRB.conf[:PROMPT] = "foo" + RUBY + end + + def test_eval_history_only_accepts_integers + assert_irb_validation_error(<<~'RUBY', "IRB.conf[:EVAL_HISTORY] should be an integer. Got \"foo\".") + IRB.conf[:EVAL_HISTORY] = "foo" + RUBY + end + + private + + def assert_irb_validation_error(rc_content, error_message) + write_rc rc_content + + assert_raise_with_message(TypeError, error_message) do + IRB.setup(__FILE__) + end + end + + def assert_no_irb_validation_error(rc_content) + write_rc rc_content + + assert_nothing_raised do + IRB.setup(__FILE__) + end + end + + def write_rc(content) + @irbrc = Tempfile.new('irbrc') + @irbrc.write(content) + @irbrc.close + ENV['IRBRC'] = @irbrc.path + end + end + class InitIntegrationTest < IntegrationTestCase - def test_load_error_in_rc_file_is_warned - write_rc <<~'IRBRC' - require "file_that_does_not_exist" - IRBRC + def setup + super write_ruby <<~'RUBY' binding.irb RUBY + end + + def test_load_error_in_rc_file_is_warned + write_rc <<~'IRBRC' + require "file_that_does_not_exist" + IRBRC output = run_ruby_file do type "'foobar'" @@ -293,10 +373,6 @@ module TestIRB raise "I'm an error" IRBRC - write_ruby <<~'RUBY' - binding.irb - RUBY - output = run_ruby_file do type "'foobar'" type "exit" diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index 84b9ee3644..28be744088 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -125,6 +125,26 @@ module TestIRB end end + class NestedBindingIrbTest < IntegrationTestCase + def test_current_context_restore + write_ruby <<~'RUBY' + binding.irb + RUBY + + output = run_ruby_file do + type '$ctx = IRB.CurrentContext' + type 'binding.irb' + type 'p context_changed: IRB.CurrentContext != $ctx' + type 'exit' + type 'p context_restored: IRB.CurrentContext == $ctx' + type 'exit' + end + + assert_include output, '{:context_changed=>true}' + assert_include output, '{:context_restored=>true}' + end + end + class IrbIOConfigurationTest < TestCase Row = Struct.new(:content, :current_line_spaces, :new_line_spaces, :indent_level) @@ -803,4 +823,95 @@ module TestIRB IRB::Irb.new(workspace, TestInputMethod.new) end end + + class BacktraceFilteringTest < TestIRB::IntegrationTestCase + def test_backtrace_filtering + write_ruby <<~'RUBY' + def foo + raise "error" + end + + def bar + foo + end + + binding.irb + RUBY + + output = run_ruby_file do + type "bar" + type "exit" + end + + assert_match(/irbtest-.*\.rb:2:in (`|'Object#)foo': error \(RuntimeError\)/, output) + frame_traces = output.split("\n").select { |line| line.strip.match?(/from /) }.map(&:strip) + + expected_traces = if RUBY_VERSION >= "3.3.0" + [ + /from .*\/irbtest-.*.rb:6:in (`|'Object#)bar'/, + /from .*\/irbtest-.*.rb\(irb\):1:in [`']<main>'/, + /from <internal:kernel>:\d+:in (`|'Kernel#)loop'/, + /from <internal:prelude>:\d+:in (`|'Binding#)irb'/, + /from .*\/irbtest-.*.rb:9:in [`']<main>'/ + ] + else + [ + /from .*\/irbtest-.*.rb:6:in (`|'Object#)bar'/, + /from .*\/irbtest-.*.rb\(irb\):1:in [`']<main>'/, + /from <internal:prelude>:\d+:in (`|'Binding#)irb'/, + /from .*\/irbtest-.*.rb:9:in [`']<main>'/ + ] + end + + expected_traces.reverse! if RUBY_VERSION < "3.0.0" + + expected_traces.each_with_index do |expected_trace, index| + assert_match(expected_trace, frame_traces[index]) + end + end + + def test_backtrace_filtering_with_backtrace_filter + write_rc <<~'RUBY' + class TestBacktraceFilter + def self.call(backtrace) + backtrace.reject { |line| line.include?("internal") } + end + end + + IRB.conf[:BACKTRACE_FILTER] = TestBacktraceFilter + RUBY + + write_ruby <<~'RUBY' + def foo + raise "error" + end + + def bar + foo + end + + binding.irb + RUBY + + output = run_ruby_file do + type "bar" + type "exit" + end + + assert_match(/irbtest-.*\.rb:2:in (`|'Object#)foo': error \(RuntimeError\)/, output) + frame_traces = output.split("\n").select { |line| line.strip.match?(/from /) }.map(&:strip) + + expected_traces = [ + /from .*\/irbtest-.*.rb:6:in (`|'Object#)bar'/, + /from .*\/irbtest-.*.rb\(irb\):1:in [`']<main>'/, + /from .*\/irbtest-.*.rb:9:in [`']<main>'/ + ] + + expected_traces.reverse! if RUBY_VERSION < "3.0.0" + + expected_traces.each_with_index do |expected_trace, index| + assert_match(expected_trace, frame_traces[index]) + end + end + end end diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index b798b897b4..17f73e4433 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -416,7 +416,7 @@ class TestObjSpace < Test::Unit::TestCase assert_equal('true', ObjectSpace.dump(true)) assert_equal('false', ObjectSpace.dump(false)) assert_equal('0', ObjectSpace.dump(0)) - assert_equal('{"type":"SYMBOL", "value":"foo"}', ObjectSpace.dump(:foo)) + assert_equal('{"type":"SYMBOL", "value":"test_dump_special_consts"}', ObjectSpace.dump(:test_dump_special_consts)) end def test_dump_singleton_class diff --git a/test/objspace/test_ractor.rb b/test/objspace/test_ractor.rb index b7008ea731..4901eeae2e 100644 --- a/test/objspace/test_ractor.rb +++ b/test/objspace/test_ractor.rb @@ -1,17 +1,17 @@ require "test/unit" class TestObjSpaceRactor < Test::Unit::TestCase - def test_tracing_does_not_crash - assert_ractor(<<~RUBY, require: 'objspace') - ObjectSpace.trace_object_allocations do - r = Ractor.new do - obj = 'a' * 1024 - Ractor.yield obj - end + def test_tracing_does_not_crash + assert_ractor(<<~RUBY, require: 'objspace') + ObjectSpace.trace_object_allocations do + r = Ractor.new do + obj = 'a' * 1024 + Ractor.yield obj + end - r.take - r.take - end - RUBY - end + r.take + r.take + end + RUBY + end end diff --git a/test/openssl/test_cipher.rb b/test/openssl/test_cipher.rb index 8faa570648..41885fd59b 100644 --- a/test/openssl/test_cipher.rb +++ b/test/openssl/test_cipher.rb @@ -331,6 +331,22 @@ class OpenSSL::TestCipher < OpenSSL::TestCase assert_equal tag1, tag2 end + def test_aes_keywrap_pad + # RFC 5649 Section 6; The second example + kek = ["5840df6e29b02af1ab493b705bf16ea1ae8338f4dcc176a8"].pack("H*") + key = ["466f7250617369"].pack("H*") + wrap = ["afbeb0f07dfbf5419200f2ccb50bb24f"].pack("H*") + + begin + cipher = OpenSSL::Cipher.new("id-aes192-wrap-pad").encrypt + rescue OpenSSL::Cipher::CipherError, RuntimeError + omit "id-aes192-wrap-pad is not supported: #$!" + end + cipher.key = kek + ct = cipher.update(key) << cipher.final + assert_equal wrap, ct + end + def test_non_aead_cipher_set_auth_data assert_raise(OpenSSL::Cipher::CipherError) { cipher = OpenSSL::Cipher.new("aes-128-cfb").encrypt diff --git a/test/openssl/test_digest.rb b/test/openssl/test_digest.rb index b0b5b9bed1..988330e405 100644 --- a/test/openssl/test_digest.rb +++ b/test/openssl/test_digest.rb @@ -88,7 +88,7 @@ class OpenSSL::TestDigest < OpenSSL::TestCase end def test_sha512_truncate - pend "SHA512_224 is not implemented" unless digest_available?('SHA512-224') + pend "SHA512_224 is not implemented" unless digest_available?('sha512-224') sha512_224_a = "d5cdb9ccc769a5121d4175f2bfdd13d6310e0d3d361ea75d82108327" sha512_256_a = "455e518824bc0601f9fb858ff5c37d417d67c2f8e0df2babe4808858aea830f8" @@ -100,7 +100,7 @@ class OpenSSL::TestDigest < OpenSSL::TestCase end def test_sha3 - pend "SHA3 is not implemented" unless digest_available?('SHA3-224') + pend "SHA3 is not implemented" unless digest_available?('sha3-224') s224 = '6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7' s256 = 'a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a' s384 = '0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004' @@ -126,6 +126,15 @@ class OpenSSL::TestDigest < OpenSSL::TestCase end end + def test_digests + digests = OpenSSL::Digest.digests + assert_kind_of Array, digests + assert_include digests, "md5" + assert_include digests, "sha1" + assert_include digests, "sha256" + assert_include digests, "sha512" + end + private def check_digest(oid) @@ -138,11 +147,8 @@ class OpenSSL::TestDigest < OpenSSL::TestCase end def digest_available?(name) - begin - OpenSSL::Digest.new(name) - rescue RuntimeError - false - end + @digests ||= OpenSSL::Digest.digests + @digests.include?(name) end end diff --git a/test/openssl/test_pair.rb b/test/openssl/test_pair.rb index b616883925..66e36a7ab4 100644 --- a/test/openssl/test_pair.rb +++ b/test/openssl/test_pair.rb @@ -250,12 +250,17 @@ module OpenSSL::TestPairM buf = +"garbage" assert_equal :wait_readable, s2.read_nonblock(100, buf, exception: false) - assert_equal "", buf + assert_equal "garbage", buf s1.close buf = +"garbage" - assert_equal nil, s2.read(100, buf) + assert_nil s2.read(100, buf) assert_equal "", buf + + buf = +"garbage" + ret = s2.read(0, buf) + assert_same buf, ret + assert_equal "", ret } end diff --git a/test/openssl/test_pkcs7.rb b/test/openssl/test_pkcs7.rb index ba8b93d034..c049ed444a 100644 --- a/test/openssl/test_pkcs7.rb +++ b/test/openssl/test_pkcs7.rb @@ -155,6 +155,21 @@ class OpenSSL::TestPKCS7 < OpenSSL::TestCase assert_equal(data, p7.decrypt(@rsa1024)) end + def test_empty_signed_data_ruby_bug_19974 + data = "-----BEGIN PKCS7-----\nMAsGCSqGSIb3DQEHAg==\n-----END PKCS7-----\n" + assert_raise(ArgumentError) { OpenSSL::PKCS7.new(data) } + + data = <<END +MIME-Version: 1.0 +Content-Disposition: attachment; filename="smime.p7m" +Content-Type: application/x-pkcs7-mime; smime-type=signed-data; name="smime.p7m" +Content-Transfer-Encoding: base64 + +#{data} +END + assert_raise(OpenSSL::PKCS7::PKCS7Error) { OpenSSL::PKCS7.read_smime(data) } + end + def test_graceful_parsing_failure #[ruby-core:43250] contents = File.read(__FILE__) assert_raise(ArgumentError) { OpenSSL::PKCS7.new(contents) } @@ -212,6 +227,12 @@ END assert_equal(p7.to_der, OpenSSL::PKCS7.read_smime(smime).to_der) end + def test_to_text + p7 = OpenSSL::PKCS7.new + p7.type = "signed" + assert_match(/signed/, p7.to_text) + end + def test_degenerate_pkcs7 ca_cert_pem = <<END -----BEGIN CERTIFICATE----- diff --git a/test/openssl/test_ssl.rb b/test/openssl/test_ssl.rb index 66d63a981d..1471b0cb36 100644 --- a/test/openssl/test_ssl.rb +++ b/test/openssl/test_ssl.rb @@ -117,6 +117,30 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase } end + def test_socket_close_write + server_proc = proc do |ctx, ssl| + message = ssl.read + ssl.write(message) + ssl.close_write + ensure + ssl.close + end + + start_server(server_proc: server_proc) do |port| + ctx = OpenSSL::SSL::SSLContext.new + ssl = OpenSSL::SSL::SSLSocket.open("127.0.0.1", port, context: ctx) + ssl.sync_close = true + ssl.connect + + message = "abc"*1024 + ssl.write message + ssl.close_write + assert_equal message, ssl.read + ensure + ssl&.close + end + end + def test_add_certificate ctx_proc = -> ctx { # Unset values set by start_server diff --git a/test/openssl/test_ts.rb b/test/openssl/test_ts.rb index 7cb1a1fe8e..ac0469ad56 100644 --- a/test/openssl/test_ts.rb +++ b/test/openssl/test_ts.rb @@ -323,6 +323,8 @@ _end_of_pem_ resp = fac.create_timestamp(ee_key, ts_cert_ee, req) assert_equal(OpenSSL::Timestamp::Response::GRANTED, resp.status) assert_equal("1.2.3.4.6", resp.token_info.policy_id) + + assert_match(/1\.2\.3\.4\.6/, resp.to_text) end def test_response_bad_purpose diff --git a/test/prism/comments_test.rb b/test/prism/comments_test.rb index b99c00268c..952d03239c 100644 --- a/test/prism/comments_test.rb +++ b/test/prism/comments_test.rb @@ -6,7 +6,7 @@ module Prism class CommentsTest < TestCase def test_comment_inline source = "# comment" - assert_equal [0], Debug.newlines(source) + assert_equal [0], Prism.parse(source).source.offsets assert_comment( source, diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index f989daa361..5f4acb0120 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -99,7 +99,7 @@ module Prism ) assert_errors expected, "BEGIN { 1 + }", [ - ["expected an expression after the operator", 10..11], + ["unexpected '}'; expected an expression after the operator", 12..13], ["unexpected '}', assuming it is closing the parent 'BEGIN' block", 12..13] ] end @@ -117,25 +117,25 @@ module Prism def test_unterminated_i_list assert_errors expression("%i["), "%i[", [ - ["expected a closing delimiter for the `%i` list", 0..3] + ["unterminated list; expected a closing delimiter for the `%i`", 0..3] ] end def test_unterminated_w_list assert_errors expression("%w["), "%w[", [ - ["expected a closing delimiter for the `%w` list", 0..3] + ["unterminated list; expected a closing delimiter for the `%w`", 0..3] ] end def test_unterminated_W_list assert_errors expression("%W["), "%W[", [ - ["expected a closing delimiter for the `%W` list", 0..3] + ["unterminated list; expected a closing delimiter for the `%W`", 0..3] ] end def test_unterminated_regular_expression assert_errors expression("/hello"), "/hello", [ - ["expected a closing delimiter for the regular expression", 0..1] + ["unterminated regexp meets end of file; expected a closing delimiter", 0..1] ] end @@ -143,7 +143,7 @@ module Prism source = "<<-END + /b\nEND\n" assert_errors expression(source), source, [ - ["expected a closing delimiter for the regular expression", 9..10] + ["unterminated regexp meets end of file; expected a closing delimiter", 9..10] ] end @@ -189,14 +189,13 @@ module Prism def test_unterminated_s_symbol assert_errors expression("%s[abc"), "%s[abc", [ - ["expected a closing delimiter for the dynamic symbol", 0..3] + ["unterminated quoted string; expected a closing delimiter for the dynamic symbol", 0..3] ] end def test_unterminated_parenthesized_expression assert_errors expression('(1 + 2'), '(1 + 2', [ - ["unexpected end of file, expecting end-of-input", 6..6], - ["unexpected end of file, assuming it is closing the parent top level context", 6..6], + ["unexpected end-of-input, assuming it is closing the parent top level context", 6..6], ["expected a matching `)`", 6..6] ] end @@ -209,21 +208,21 @@ module Prism def test_unterminated_argument_expression assert_errors expression('a %'), 'a %', [ - ["invalid `%` token", 2..3], - ["expected an expression after the operator", 2..3], - ["unexpected end of file, assuming it is closing the parent top level context", 3..3] + ["unterminated quoted string meets end of file", 2..3], + ["unexpected end-of-input; expected an expression after the operator", 3..3], + ["unexpected end-of-input, assuming it is closing the parent top level context", 3..3] ] end def test_unterminated_interpolated_symbol assert_error_messages ":\"#", [ - "expected a closing delimiter for the interpolated symbol" + "unterminated symbol; expected a closing delimiter for the interpolated symbol" ] end def test_cr_without_lf_in_percent_expression assert_errors expression("%\r"), "%\r", [ - ["invalid `%` token", 0..2], + ["unterminated string meets end of file", 2..2], ] end @@ -365,7 +364,7 @@ module Prism assert_error_messages "x.each { x end", [ "unexpected 'end', expecting end-of-input", "unexpected 'end', ignoring it", - "unexpected end of file, assuming it is closing the parent top level context", + "unexpected end-of-input, assuming it is closing the parent top level context", "expected a block beginning with `{` to end with `}`" ] end @@ -378,10 +377,13 @@ module Prism :a, Location(), Location(), - ArgumentsNode(1, [ - KeywordHashNode(0, [AssocSplatNode(expression("kwargs"), Location())]), - SplatNode(Location(), expression("args")) - ]), + ArgumentsNode( + ArgumentsNodeFlags::CONTAINS_KEYWORDS | ArgumentsNodeFlags::CONTAINS_KEYWORD_SPLAT, + [ + KeywordHashNode(0, [AssocSplatNode(expression("kwargs"), Location())]), + SplatNode(Location(), expression("args")) + ] + ), Location(), nil ) @@ -425,7 +427,7 @@ module Prism :a, Location(), Location(), - ArgumentsNode(0, [ + ArgumentsNode(ArgumentsNodeFlags::CONTAINS_KEYWORDS, [ KeywordHashNode(1, [ AssocNode( SymbolNode(SymbolFlags::FORCED_US_ASCII_ENCODING, nil, Location(), Location(), "foo"), @@ -690,13 +692,13 @@ module Prism expected = StringNode(StringFlags::FORCED_UTF8_ENCODING, Location(), Location(), nil, "\u0001\u0002") assert_errors expected, '?\u{0001 0002}', [ - ["invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", 9..12] + ["invalid Unicode escape sequence; Multiple codepoints at single character literal are disallowed", 9..12] ] end def test_invalid_hex_escape assert_errors expression('"\\xx"'), '"\\xx"', [ - ["invalid hexadecimal escape sequence", 1..3], + ["invalid hex escape sequence", 1..3], ] end @@ -713,12 +715,13 @@ module Prism assert_errors expected, '"\u{000z}"', [ ["invalid Unicode escape sequence", 7..7], + ["unterminated Unicode escape", 7..7] ] end def test_unterminated_unicode_brackets_should_be_a_syntax_error assert_errors expression('?\\u{3'), '?\\u{3', [ - ["invalid Unicode escape sequence; needs closing `}`", 1..5], + ["unterminated Unicode escape", 1..5], ] end @@ -861,7 +864,7 @@ module Prism :foo, Location(), nil, - ParametersNode([], [], nil, [], [], ForwardingParameterNode(), nil), + ParametersNode([], [], nil, [ForwardingParameterNode()], [], ForwardingParameterNode(), nil), nil, [], Location(), @@ -1238,7 +1241,7 @@ module Prism expected = CallNode(0, receiver, Location(), :foo, Location(), nil, nil, nil, nil) assert_errors expected, "<<~FOO.foo\n", [ - ["could not find a terminator for the heredoc", 11..11] + ["unterminated heredoc; can't find string \"FOO\" anywhere before EOF", 3..6] ] end @@ -1353,14 +1356,14 @@ module Prism if RUBY_VERSION >= "3.0" def test_writing_numbered_parameter - assert_errors expression("-> { _1 = 0 }"), "-> { _1 = 0 }", [ - ["_1 is reserved for numbered parameters", 5..7] + assert_error_messages "-> { _1 = 0 }", [ + "_1 is reserved for numbered parameters" ] end def test_targeting_numbered_parameter - assert_errors expression("-> { _1, = 0 }"), "-> { _1, = 0 }", [ - ["_1 is reserved for numbered parameters", 5..7] + assert_error_messages "-> { _1, = 0 }", [ + "_1 is reserved for numbered parameters" ] end @@ -1374,14 +1377,13 @@ module Prism def test_double_scope_numbered_parameters source = "-> { _1 + -> { _2 } }" - errors = [["numbered parameter is already used in outer scope", 15..17]] + errors = [["numbered parameter is already used in outer block", 15..17]] assert_errors expression(source), source, errors end def test_invalid_number_underscores error_messages = ["invalid underscore placement in number"] - assert_error_messages "1__1", error_messages assert_error_messages "0b1__1", error_messages assert_error_messages "0o1__1", error_messages @@ -1389,6 +1391,7 @@ module Prism assert_error_messages "0d1__1", error_messages assert_error_messages "0x1__1", error_messages + error_messages = ["trailing '_' in number"] assert_error_messages "1_1_", error_messages assert_error_messages "0b1_1_", error_messages assert_error_messages "0o1_1_", error_messages @@ -1398,7 +1401,7 @@ module Prism end def test_alnum_delimiters - error_messages = ["invalid `%` token"] + error_messages = ["unknown type of %string"] assert_error_messages "%qXfooX", error_messages assert_error_messages "%QXfooX", error_messages @@ -1463,7 +1466,7 @@ module Prism def test_forwarding_arg_after_keyword_rest source = "def f(**,...);end" assert_errors expression(source), source, [ - ["unexpected `...` in parameters", 9..12], + ["unexpected parameter order", 9..12] ] end @@ -1479,8 +1482,7 @@ module Prism assert_errors expression(source), source, [ ["expected a `do` keyword or a `{` to open the lambda block", 3..3], - ["unexpected end of file, expecting end-of-input", 7..7], - ["unexpected end of file, assuming it is closing the parent top level context", 7..7], + ["unexpected end-of-input, assuming it is closing the parent top level context", 7..7], ["expected a lambda block beginning with `do` to end with `end`", 7..7] ] end @@ -1538,7 +1540,7 @@ module Prism assert_errors expression(source), source, [ ["expected a predicate expression for the `while` statement", 22..22], - ["unexpected end of file, assuming it is closing the parent top level context", 22..22], + ["unexpected end-of-input, assuming it is closing the parent top level context", 22..22], ["expected an `end` to close the `while` statement", 22..22] ] end @@ -1628,6 +1630,41 @@ module Prism ] end + def test_void_value_expression_in_begin_statement + source = <<~RUBY + x = return 1 + x = return, 1 + x = 1, return + x, y = return + x = begin return ensure end + x = begin ensure return end + x = begin return ensure return end + x = begin return; rescue; return end + x = begin return; rescue; return; else return end + x = begin; return; rescue; retry; end + RUBY + + message = 'unexpected void value expression' + assert_errors expression(source), source, [ + [message, 4..12], + [message, 17..23], + [message, 34..40], + [message, 48..54], + [message, 65..71], + [message, 100..106], + [message, 121..127], + [message, 156..162], + [message, 222..228], + [message, 244..250], + ] + + refute_error_messages("x = begin return; rescue; end") + refute_error_messages("x = begin return; rescue; return; else end") + refute_error_messages("x = begin; rescue; retry; end") + refute_error_messages("x = begin 1; rescue; retry; ensure; end") + refute_error_messages("x = begin 1; rescue; return; end") + end + def test_void_value_expression_in_def source = <<~RUBY def (return).x @@ -1939,10 +1976,10 @@ module Prism RUBY assert_errors expression(source), source, [ - ["unexpected '..', expecting end-of-input", 3..5], - ["unexpected '..', ignoring it", 3..5], - ["unexpected '..', expecting end-of-input", 10..12], - ["unexpected '..', ignoring it", 10..12] + ["unexpected .., expecting end-of-input", 3..5], + ["unexpected .., ignoring it", 3..5], + ["unexpected .., expecting end-of-input", 10..12], + ["unexpected .., ignoring it", 10..12] ] end @@ -2079,7 +2116,7 @@ module Prism def test_forwarding_arg_and_block source = 'def foo(...) = foo(...) { }' assert_errors expression(source), source, [ - ['both a block argument and a forwarding argument; only one block is allowed', 24..27] + ['both block arg and actual block given; only one block is allowed', 24..27] ] end diff --git a/test/prism/format_errors_test.rb b/test/prism/format_errors_test.rb deleted file mode 100644 index a1edbef2e8..0000000000 --- a/test/prism/format_errors_test.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -return if Prism::BACKEND == :FFI - -module Prism - class FormatErrorsTest < TestCase - def test_format_errors - assert_equal <<~ERROR, Debug.format_errors("<>", false) - > 1 | <> - | ^ unexpected '<', ignoring it - | ^ unexpected '>', ignoring it - ERROR - - assert_equal <<~'ERROR', Debug.format_errors('"%W"\u"', false) - > 1 | "%W"\u" - | ^ unexpected backslash, ignoring it - | ^ unexpected local variable or method, expecting end-of-input - | ^ unterminated string meets end of file - ERROR - end - end -end diff --git a/test/prism/index_write_test.rb b/test/prism/index_write_test.rb index 1c6f7bce89..cf90eb082f 100644 --- a/test/prism/index_write_test.rb +++ b/test/prism/index_write_test.rb @@ -4,7 +4,7 @@ require_relative "test_helper" module Prism class IndexWriteTest < TestCase - def test_keywords_3_3_0 + def test_keywords_3_3 assert_parse_success(<<~RUBY, "3.3.0") foo[bar: 1] = 1 foo[bar: 1] &&= 1 @@ -22,7 +22,7 @@ module Prism RUBY end - def test_block_3_3_0 + def test_block_3_3 assert_parse_success(<<~RUBY, "3.3.0") foo[&bar] = 1 foo[&bar] &&= 1 @@ -40,41 +40,41 @@ module Prism RUBY end - # def test_keywords_latest - # assert_parse_failure(<<~RUBY) - # foo[bar: 1] = 1 - # foo[bar: 1] &&= 1 - # foo[bar: 1] ||= 1 - # foo[bar: 1] += 1 - # RUBY + def test_keywords_latest + assert_parse_failure(<<~RUBY) + foo[bar: 1] = 1 + foo[bar: 1] &&= 1 + foo[bar: 1] ||= 1 + foo[bar: 1] += 1 + RUBY - # assert_parse_failure(<<~RUBY) - # def foo(**) - # bar[**] = 1 - # bar[**] &&= 1 - # bar[**] ||= 1 - # bar[**] += 1 - # end - # RUBY - # end + assert_parse_failure(<<~RUBY) + def foo(**) + bar[**] = 1 + bar[**] &&= 1 + bar[**] ||= 1 + bar[**] += 1 + end + RUBY + end - # def test_block_latest - # assert_parse_failure(<<~RUBY) - # foo[&bar] = 1 - # foo[&bar] &&= 1 - # foo[&bar] ||= 1 - # foo[&bar] += 1 - # RUBY + def test_block_latest + assert_parse_failure(<<~RUBY) + foo[&bar] = 1 + foo[&bar] &&= 1 + foo[&bar] ||= 1 + foo[&bar] += 1 + RUBY - # assert_parse_failure(<<~RUBY) - # def foo(&) - # bar[&] = 1 - # bar[&] &&= 1 - # bar[&] ||= 1 - # bar[&] += 1 - # end - # RUBY - # end + assert_parse_failure(<<~RUBY) + def foo(&) + bar[&] = 1 + bar[&] &&= 1 + bar[&] ||= 1 + bar[&] += 1 + end + RUBY + end private diff --git a/test/prism/integer_parse_test.rb b/test/prism/integer_parse_test.rb index f42e817e79..11aee174c5 100644 --- a/test/prism/integer_parse_test.rb +++ b/test/prism/integer_parse_test.rb @@ -2,8 +2,6 @@ require_relative "test_helper" -return if Prism::BACKEND == :FFI - module Prism class IntegerParseTest < TestCase def test_integer_parse @@ -37,9 +35,7 @@ module Prism private def assert_integer_parse(expected, source = expected.to_s) - integer, string = Debug.integer_parse(source) - assert_equal expected, integer - assert_equal expected.to_s, string + assert_equal expected, Prism.parse(source).value.statements.body.first.value end end end diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb index 0eb73f1b9c..0e57a9a80c 100644 --- a/test/prism/locals_test.rb +++ b/test/prism/locals_test.rb @@ -41,8 +41,8 @@ module Prism def assert_locals(filepath) source = File.read(filepath) - expected = Debug.cruby_locals(source) - actual = Debug.prism_locals(source) + expected = cruby_locals(source) + actual = prism_locals(source) assert_equal(expected, actual) end @@ -54,5 +54,186 @@ module Prism ensure $VERBOSE = previous_verbosity end + + # A wrapper around a RubyVM::InstructionSequence that provides a more + # convenient interface for accessing parts of the iseq. + class ISeq + attr_reader :parts + + def initialize(parts) + @parts = parts + end + + def type + parts[0] + end + + def local_table + parts[10] + end + + def instructions + parts[13] + end + + def each_child + instructions.each do |instruction| + # Only look at arrays. Other instructions are line numbers or + # tracepoint events. + next unless instruction.is_a?(Array) + + instruction.each do |opnd| + # Only look at arrays. Other operands are literals. + next unless opnd.is_a?(Array) + + # Only look at instruction sequences. Other operands are literals. + next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat" + + yield ISeq.new(opnd) + end + end + end + end + + # Used to hold the place of a local that will be in the local table but + # cannot be accessed directly from the source code. For example, the + # iteration variable in a for loop or the positional parameter on a method + # definition that is destructured. + AnonymousLocal = Object.new + + # For the given source, compiles with CRuby and returns a list of all of the + # sets of local variables that were encountered. + def cruby_locals(source) + verbose, $VERBOSE = $VERBOSE, nil + + begin + locals = [] #: Array[Array[Symbol | Integer]] + stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)] + + while (iseq = stack.pop) + names = [*iseq.local_table] + names.map!.with_index do |name, index| + # When an anonymous local variable is present in the iseq's local + # table, it is represented as the stack offset from the top. + # However, when these are dumped to binary and read back in, they + # are replaced with the symbol :#arg_rest. To consistently handle + # this, we replace them here with their index. + if name == :"#arg_rest" + names.length - index + 1 + else + name + end + end + + locals << names + iseq.each_child { |child| stack << child } + end + + locals + ensure + $VERBOSE = verbose + end + end + + # For the given source, parses with prism and returns a list of all of the + # sets of local variables that were encountered. + def prism_locals(source) + locals = [] #: Array[Array[Symbol | Integer]] + stack = [Prism.parse(source).value] #: Array[Prism::node] + + while (node = stack.pop) + case node + when BlockNode, DefNode, LambdaNode + names = node.locals + params = + if node.is_a?(DefNode) + node.parameters + elsif node.parameters.is_a?(NumberedParametersNode) + nil + else + node.parameters&.parameters + end + + # prism places parameters in the same order that they appear in the + # source. CRuby places them in the order that they need to appear + # according to their own internal calling convention. We mimic that + # order here so that we can compare properly. + if params + sorted = [ + *params.requireds.map do |required| + if required.is_a?(RequiredParameterNode) + required.name + else + AnonymousLocal + end + end, + *params.optionals.map(&:name), + *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)), + *params.posts.map do |post| + if post.is_a?(RequiredParameterNode) + post.name + else + AnonymousLocal + end + end, + *params.keywords.grep(RequiredKeywordParameterNode).map(&:name), + *params.keywords.grep(OptionalKeywordParameterNode).map(&:name), + ] + + sorted << AnonymousLocal if params.keywords.any? + + if params.keyword_rest.is_a?(ForwardingParameterNode) + sorted.push(:*, :**, :&, :"...") + elsif params.keyword_rest.is_a?(KeywordRestParameterNode) + sorted << (params.keyword_rest.name || :**) + end + + # Recurse down the parameter tree to find any destructured + # parameters and add them after the other parameters. + param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse + while (param = param_stack.pop) + case param + when MultiTargetNode + param_stack.concat(param.rights.reverse) + param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name) + param_stack.concat(param.lefts.reverse) + when RequiredParameterNode + sorted << param.name + when SplatNode + sorted << param.expression.name + end + end + + if params.block + sorted << (params.block.name || :&) + end + + names = sorted.concat(names - sorted) + end + + names.map!.with_index do |name, index| + if name == AnonymousLocal + names.length - index + 1 + else + name + end + end + + locals << names + when ClassNode, ModuleNode, ProgramNode, SingletonClassNode + locals << node.locals + when ForNode + locals << [2] + when PostExecutionNode + locals.push([], []) + when InterpolatedRegularExpressionNode + locals << [] if node.once? + end + + stack.concat(node.compact_child_nodes) + end + + locals + end end end diff --git a/test/prism/location_test.rb b/test/prism/location_test.rb index 81417fbcb3..256e5b41e4 100644 --- a/test/prism/location_test.rb +++ b/test/prism/location_test.rb @@ -175,14 +175,6 @@ module Prism assert_location(CallNode, "foo bar baz") assert_location(CallNode, "foo bar('baz')") - - assert_location(CallNode, "-> { it }", 5...7, version: "3.3.0") do |node| - node.body.body.first - end - - assert_location(LocalVariableReadNode, "-> { it }", 5...7, version: "3.4.0") do |node| - node.body.body.first - end end def test_CallAndWriteNode @@ -298,7 +290,6 @@ module Prism def test_ConstantReadNode assert_location(ConstantReadNode, "Foo") - assert_location(ConstantReadNode, "Foo::Bar", 5...8, &:child) end def test_ConstantTargetNode @@ -478,7 +469,7 @@ module Prism end def test_IndexTargetNode - assert_location(IndexTargetNode, "foo[bar, &baz], = qux", 0...14) do |node| + assert_location(IndexTargetNode, "foo[bar], = qux", 0...8) do |node| node.lefts.first end end @@ -544,6 +535,24 @@ module Prism assert_location(InterpolatedXStringNode, '`foo #{bar} baz`') end + def test_ItLocalVariableReadNode + assert_location(ItLocalVariableReadNode, "-> { it }", 5...7) do |node| + node.body.body.first + end + + assert_location(ItLocalVariableReadNode, "foo { it }", 6...8) do |node| + node.block.body.body.first + end + + assert_location(CallNode, "-> { it }", 5...7, version: "3.3.0") do |node| + node.body.body.first + end + + assert_location(ItLocalVariableReadNode, "-> { it }", 5...7, version: "3.4.0") do |node| + node.body.body.first + end + end + def test_ItParametersNode assert_location(ItParametersNode, "-> { it }", &:parameters) end @@ -584,12 +593,6 @@ module Prism def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) - assert_location(LocalVariableReadNode, "-> { it }", 5...7) do |node| - node.body.body.first - end - assert_location(LocalVariableReadNode, "foo { it }", 6...8) do |node| - node.block.body.body.first - end end def test_LocalVariableTargetNode diff --git a/test/prism/memsize_test.rb b/test/prism/memsize_test.rb deleted file mode 100644 index d7e1448dbc..0000000000 --- a/test/prism/memsize_test.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -return if Prism::BACKEND == :FFI - -module Prism - class MemsizeTest < TestCase - def test_memsize - result = Debug.memsize("2 + 3") - - assert_equal 5, result[:length] - assert_kind_of Integer, result[:memsize] - assert_equal 6, result[:node_count] - end - end -end diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index e9975b346e..75593d34bf 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -7,7 +7,16 @@ return unless defined?(RubyVM::InstructionSequence) module Prism class NewlineTest < TestCase base = File.expand_path("../", __FILE__) - filepaths = Dir["*.rb", base: base] - %w[encoding_test.rb errors_test.rb parser_test.rb static_literals_test.rb unescape_test.rb] + filepaths = Dir["*.rb", base: base] - %w[ + encoding_test.rb + errors_test.rb + locals_test.rb + parser_test.rb + regexp_test.rb + static_literals_test.rb + unescape_test.rb + warnings_test.rb + ] filepaths.each do |relative| define_method("test_newline_flags_#{relative}") do diff --git a/test/prism/parse_test.rb b/test/prism/parse_test.rb index afb53e0668..5c66caebb9 100644 --- a/test/prism/parse_test.rb +++ b/test/prism/parse_test.rb @@ -265,7 +265,7 @@ module Prism # Next, assert that the newlines are in the expected places. expected_newlines = [0] source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } - assert_equal expected_newlines, Debug.newlines(source) + assert_equal expected_newlines, Prism.parse(source).source.offsets if ripper_should_match # Finally, assert that we can lex the source and get the same tokens as diff --git a/test/prism/regexp_test.rb b/test/prism/regexp_test.rb index 0a5fc2b4fc..35be217f79 100644 --- a/test/prism/regexp_test.rb +++ b/test/prism/regexp_test.rb @@ -2,78 +2,84 @@ require_relative "test_helper" -return if Prism::BACKEND == :FFI - module Prism class RegexpTest < TestCase - ############################################################################## + ############################################################################ # These tests test the actual use case of extracting named capture groups - ############################################################################## + ############################################################################ def test_named_captures_with_arrows - assert_equal(["foo"], named_captures("(?<foo>bar)")) + assert_equal([:foo], named_captures("(?<foo>bar)")) end def test_named_captures_with_single_quotes - assert_equal(["foo"], named_captures("(?'foo'bar)")) + assert_equal([:foo], named_captures("(?'foo'bar)")) end def test_nested_named_captures_with_arrows - assert_equal(["foo", "bar"], named_captures("(?<foo>(?<bar>baz))")) + assert_equal([:foo, :bar], named_captures("(?<foo>(?<bar>baz))")) end def test_nested_named_captures_with_single_quotes - assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) + assert_equal([:foo, :bar], named_captures("(?'foo'(?'bar'baz))")) end def test_allows_duplicate_named_captures - assert_equal(["foo", "foo"], named_captures("(?<foo>bar)(?<foo>baz)")) + assert_equal([:foo], named_captures("(?<foo>bar)(?<foo>baz)")) end def test_named_capture_inside_fake_range_quantifier - assert_equal(["foo"], named_captures("foo{1, (?<foo>2)}")) + assert_equal([:foo], named_captures("foo{1, (?<foo>2)}")) + end + + def test_fake_named_captures_inside_character_sets + assert_equal([], named_captures("[a-z(?<foo>)]")) end - ############################################################################## + def test_fake_named_capture_inside_character_set_with_escaped_ending + assert_equal([], named_captures("[a-z\\](?<foo>)]")) + end + + ############################################################################ # These tests test the rest of the AST. They are not exhaustive, but they # should cover the most common cases. We test these to make sure we don't # accidentally regress and stop being able to extract named captures. - ############################################################################## + ############################################################################ def test_alternation - refute_nil(named_captures("foo|bar")) + assert_valid_regexp("foo|bar") end def test_anchors - refute_nil(named_captures("^foo$")) + assert_valid_regexp("^foo$") end def test_any - refute_nil(named_captures(".")) + assert_valid_regexp(".") end def test_posix_character_classes - refute_nil(named_captures("[[:digit:]]")) + assert_valid_regexp("[[:digit:]]") end def test_negated_posix_character_classes - refute_nil(named_captures("[[:^digit:]]")) + assert_valid_regexp("[[:^digit:]]") end def test_invalid_posix_character_classes_should_fall_back_to_regular_classes - refute_nil(named_captures("[[:foo]]")) + assert_valid_regexp("[[:foo]]") end def test_character_sets - refute_nil(named_captures("[abc]")) + assert_valid_regexp("[abc]") end def test_nested_character_sets - refute_nil(named_captures("[[abc]]")) + assert_valid_regexp("[[abc]]") end def test_nested_character_sets_with_operators - refute_nil(named_captures("[[abc] && [def]]")) + assert_valid_regexp("[[abc] && [def]]") end def test_named_capture_inside_nested_character_set @@ -81,120 +87,108 @@ module Prism end def test_negated_character_sets - refute_nil(named_captures("[^abc]")) + assert_valid_regexp("[^abc]") end def test_character_ranges - refute_nil(named_captures("[a-z]")) + assert_valid_regexp("[a-z]") end def test_negated_character_ranges - refute_nil(named_captures("[^a-z]")) - end - - def test_fake_named_captures_inside_character_sets - assert_equal([], named_captures("[a-z(?<foo>)]")) - end - - def test_fake_named_capture_inside_character_set_with_escaped_ending - assert_equal([], named_captures("[a-z\\](?<foo>)]")) + assert_valid_regexp("[^a-z]") end def test_comments - refute_nil(named_captures("(?#foo)")) + assert_valid_regexp("(?#foo)") end def test_comments_with_escaped_parentheses - refute_nil(named_captures("(?#foo\\)\\))")) + assert_valid_regexp("(?#foo\\)\\))") end def test_non_capturing_groups - refute_nil(named_captures("(?:foo)")) + assert_valid_regexp("(?:foo)") end def test_positive_lookaheads - refute_nil(named_captures("(?=foo)")) + assert_valid_regexp("(?=foo)") end def test_negative_lookaheads - refute_nil(named_captures("(?!foo)")) + assert_valid_regexp("(?!foo)") end def test_positive_lookbehinds - refute_nil(named_captures("(?<=foo)")) + assert_valid_regexp("(?<=foo)") end def test_negative_lookbehinds - refute_nil(named_captures("(?<!foo)")) + assert_valid_regexp("(?<!foo)") end def test_atomic_groups - refute_nil(named_captures("(?>foo)")) + assert_valid_regexp("(?>foo)") end def test_absence_operator - refute_nil(named_captures("(?~foo)")) + assert_valid_regexp("(?~foo)") end def test_conditional_expression_with_index - refute_nil(named_captures("(?(1)foo)")) + assert_valid_regexp("(?(1)foo)") end def test_conditional_expression_with_name - refute_nil(named_captures("(?(foo)bar)")) + assert_valid_regexp("(?(foo)bar)") end def test_conditional_expression_with_group - refute_nil(named_captures("(?(<foo>)bar)")) + assert_valid_regexp("(?(<foo>)bar)") end def test_options_on_groups - refute_nil(named_captures("(?imxdau:foo)")) - end - - def test_options_on_groups_with_invalid_options - assert_nil(named_captures("(?z:bar)")) + assert_valid_regexp("(?imxdau:foo)") end def test_options_on_groups_getting_turned_off - refute_nil(named_captures("(?-imx:foo)")) + assert_valid_regexp("(?-imx:foo)") end def test_options_on_groups_some_getting_turned_on_some_getting_turned_off - refute_nil(named_captures("(?im-x:foo)")) + assert_valid_regexp("(?im-x:foo)") end def test_star_quantifier - refute_nil(named_captures("foo*")) + assert_valid_regexp("foo*") end def test_plus_quantifier - refute_nil(named_captures("foo+")) + assert_valid_regexp("foo+") end def test_question_mark_quantifier - refute_nil(named_captures("foo?")) + assert_valid_regexp("foo?") end def test_endless_range_quantifier - refute_nil(named_captures("foo{1,}")) + assert_valid_regexp("foo{1,}") end def test_beginless_range_quantifier - refute_nil(named_captures("foo{,1}")) + assert_valid_regexp("foo{,1}") end def test_range_quantifier - refute_nil(named_captures("foo{1,2}")) + assert_valid_regexp("foo{1,2}") end def test_fake_range_quantifier_because_of_spaces - refute_nil(named_captures("foo{1, 2}")) + assert_valid_regexp("foo{1, 2}") end - ############################################################################## + ############################################################################ # These test that flag values are correct. - ############################################################################## + ############################################################################ def test_flag_ignorecase assert_equal(Regexp::IGNORECASE, options("i")) @@ -241,8 +235,12 @@ module Prism private + def assert_valid_regexp(source) + assert Prism.parse_success?("/#{source}/ =~ \"\"") + end + def named_captures(source) - Debug.named_captures(source) + Prism.parse("/#{source}/ =~ \"\"").value.locals end def options(flags) diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb index 9e408d1edd..a1e2592d3d 100644 --- a/test/prism/ruby_api_test.rb +++ b/test/prism/ruby_api_test.rb @@ -266,6 +266,38 @@ module Prism refute_operator parse_expression(complex_source_1), :===, parse_expression(complex_source_2) end + def test_node_tunnel + program = Prism.parse("foo(1) +\n bar(2, 3) +\n baz(3, 4, 5)").value + + tunnel = program.tunnel(1, 4).last + assert_kind_of IntegerNode, tunnel + assert_equal 1, tunnel.value + + tunnel = program.tunnel(2, 6).last + assert_kind_of IntegerNode, tunnel + assert_equal 2, tunnel.value + + tunnel = program.tunnel(3, 9).last + assert_kind_of IntegerNode, tunnel + assert_equal 4, tunnel.value + + tunnel = program.tunnel(3, 8) + assert_equal [ProgramNode, StatementsNode, CallNode, ArgumentsNode, CallNode, ArgumentsNode], tunnel.map(&:class) + end + + def test_location_adjoin + program = Prism.parse("foo.bar = 1").value + + location = program.statements.body.first.message_loc + adjoined = location.adjoin("=") + + assert_kind_of Location, adjoined + refute_equal location, adjoined + + assert_equal 4, adjoined.start_offset + assert_equal 9, adjoined.end_offset + end + private def parse_expression(source) diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index 8edeac4b4f..0fd96d42b5 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -28,28 +28,21 @@ module Prism base = File.join(__dir__, "fixtures") todos = %w[ - heredocs_nested.txt newline_terminated.txt regex_char_width.txt seattlerb/bug169.txt - seattlerb/dstr_evstr.txt - seattlerb/heredoc_squiggly_interp.txt seattlerb/masgn_colon3.txt seattlerb/messy_op_asgn_lineno.txt seattlerb/op_asgn_primary_colon_const_command_call.txt - seattlerb/parse_line_evstr_after_break.txt seattlerb/regexp_esc_C_slash.txt seattlerb/str_lit_concat_bad_encodings.txt - seattlerb/str_pct_nested_nested.txt unescaping.txt unparser/corpus/literal/kwbegin.txt unparser/corpus/literal/send.txt - unparser/corpus/semantic/dstr.txt whitequark/masgn_const.txt whitequark/ruby_bug_12402.txt whitequark/ruby_bug_14690.txt whitequark/space_args_block.txt - whitequark/string_concat.txt ] # https://github.com/seattlerb/ruby_parser/issues/344 @@ -76,12 +69,10 @@ module Prism tilde_heredocs.txt unparser/corpus/literal/literal.txt while.txt - whitequark/class_definition_in_while_cond.txt whitequark/cond_eflipflop.txt whitequark/cond_iflipflop.txt whitequark/cond_match_current_line.txt whitequark/dedenting_heredoc.txt - whitequark/if_while_after_class__since_32.txt whitequark/lvar_injecting_match.txt whitequark/not.txt whitequark/numparam_ruby_bug_19025.txt diff --git a/test/prism/snapshots/arrays.txt b/test/prism/snapshots/arrays.txt index e8e53aacb9..90a4d8f3bb 100644 --- a/test/prism/snapshots/arrays.txt +++ b/test/prism/snapshots/arrays.txt @@ -960,8 +960,8 @@ │ ├── arguments: ∅ │ ├── closing_loc: (84,4)-(84,5) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (84,6)-(84,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (84,6)-(84,8) = "+=" │ └── value: │ @ IntegerNode (location: (84,9)-(84,10)) │ ├── flags: decimal @@ -1040,8 +1040,8 @@ │ ├── arguments: ∅ │ ├── closing_loc: (90,8)-(90,9) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (90,10)-(90,12) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (90,10)-(90,12) = "+=" │ └── value: │ @ IntegerNode (location: (90,13)-(90,14)) │ ├── flags: decimal @@ -1143,8 +1143,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (96,7)-(96,8) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (96,9)-(96,11) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (96,9)-(96,11) = "+=" │ └── value: │ @ IntegerNode (location: (96,12)-(96,13)) │ ├── flags: decimal @@ -1262,8 +1262,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (102,11)-(102,12) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (102,13)-(102,15) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (102,13)-(102,15) = "+=" │ └── value: │ @ IntegerNode (location: (102,16)-(102,17)) │ ├── flags: decimal @@ -1633,8 +1633,8 @@ │ │ │ └── expression: ∅ │ │ ├── closing_loc: (116,13)-(116,14) = "]" │ │ ├── block: ∅ - │ │ ├── operator: :+ - │ │ ├── operator_loc: (116,15)-(116,17) = "+=" + │ │ ├── binary_operator: :+ + │ │ ├── binary_operator_loc: (116,15)-(116,17) = "+=" │ │ └── value: │ │ @ IntegerNode (location: (116,18)-(116,19)) │ │ ├── flags: decimal diff --git a/test/prism/snapshots/blocks.txt b/test/prism/snapshots/blocks.txt index 0b1ec52e38..1c996ebd09 100644 --- a/test/prism/snapshots/blocks.txt +++ b/test/prism/snapshots/blocks.txt @@ -158,13 +158,13 @@ │ │ └── body: (length: 1) │ │ └── @ LocalVariableOperatorWriteNode (location: (7,24)-(7,33)) │ │ ├── name_loc: (7,24)-(7,28) = "memo" - │ │ ├── operator_loc: (7,29)-(7,31) = "+=" + │ │ ├── binary_operator_loc: (7,29)-(7,31) = "+=" │ │ ├── value: │ │ │ @ LocalVariableReadNode (location: (7,32)-(7,33)) │ │ │ ├── name: :x │ │ │ └── depth: 0 │ │ ├── name: :memo - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ ├── opening_loc: (7,12)-(7,13) = "{" │ └── closing_loc: (7,34)-(7,35) = "}" diff --git a/test/prism/snapshots/boolean_operators.txt b/test/prism/snapshots/boolean_operators.txt index ace8047e18..3bf33430c9 100644 --- a/test/prism/snapshots/boolean_operators.txt +++ b/test/prism/snapshots/boolean_operators.txt @@ -21,7 +21,7 @@ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,6)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ CallNode (location: (3,5)-(3,6)) │ │ ├── flags: variable_call, ignore_visibility @@ -34,7 +34,7 @@ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ LocalVariableOrWriteNode (location: (5,0)-(5,7)) ├── name_loc: (5,0)-(5,1) = "a" diff --git a/test/prism/snapshots/constants.txt b/test/prism/snapshots/constants.txt index 59e234148a..1251833663 100644 --- a/test/prism/snapshots/constants.txt +++ b/test/prism/snapshots/constants.txt @@ -7,24 +7,21 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (1,0)-(1,1)) │ │ └── name: :A - │ ├── child: - │ │ @ ConstantReadNode (location: (1,3)-(1,4)) - │ │ └── name: :B - │ └── delimiter_loc: (1,1)-(1,3) = "::" + │ ├── name: :B + │ ├── delimiter_loc: (1,1)-(1,3) = "::" + │ └── name_loc: (1,3)-(1,4) = "B" ├── @ ConstantPathNode (location: (3,0)-(3,7)) │ ├── parent: │ │ @ ConstantPathNode (location: (3,0)-(3,4)) │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (3,0)-(3,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (3,3)-(3,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (3,1)-(3,3) = "::" - │ ├── child: - │ │ @ ConstantReadNode (location: (3,6)-(3,7)) - │ │ └── name: :C - │ └── delimiter_loc: (3,4)-(3,6) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (3,1)-(3,3) = "::" + │ │ └── name_loc: (3,3)-(3,4) = "B" + │ ├── name: :C + │ ├── delimiter_loc: (3,4)-(3,6) = "::" + │ └── name_loc: (3,6)-(3,7) = "C" ├── @ ConstantPathNode (location: (5,0)-(5,4)) │ ├── parent: │ │ @ CallNode (location: (5,0)-(5,1)) @@ -37,20 +34,18 @@ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (5,3)-(5,4)) - │ │ └── name: :B - │ └── delimiter_loc: (5,1)-(5,3) = "::" + │ ├── name: :B + │ ├── delimiter_loc: (5,1)-(5,3) = "::" + │ └── name_loc: (5,3)-(5,4) = "B" ├── @ ConstantPathWriteNode (location: (7,0)-(7,8)) │ ├── target: │ │ @ ConstantPathNode (location: (7,0)-(7,4)) │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (7,0)-(7,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (7,3)-(7,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (7,1)-(7,3) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (7,1)-(7,3) = "::" + │ │ └── name_loc: (7,3)-(7,4) = "B" │ ├── operator_loc: (7,5)-(7,6) = "=" │ └── value: │ @ IntegerNode (location: (7,7)-(7,8)) @@ -117,7 +112,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (17,4)-(17,9)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (17,4)-(17,9)) │ │ ├── flags: ∅ @@ -199,7 +194,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (23,9)-(23,14)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (23,9)-(23,14)) │ │ ├── flags: ∅ @@ -249,10 +244,9 @@ │ ├── receiver: │ │ @ ConstantPathNode (location: (27,0)-(27,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (27,2)-(27,3)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (27,0)-(27,2) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (27,0)-(27,2) = "::" + │ │ └── name_loc: (27,2)-(27,3) = "A" │ ├── call_operator_loc: (27,3)-(27,5) = "::" │ ├── name: :foo │ ├── message_loc: (27,5)-(27,8) = "foo" @@ -264,10 +258,9 @@ │ ├── target: │ │ @ ConstantPathNode (location: (29,0)-(29,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (29,2)-(29,3)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (29,0)-(29,2) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (29,0)-(29,2) = "::" + │ │ └── name_loc: (29,2)-(29,3) = "A" │ ├── operator_loc: (29,4)-(29,5) = "=" │ └── value: │ @ IntegerNode (location: (29,6)-(29,7)) @@ -279,14 +272,12 @@ │ │ ├── parent: │ │ │ @ ConstantPathNode (location: (31,0)-(31,3)) │ │ │ ├── parent: ∅ - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (31,2)-(31,3)) - │ │ │ │ └── name: :A - │ │ │ └── delimiter_loc: (31,0)-(31,2) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (31,5)-(31,6)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (31,3)-(31,5) = "::" + │ │ │ ├── name: :A + │ │ │ ├── delimiter_loc: (31,0)-(31,2) = "::" + │ │ │ └── name_loc: (31,2)-(31,3) = "A" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (31,3)-(31,5) = "::" + │ │ └── name_loc: (31,5)-(31,6) = "B" │ ├── operator_loc: (31,7)-(31,8) = "=" │ └── value: │ @ IntegerNode (location: (31,9)-(31,10)) @@ -296,20 +287,17 @@ │ ├── parent: │ │ @ ConstantPathNode (location: (33,0)-(33,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (33,2)-(33,3)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (33,0)-(33,2) = "::" - │ ├── child: - │ │ @ ConstantReadNode (location: (33,5)-(33,6)) - │ │ └── name: :B - │ └── delimiter_loc: (33,3)-(33,5) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (33,0)-(33,2) = "::" + │ │ └── name_loc: (33,2)-(33,3) = "A" + │ ├── name: :B + │ ├── delimiter_loc: (33,3)-(33,5) = "::" + │ └── name_loc: (33,5)-(33,6) = "B" ├── @ ConstantPathNode (location: (35,0)-(35,3)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (35,2)-(35,3)) - │ │ └── name: :A - │ └── delimiter_loc: (35,0)-(35,2) = "::" + │ ├── name: :A + │ ├── delimiter_loc: (35,0)-(35,2) = "::" + │ └── name_loc: (35,2)-(35,3) = "A" ├── @ CallNode (location: (37,0)-(37,8)) │ ├── flags: ∅ │ ├── receiver: @@ -329,10 +317,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (39,0)-(39,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (39,3)-(39,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (39,1)-(39,3) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (39,1)-(39,3) = "::" + │ │ └── name_loc: (39,3)-(39,4) = "B" │ ├── call_operator_loc: (39,4)-(39,6) = "::" │ ├── name: :true │ ├── message_loc: (39,6)-(39,10) = "true" @@ -488,10 +475,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (65,0)-(65,1)) │ │ └── name: :A - │ ├── child: - │ │ @ ConstantReadNode (location: (67,0)-(67,1)) - │ │ └── name: :C - │ └── delimiter_loc: (65,1)-(65,3) = "::" + │ ├── name: :C + │ ├── delimiter_loc: (65,1)-(65,3) = "::" + │ └── name_loc: (67,0)-(67,1) = "C" ├── @ CallNode (location: (69,0)-(69,8)) │ ├── flags: ∅ │ ├── receiver: @@ -532,10 +518,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (75,0)-(75,1)) │ │ └── name: :A - │ ├── child: - │ │ @ ConstantReadNode (location: (75,3)-(75,8)) - │ │ └── name: :BEGIN - │ └── delimiter_loc: (75,1)-(75,3) = "::" + │ ├── name: :BEGIN + │ ├── delimiter_loc: (75,1)-(75,3) = "::" + │ └── name_loc: (75,3)-(75,8) = "BEGIN" ├── @ CallNode (location: (77,0)-(77,8)) │ ├── flags: ∅ │ ├── receiver: @@ -636,10 +621,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (93,0)-(93,1)) │ │ └── name: :A - │ ├── child: - │ │ @ ConstantReadNode (location: (93,3)-(93,6)) - │ │ └── name: :END - │ └── delimiter_loc: (93,1)-(93,3) = "::" + │ ├── name: :END + │ ├── delimiter_loc: (93,1)-(93,3) = "::" + │ └── name_loc: (93,3)-(93,6) = "END" ├── @ CallNode (location: (95,0)-(95,9)) │ ├── flags: ∅ │ ├── receiver: @@ -1207,10 +1191,9 @@ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (180,0)-(180,1)) - │ │ └── name: :C - │ └── delimiter_loc: (179,4)-(179,6) = "::" + │ ├── name: :C + │ ├── delimiter_loc: (179,4)-(179,6) = "::" + │ └── name_loc: (180,0)-(180,1) = "C" └── @ RangeNode (location: (182,0)-(184,10)) ├── flags: ∅ ├── left: diff --git a/test/prism/snapshots/defined.txt b/test/prism/snapshots/defined.txt index 53a5081811..c60173ff37 100644 --- a/test/prism/snapshots/defined.txt +++ b/test/prism/snapshots/defined.txt @@ -28,13 +28,13 @@ │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (3,9)-(3,15)) │ │ ├── name_loc: (3,9)-(3,10) = "x" - │ │ ├── operator_loc: (3,11)-(3,13) = "%=" + │ │ ├── binary_operator_loc: (3,11)-(3,13) = "%=" │ │ ├── value: │ │ │ @ IntegerNode (location: (3,14)-(3,15)) │ │ │ ├── flags: decimal │ │ │ └── value: 2 │ │ ├── name: :x - │ │ ├── operator: :% + │ │ ├── binary_operator: :% │ │ └── depth: 0 │ ├── rparen_loc: (3,15)-(3,16) = ")" │ └── keyword_loc: (3,0)-(3,8) = "defined?" diff --git a/test/prism/snapshots/heredocs_nested.txt b/test/prism/snapshots/heredocs_nested.txt index f830b028c7..da13e48c51 100644 --- a/test/prism/snapshots/heredocs_nested.txt +++ b/test/prism/snapshots/heredocs_nested.txt @@ -4,7 +4,7 @@ @ StatementsNode (location: (1,0)-(12,4)) └── body: (length: 2) ├── @ InterpolatedStringNode (location: (1,0)-(1,7)) - │ ├── flags: ∅ + │ ├── flags: mutable │ ├── opening_loc: (1,0)-(1,7) = "<<~RUBY" │ ├── parts: (length: 4) │ │ ├── @ StringNode (location: (2,0)-(3,0)) @@ -19,7 +19,7 @@ │ │ │ │ @ StatementsNode (location: (4,0)-(4,6)) │ │ │ │ └── body: (length: 1) │ │ │ │ └── @ StringNode (location: (4,0)-(4,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (4,0)-(4,6) = "<<RUBY" │ │ │ │ ├── content_loc: (5,0)-(6,0) = " hello\n" │ │ │ │ ├── closing_loc: (6,0)-(7,0) = "RUBY\n" diff --git a/test/prism/snapshots/if.txt b/test/prism/snapshots/if.txt index 31c33d368f..4114d22722 100644 --- a/test/prism/snapshots/if.txt +++ b/test/prism/snapshots/if.txt @@ -307,7 +307,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── arguments: │ │ │ @ ArgumentsNode (location: (25,4)-(25,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: contains_keywords │ │ │ └── arguments: (length: 1) │ │ │ └── @ KeywordHashNode (location: (25,4)-(25,6)) │ │ │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/method_calls.txt b/test/prism/snapshots/method_calls.txt index de9ba71ae0..6082b567f7 100644 --- a/test/prism/snapshots/method_calls.txt +++ b/test/prism/snapshots/method_calls.txt @@ -308,7 +308,7 @@ │ ├── opening_loc: (27,1)-(27,2) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (27,2)-(27,10)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (27,2)-(27,10)) │ │ ├── flags: ∅ @@ -779,7 +779,7 @@ │ ├── opening_loc: (60,3)-(60,4) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (60,4)-(60,32)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ SymbolNode (location: (60,4)-(60,6)) │ │ │ ├── flags: forced_us_ascii_encoding @@ -912,7 +912,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (64,4)-(64,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ SymbolNode (location: (64,4)-(64,6)) │ │ │ ├── flags: forced_us_ascii_encoding @@ -988,7 +988,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (66,3)-(66,17)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (66,3)-(66,17)) │ │ ├── flags: symbol_keys @@ -1020,7 +1020,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (68,3)-(68,40)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (68,3)-(68,40)) │ │ ├── flags: ∅ @@ -1075,7 +1075,7 @@ │ ├── opening_loc: (70,2)-(70,3) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (70,3)-(70,40)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (70,3)-(70,40)) │ │ ├── flags: ∅ @@ -1178,7 +1178,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (74,3)-(74,20)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (74,3)-(74,20)) │ │ ├── flags: symbol_keys @@ -1235,7 +1235,7 @@ │ ├── opening_loc: (80,3)-(80,4) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (81,0)-(82,5)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ SymbolNode (location: (81,0)-(81,2)) │ │ │ ├── flags: forced_us_ascii_encoding @@ -1292,7 +1292,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (87,4)-(87,21)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (87,4)-(87,21)) │ │ ├── flags: symbol_keys @@ -1339,7 +1339,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (89,10)-(89,21)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ IntegerNode (location: (89,10)-(89,11)) │ │ │ ├── flags: decimal @@ -1443,10 +1443,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (97,0)-(97,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (97,3)-(97,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (97,1)-(97,3) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (97,1)-(97,3) = "::" + │ │ └── name_loc: (97,3)-(97,4) = "B" │ ├── call_operator_loc: (97,4)-(97,6) = "::" │ ├── name: :C │ ├── message_loc: (97,6)-(97,7) = "C" @@ -1470,10 +1469,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (99,0)-(99,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (99,3)-(99,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (99,1)-(99,3) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (99,1)-(99,3) = "::" + │ │ └── name_loc: (99,3)-(99,4) = "B" │ ├── call_operator_loc: (99,4)-(99,6) = "::" │ ├── name: :C │ ├── message_loc: (99,6)-(99,7) = "C" @@ -1497,10 +1495,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (101,0)-(101,1)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (101,3)-(101,4)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (101,1)-(101,3) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (101,1)-(101,3) = "::" + │ │ └── name_loc: (101,3)-(101,4) = "B" │ ├── call_operator_loc: (101,4)-(101,6) = "::" │ ├── name: :C │ ├── message_loc: (101,6)-(101,7) = "C" @@ -1532,7 +1529,7 @@ │ ├── opening_loc: (103,3)-(103,4) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (103,4)-(103,11)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (103,4)-(103,11)) │ │ ├── flags: symbol_keys @@ -1561,7 +1558,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (105,4)-(105,28)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (105,4)-(105,28)) │ │ ├── flags: symbol_keys @@ -1617,7 +1614,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (107,4)-(107,24)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (107,4)-(107,24)) │ │ ├── flags: symbol_keys @@ -2417,7 +2414,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (156,5)-(156,19)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (156,5)-(156,19)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/methods.txt b/test/prism/snapshots/methods.txt index 76c0361827..b38640399b 100644 --- a/test/prism/snapshots/methods.txt +++ b/test/prism/snapshots/methods.txt @@ -1296,7 +1296,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── arguments: │ │ │ @ ArgumentsNode (location: (139,11)-(139,30)) - │ │ │ ├── flags: contains_keyword_splat + │ │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ │ └── arguments: (length: 1) │ │ │ └── @ KeywordHashNode (location: (139,11)-(139,30)) │ │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/modules.txt b/test/prism/snapshots/modules.txt index 1a0eb6328a..de1ea8feeb 100644 --- a/test/prism/snapshots/modules.txt +++ b/test/prism/snapshots/modules.txt @@ -72,10 +72,9 @@ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (5,10)-(5,11)) - │ │ │ └── name: :M - │ │ └── delimiter_loc: (5,8)-(5,10) = "::" + │ │ ├── name: :M + │ │ ├── delimiter_loc: (5,8)-(5,10) = "::" + │ │ └── name_loc: (5,10)-(5,11) = "M" │ ├── body: ∅ │ ├── end_keyword_loc: (6,0)-(6,3) = "end" │ └── name: :M @@ -119,10 +118,9 @@ │ ├── constant_path: │ │ @ ConstantPathNode (location: (11,7)-(11,10)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (11,9)-(11,10)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (11,7)-(11,9) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (11,7)-(11,9) = "::" + │ │ └── name_loc: (11,9)-(11,10) = "A" │ ├── body: ∅ │ ├── end_keyword_loc: (12,0)-(12,3) = "end" │ └── name: :A @@ -144,10 +142,9 @@ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: (14,9)-(14,10) = "]" │ │ │ └── block: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (14,12)-(14,13)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (14,10)-(14,12) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (14,10)-(14,12) = "::" + │ │ └── name_loc: (14,12)-(14,13) = "B" │ ├── body: ∅ │ ├── end_keyword_loc: (15,0)-(15,3) = "end" │ └── name: :B @@ -175,10 +172,9 @@ │ │ │ └── value: 1 │ │ ├── closing_loc: (17,10)-(17,11) = "]" │ │ └── block: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (17,13)-(17,14)) - │ │ └── name: :B - │ └── delimiter_loc: (17,11)-(17,13) = "::" + │ ├── name: :B + │ ├── delimiter_loc: (17,11)-(17,13) = "::" + │ └── name_loc: (17,13)-(17,14) = "B" ├── body: ∅ ├── end_keyword_loc: (18,0)-(18,3) = "end" └── name: :B diff --git a/test/prism/snapshots/numbers.txt b/test/prism/snapshots/numbers.txt index 740f3f5a2a..58aea454fa 100644 --- a/test/prism/snapshots/numbers.txt +++ b/test/prism/snapshots/numbers.txt @@ -65,52 +65,48 @@ │ ├── flags: decimal │ └── value: 1 ├── @ RationalNode (location: (41,0)-(41,2)) - │ └── numeric: - │ @ IntegerNode (location: (41,0)-(41,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ IntegerNode (location: (43,0)-(43,2)) │ ├── flags: decimal │ └── value: -1 ├── @ ImaginaryNode (location: (45,0)-(45,3)) │ └── numeric: │ @ RationalNode (location: (45,0)-(45,2)) - │ └── numeric: - │ @ IntegerNode (location: (45,0)-(45,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (47,0)-(47,4)) - │ └── numeric: - │ @ FloatNode (location: (47,0)-(47,3)) - │ └── value: 1.2 + │ ├── flags: decimal + │ ├── numerator: 6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (49,0)-(49,5)) │ └── numeric: │ @ RationalNode (location: (49,0)-(49,4)) - │ └── numeric: - │ @ FloatNode (location: (49,0)-(49,3)) - │ └── value: 1.2 + │ ├── flags: decimal + │ ├── numerator: 6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (51,0)-(51,4)) │ └── numeric: │ @ RationalNode (location: (51,0)-(51,3)) - │ └── numeric: - │ @ IntegerNode (location: (51,0)-(51,2)) - │ ├── flags: decimal - │ └── value: -1 + │ ├── flags: decimal + │ ├── numerator: -1 + │ └── denominator: 1 ├── @ RationalNode (location: (53,0)-(53,5)) - │ └── numeric: - │ @ FloatNode (location: (53,0)-(53,4)) - │ └── value: -1.2 + │ ├── flags: decimal + │ ├── numerator: -6 + │ └── denominator: 5 ├── @ ImaginaryNode (location: (55,0)-(55,6)) │ └── numeric: │ @ RationalNode (location: (55,0)-(55,5)) - │ └── numeric: - │ @ FloatNode (location: (55,0)-(55,4)) - │ └── value: -1.2 + │ ├── flags: decimal + │ ├── numerator: -6 + │ └── denominator: 5 ├── @ RationalNode (location: (57,0)-(57,4)) - │ └── numeric: - │ @ IntegerNode (location: (57,0)-(57,3)) - │ ├── flags: octal - │ └── value: 1 + │ ├── flags: octal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ ImaginaryNode (location: (59,0)-(59,4)) │ └── numeric: │ @ IntegerNode (location: (59,0)-(59,3)) @@ -119,15 +115,13 @@ ├── @ ImaginaryNode (location: (61,0)-(61,5)) │ └── numeric: │ @ RationalNode (location: (61,0)-(61,4)) - │ └── numeric: - │ @ IntegerNode (location: (61,0)-(61,3)) - │ ├── flags: octal - │ └── value: 1 + │ ├── flags: octal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (63,0)-(63,4)) - │ └── numeric: - │ @ IntegerNode (location: (63,0)-(63,3)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ ImaginaryNode (location: (65,0)-(65,4)) │ └── numeric: │ @ IntegerNode (location: (65,0)-(65,3)) @@ -136,7 +130,6 @@ └── @ ImaginaryNode (location: (67,0)-(67,5)) └── numeric: @ RationalNode (location: (67,0)-(67,4)) - └── numeric: - @ IntegerNode (location: (67,0)-(67,3)) - ├── flags: binary - └── value: 1 + ├── flags: binary + ├── numerator: 1 + └── denominator: 1 diff --git a/test/prism/snapshots/patterns.txt b/test/prism/snapshots/patterns.txt index 5662129dae..17aa23b4b9 100644 --- a/test/prism/snapshots/patterns.txt +++ b/test/prism/snapshots/patterns.txt @@ -86,10 +86,9 @@ │ │ └── block: ∅ │ ├── pattern: │ │ @ RationalNode (location: (5,7)-(5,9)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (5,7)-(5,8)) - │ │ ├── flags: decimal - │ │ └── value: 1 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ └── operator_loc: (5,4)-(5,6) = "=>" ├── @ MatchRequiredNode (location: (6,0)-(6,11)) │ ├── value: @@ -598,16 +597,14 @@ │ │ ├── flags: ∅ │ │ ├── left: │ │ │ @ RationalNode (location: (31,7)-(31,9)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (31,7)-(31,8)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ ├── right: │ │ │ @ RationalNode (location: (31,13)-(31,15)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (31,13)-(31,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ └── operator_loc: (31,10)-(31,12) = ".." │ └── operator_loc: (31,4)-(31,6) = "=>" ├── @ MatchRequiredNode (location: (32,0)-(32,19)) @@ -1454,14 +1451,12 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (64,7)-(64,10)) │ │ │ │ └── name: :Foo - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (64,12)-(64,15)) - │ │ │ │ └── name: :Bar - │ │ │ └── delimiter_loc: (64,10)-(64,12) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (64,17)-(64,20)) - │ │ │ └── name: :Baz - │ │ └── delimiter_loc: (64,15)-(64,17) = "::" + │ │ │ ├── name: :Bar + │ │ │ ├── delimiter_loc: (64,10)-(64,12) = "::" + │ │ │ └── name_loc: (64,12)-(64,15) = "Bar" + │ │ ├── name: :Baz + │ │ ├── delimiter_loc: (64,15)-(64,17) = "::" + │ │ └── name_loc: (64,17)-(64,20) = "Baz" │ └── operator_loc: (64,4)-(64,6) = "=>" ├── @ MatchRequiredNode (location: (65,0)-(65,12)) │ ├── value: @@ -1478,10 +1473,9 @@ │ ├── pattern: │ │ @ ConstantPathNode (location: (65,7)-(65,12)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (65,9)-(65,12)) - │ │ │ └── name: :Foo - │ │ └── delimiter_loc: (65,7)-(65,9) = "::" + │ │ ├── name: :Foo + │ │ ├── delimiter_loc: (65,7)-(65,9) = "::" + │ │ └── name_loc: (65,9)-(65,12) = "Foo" │ └── operator_loc: (65,4)-(65,6) = "=>" ├── @ MatchRequiredNode (location: (66,0)-(66,22)) │ ├── value: @@ -1502,18 +1496,15 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantPathNode (location: (66,7)-(66,12)) │ │ │ │ ├── parent: ∅ - │ │ │ │ ├── child: - │ │ │ │ │ @ ConstantReadNode (location: (66,9)-(66,12)) - │ │ │ │ │ └── name: :Foo - │ │ │ │ └── delimiter_loc: (66,7)-(66,9) = "::" - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (66,14)-(66,17)) - │ │ │ │ └── name: :Bar - │ │ │ └── delimiter_loc: (66,12)-(66,14) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (66,19)-(66,22)) - │ │ │ └── name: :Baz - │ │ └── delimiter_loc: (66,17)-(66,19) = "::" + │ │ │ │ ├── name: :Foo + │ │ │ │ ├── delimiter_loc: (66,7)-(66,9) = "::" + │ │ │ │ └── name_loc: (66,9)-(66,12) = "Foo" + │ │ │ ├── name: :Bar + │ │ │ ├── delimiter_loc: (66,12)-(66,14) = "::" + │ │ │ └── name_loc: (66,14)-(66,17) = "Bar" + │ │ ├── name: :Baz + │ │ ├── delimiter_loc: (66,17)-(66,19) = "::" + │ │ └── name_loc: (66,19)-(66,22) = "Baz" │ └── operator_loc: (66,4)-(66,6) = "=>" ├── @ MatchRequiredNode (location: (68,0)-(68,12)) │ ├── value: @@ -2467,10 +2458,9 @@ │ │ └── block: ∅ │ ├── pattern: │ │ @ RationalNode (location: (108,7)-(108,9)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (108,7)-(108,8)) - │ │ ├── flags: decimal - │ │ └── value: 1 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ └── operator_loc: (108,4)-(108,6) = "in" ├── @ MatchPredicateNode (location: (109,0)-(109,11)) │ ├── value: @@ -3023,10 +3013,9 @@ │ │ └── @ InNode (location: (139,10)-(139,20)) │ │ ├── pattern: │ │ │ @ RationalNode (location: (139,13)-(139,15)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (139,13)-(139,14)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ ├── statements: ∅ │ │ ├── in_loc: (139,10)-(139,12) = "in" │ │ └── then_loc: (139,16)-(139,20) = "then" @@ -3764,10 +3753,9 @@ │ │ │ │ @ StatementsNode (location: (166,13)-(166,15)) │ │ │ │ └── body: (length: 1) │ │ │ │ └── @ RationalNode (location: (166,13)-(166,15)) - │ │ │ │ └── numeric: - │ │ │ │ @ IntegerNode (location: (166,13)-(166,14)) - │ │ │ │ ├── flags: decimal - │ │ │ │ └── value: 1 + │ │ │ │ ├── flags: decimal + │ │ │ │ ├── numerator: 1 + │ │ │ │ └── denominator: 1 │ │ │ ├── consequent: ∅ │ │ │ └── end_keyword_loc: ∅ │ │ ├── statements: ∅ diff --git a/test/prism/snapshots/rescue.txt b/test/prism/snapshots/rescue.txt index 2bdbfdaff3..390b08ae0e 100644 --- a/test/prism/snapshots/rescue.txt +++ b/test/prism/snapshots/rescue.txt @@ -380,7 +380,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: │ │ │ │ @ ArgumentsNode (location: (29,4)-(29,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: contains_keywords │ │ │ │ └── arguments: (length: 1) │ │ │ │ └── @ KeywordHashNode (location: (29,4)-(29,6)) │ │ │ │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/assoc_label.txt b/test/prism/snapshots/seattlerb/assoc_label.txt index 923f5450f4..70490c0da4 100644 --- a/test/prism/snapshots/seattlerb/assoc_label.txt +++ b/test/prism/snapshots/seattlerb/assoc_label.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,5)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,5)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/bug_249.txt b/test/prism/snapshots/seattlerb/bug_249.txt index 569bea14c5..ad61501a07 100644 --- a/test/prism/snapshots/seattlerb/bug_249.txt +++ b/test/prism/snapshots/seattlerb/bug_249.txt @@ -12,7 +12,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (1,6)-(4,28)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ CallNode (location: (1,6)-(4,9)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/bug_hash_args.txt b/test/prism/snapshots/seattlerb/bug_hash_args.txt index 6f17e88714..e138db4d49 100644 --- a/test/prism/snapshots/seattlerb/bug_hash_args.txt +++ b/test/prism/snapshots/seattlerb/bug_hash_args.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,3)-(1,4) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,18)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ SymbolNode (location: (1,4)-(1,8)) │ │ ├── flags: forced_us_ascii_encoding diff --git a/test/prism/snapshots/seattlerb/bug_hash_args_trailing_comma.txt b/test/prism/snapshots/seattlerb/bug_hash_args_trailing_comma.txt index e7256b337b..fe2d7f73c9 100644 --- a/test/prism/snapshots/seattlerb/bug_hash_args_trailing_comma.txt +++ b/test/prism/snapshots/seattlerb/bug_hash_args_trailing_comma.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,3)-(1,4) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,18)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ SymbolNode (location: (1,4)-(1,8)) │ │ ├── flags: forced_us_ascii_encoding diff --git a/test/prism/snapshots/seattlerb/call_arg_assoc.txt b/test/prism/snapshots/seattlerb/call_arg_assoc.txt index 27c19fd339..f489bc7f19 100644 --- a/test/prism/snapshots/seattlerb/call_arg_assoc.txt +++ b/test/prism/snapshots/seattlerb/call_arg_assoc.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,9)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ IntegerNode (location: (1,2)-(1,3)) │ │ ├── flags: decimal diff --git a/test/prism/snapshots/seattlerb/call_arg_assoc_kwsplat.txt b/test/prism/snapshots/seattlerb/call_arg_assoc_kwsplat.txt index 0193eb1dfc..5b191396de 100644 --- a/test/prism/snapshots/seattlerb/call_arg_assoc_kwsplat.txt +++ b/test/prism/snapshots/seattlerb/call_arg_assoc_kwsplat.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,15)) - │ ├── flags: contains_keyword_splat + │ ├── flags: contains_keywords, contains_keyword_splat │ └── arguments: (length: 2) │ ├── @ IntegerNode (location: (1,2)-(1,3)) │ │ ├── flags: decimal diff --git a/test/prism/snapshots/seattlerb/call_arg_kwsplat.txt b/test/prism/snapshots/seattlerb/call_arg_kwsplat.txt index 91c7725525..f95b80cf7d 100644 --- a/test/prism/snapshots/seattlerb/call_arg_kwsplat.txt +++ b/test/prism/snapshots/seattlerb/call_arg_kwsplat.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,8)) - │ ├── flags: contains_keyword_splat + │ ├── flags: contains_keywords, contains_keyword_splat │ └── arguments: (length: 2) │ ├── @ CallNode (location: (1,2)-(1,3)) │ │ ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/seattlerb/call_args_assoc_quoted.txt b/test/prism/snapshots/seattlerb/call_args_assoc_quoted.txt index 2d6f81c818..8946206a3f 100644 --- a/test/prism/snapshots/seattlerb/call_args_assoc_quoted.txt +++ b/test/prism/snapshots/seattlerb/call_args_assoc_quoted.txt @@ -12,7 +12,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (1,2)-(1,11)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,2)-(1,11)) │ │ ├── flags: ∅ @@ -55,7 +55,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (3,2)-(3,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (3,2)-(3,8)) │ │ ├── flags: symbol_keys @@ -84,7 +84,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (5,2)-(5,8)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (5,2)-(5,8)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/call_args_assoc_trailing_comma.txt b/test/prism/snapshots/seattlerb/call_args_assoc_trailing_comma.txt index 312a1981a0..0ba5891cf6 100644 --- a/test/prism/snapshots/seattlerb/call_args_assoc_trailing_comma.txt +++ b/test/prism/snapshots/seattlerb/call_args_assoc_trailing_comma.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,9)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ IntegerNode (location: (1,2)-(1,3)) │ │ ├── flags: decimal diff --git a/test/prism/snapshots/seattlerb/call_assoc.txt b/test/prism/snapshots/seattlerb/call_assoc.txt index 438c256553..60784e6095 100644 --- a/test/prism/snapshots/seattlerb/call_assoc.txt +++ b/test/prism/snapshots/seattlerb/call_assoc.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,6)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,6)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/call_assoc_new.txt b/test/prism/snapshots/seattlerb/call_assoc_new.txt index b4d7e0bf83..dc25fb2493 100644 --- a/test/prism/snapshots/seattlerb/call_assoc_new.txt +++ b/test/prism/snapshots/seattlerb/call_assoc_new.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,5)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,5)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/call_assoc_new_if_multiline.txt b/test/prism/snapshots/seattlerb/call_assoc_new_if_multiline.txt index 9587e2e074..b3d652e879 100644 --- a/test/prism/snapshots/seattlerb/call_assoc_new_if_multiline.txt +++ b/test/prism/snapshots/seattlerb/call_assoc_new_if_multiline.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(5,3)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(5,3)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/call_assoc_trailing_comma.txt b/test/prism/snapshots/seattlerb/call_assoc_trailing_comma.txt index 8d0b285172..b2012f0f75 100644 --- a/test/prism/snapshots/seattlerb/call_assoc_trailing_comma.txt +++ b/test/prism/snapshots/seattlerb/call_assoc_trailing_comma.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,6)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,6)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/call_kwsplat.txt b/test/prism/snapshots/seattlerb/call_kwsplat.txt index 4199e97a44..e0620dc5f0 100644 --- a/test/prism/snapshots/seattlerb/call_kwsplat.txt +++ b/test/prism/snapshots/seattlerb/call_kwsplat.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,1)-(1,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,5)) - │ ├── flags: contains_keyword_splat + │ ├── flags: contains_keywords, contains_keyword_splat │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,5)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/case_in_86.txt b/test/prism/snapshots/seattlerb/case_in_86.txt index 5889137844..082aa74eca 100644 --- a/test/prism/snapshots/seattlerb/case_in_86.txt +++ b/test/prism/snapshots/seattlerb/case_in_86.txt @@ -30,10 +30,9 @@ │ │ ├── requireds: (length: 1) │ │ │ └── @ ConstantPathNode (location: (2,3)-(2,13)) │ │ │ ├── parent: ∅ - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (2,5)-(2,13)) - │ │ │ │ └── name: :NilClass - │ │ │ └── delimiter_loc: (2,3)-(2,5) = "::" + │ │ │ ├── name: :NilClass + │ │ │ ├── delimiter_loc: (2,3)-(2,5) = "::" + │ │ │ └── name_loc: (2,5)-(2,13) = "NilClass" │ │ ├── rest: │ │ │ @ SplatNode (location: (2,15)-(2,16)) │ │ │ ├── operator_loc: (2,15)-(2,16) = "*" diff --git a/test/prism/snapshots/seattlerb/case_in_86_2.txt b/test/prism/snapshots/seattlerb/case_in_86_2.txt index 18ce70ae93..346264f907 100644 --- a/test/prism/snapshots/seattlerb/case_in_86_2.txt +++ b/test/prism/snapshots/seattlerb/case_in_86_2.txt @@ -35,10 +35,9 @@ │ │ ├── posts: (length: 1) │ │ │ └── @ ConstantPathNode (location: (2,6)-(2,16)) │ │ │ ├── parent: ∅ - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (2,8)-(2,16)) - │ │ │ │ └── name: :NilClass - │ │ │ └── delimiter_loc: (2,6)-(2,8) = "::" + │ │ │ ├── name: :NilClass + │ │ │ ├── delimiter_loc: (2,6)-(2,8) = "::" + │ │ │ └── name_loc: (2,8)-(2,16) = "NilClass" │ │ ├── opening_loc: ∅ │ │ └── closing_loc: ∅ │ ├── statements: diff --git a/test/prism/snapshots/seattlerb/case_in_array_pat_const2.txt b/test/prism/snapshots/seattlerb/case_in_array_pat_const2.txt index c783af9ce5..d6fb80ef90 100644 --- a/test/prism/snapshots/seattlerb/case_in_array_pat_const2.txt +++ b/test/prism/snapshots/seattlerb/case_in_array_pat_const2.txt @@ -20,10 +20,9 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (2,3)-(2,4)) │ │ │ │ └── name: :B - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (2,6)-(2,7)) - │ │ │ │ └── name: :C - │ │ │ └── delimiter_loc: (2,4)-(2,6) = "::" + │ │ │ ├── name: :C + │ │ │ ├── delimiter_loc: (2,4)-(2,6) = "::" + │ │ │ └── name_loc: (2,6)-(2,7) = "C" │ │ ├── requireds: (length: 1) │ │ │ └── @ LocalVariableTargetNode (location: (2,8)-(2,9)) │ │ │ ├── name: :d diff --git a/test/prism/snapshots/seattlerb/case_in_multiple.txt b/test/prism/snapshots/seattlerb/case_in_multiple.txt index d8597c4bfa..eba0084f96 100644 --- a/test/prism/snapshots/seattlerb/case_in_multiple.txt +++ b/test/prism/snapshots/seattlerb/case_in_multiple.txt @@ -18,10 +18,9 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (2,3)-(2,4)) │ │ │ │ └── name: :A - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (2,6)-(2,7)) - │ │ │ │ └── name: :B - │ │ │ └── delimiter_loc: (2,4)-(2,6) = "::" + │ │ │ ├── name: :B + │ │ │ ├── delimiter_loc: (2,4)-(2,6) = "::" + │ │ │ └── name_loc: (2,6)-(2,7) = "B" │ │ ├── statements: │ │ │ @ StatementsNode (location: (3,2)-(3,4)) │ │ │ └── body: (length: 1) @@ -39,10 +38,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (4,3)-(4,4)) │ │ │ └── name: :D - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (4,6)-(4,7)) - │ │ │ └── name: :E - │ │ └── delimiter_loc: (4,4)-(4,6) = "::" + │ │ ├── name: :E + │ │ ├── delimiter_loc: (4,4)-(4,6) = "::" + │ │ └── name_loc: (4,6)-(4,7) = "E" │ ├── statements: │ │ @ StatementsNode (location: (5,2)-(5,4)) │ │ └── body: (length: 1) diff --git a/test/prism/snapshots/seattlerb/const_2_op_asgn_or2.txt b/test/prism/snapshots/seattlerb/const_2_op_asgn_or2.txt index b018ac48d4..e09eed7d2f 100644 --- a/test/prism/snapshots/seattlerb/const_2_op_asgn_or2.txt +++ b/test/prism/snapshots/seattlerb/const_2_op_asgn_or2.txt @@ -9,14 +9,12 @@ │ ├── parent: │ │ @ ConstantPathNode (location: (1,0)-(1,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ │ └── name: :X - │ │ └── delimiter_loc: (1,0)-(1,2) = "::" - │ ├── child: - │ │ @ ConstantReadNode (location: (1,5)-(1,6)) - │ │ └── name: :Y - │ └── delimiter_loc: (1,3)-(1,5) = "::" + │ │ ├── name: :X + │ │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ │ └── name_loc: (1,2)-(1,3) = "X" + │ ├── name: :Y + │ ├── delimiter_loc: (1,3)-(1,5) = "::" + │ └── name_loc: (1,5)-(1,6) = "Y" ├── operator_loc: (1,7)-(1,10) = "||=" └── value: @ IntegerNode (location: (1,11)-(1,12)) diff --git a/test/prism/snapshots/seattlerb/const_3_op_asgn_or.txt b/test/prism/snapshots/seattlerb/const_3_op_asgn_or.txt index 8d9d94931b..398af888a8 100644 --- a/test/prism/snapshots/seattlerb/const_3_op_asgn_or.txt +++ b/test/prism/snapshots/seattlerb/const_3_op_asgn_or.txt @@ -7,10 +7,9 @@ ├── target: │ @ ConstantPathNode (location: (1,0)-(1,3)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ └── name: :X - │ └── delimiter_loc: (1,0)-(1,2) = "::" + │ ├── name: :X + │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ └── name_loc: (1,2)-(1,3) = "X" ├── operator_loc: (1,4)-(1,7) = "||=" └── value: @ IntegerNode (location: (1,8)-(1,9)) diff --git a/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt b/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt index b1d61b3752..f9792aebb3 100644 --- a/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt +++ b/test/prism/snapshots/seattlerb/const_op_asgn_and1.txt @@ -7,13 +7,12 @@ ├── target: │ @ ConstantPathNode (location: (1,0)-(1,3)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ └── name: :X - │ └── delimiter_loc: (1,0)-(1,2) = "::" - ├── operator_loc: (1,4)-(1,6) = "&=" + │ ├── name: :X + │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ └── name_loc: (1,2)-(1,3) = "X" + ├── binary_operator_loc: (1,4)-(1,6) = "&=" ├── value: │ @ IntegerNode (location: (1,7)-(1,8)) │ ├── flags: decimal │ └── value: 1 - └── operator: :& + └── binary_operator: :& diff --git a/test/prism/snapshots/seattlerb/const_op_asgn_and2.txt b/test/prism/snapshots/seattlerb/const_op_asgn_and2.txt index 22f6682534..146455d327 100644 --- a/test/prism/snapshots/seattlerb/const_op_asgn_and2.txt +++ b/test/prism/snapshots/seattlerb/const_op_asgn_and2.txt @@ -7,10 +7,9 @@ ├── target: │ @ ConstantPathNode (location: (1,0)-(1,3)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ └── name: :X - │ └── delimiter_loc: (1,0)-(1,2) = "::" + │ ├── name: :X + │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ └── name_loc: (1,2)-(1,3) = "X" ├── operator_loc: (1,4)-(1,7) = "&&=" └── value: @ IntegerNode (location: (1,8)-(1,9)) diff --git a/test/prism/snapshots/seattlerb/const_op_asgn_or.txt b/test/prism/snapshots/seattlerb/const_op_asgn_or.txt index 067e0fbb93..5e9dd39604 100644 --- a/test/prism/snapshots/seattlerb/const_op_asgn_or.txt +++ b/test/prism/snapshots/seattlerb/const_op_asgn_or.txt @@ -9,10 +9,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (1,0)-(1,1)) │ │ └── name: :X - │ ├── child: - │ │ @ ConstantReadNode (location: (1,3)-(1,4)) - │ │ └── name: :Y - │ └── delimiter_loc: (1,1)-(1,3) = "::" + │ ├── name: :Y + │ ├── delimiter_loc: (1,1)-(1,3) = "::" + │ └── name_loc: (1,3)-(1,4) = "Y" ├── operator_loc: (1,5)-(1,8) = "||=" └── value: @ IntegerNode (location: (1,9)-(1,10)) diff --git a/test/prism/snapshots/seattlerb/defn_kwarg_env.txt b/test/prism/snapshots/seattlerb/defn_kwarg_env.txt index f420420fc3..2aadedd964 100644 --- a/test/prism/snapshots/seattlerb/defn_kwarg_env.txt +++ b/test/prism/snapshots/seattlerb/defn_kwarg_env.txt @@ -33,7 +33,7 @@ │ ├── opening_loc: (1,30)-(1,31) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,31)-(1,40)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,31)-(1,40)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/difficult2_.txt b/test/prism/snapshots/seattlerb/difficult2_.txt index a9b3736fe3..b53d4cad3f 100644 --- a/test/prism/snapshots/seattlerb/difficult2_.txt +++ b/test/prism/snapshots/seattlerb/difficult2_.txt @@ -52,7 +52,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (2,2)-(2,6)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (2,2)-(2,6)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/dstr_evstr.txt b/test/prism/snapshots/seattlerb/dstr_evstr.txt index 8d771e88c2..add8ad6f5c 100644 --- a/test/prism/snapshots/seattlerb/dstr_evstr.txt +++ b/test/prism/snapshots/seattlerb/dstr_evstr.txt @@ -13,7 +13,7 @@ │ │ │ @ StatementsNode (location: (1,3)-(1,6)) │ │ │ └── body: (length: 1) │ │ │ └── @ StringNode (location: (1,3)-(1,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (1,3)-(1,4) = "'" │ │ │ ├── content_loc: (1,4)-(1,5) = "a" │ │ │ ├── closing_loc: (1,5)-(1,6) = "'" diff --git a/test/prism/snapshots/seattlerb/dstr_str.txt b/test/prism/snapshots/seattlerb/dstr_str.txt index 70b5752ce3..6fe0781880 100644 --- a/test/prism/snapshots/seattlerb/dstr_str.txt +++ b/test/prism/snapshots/seattlerb/dstr_str.txt @@ -4,7 +4,7 @@ @ StatementsNode (location: (1,0)-(1,10)) └── body: (length: 1) └── @ InterpolatedStringNode (location: (1,0)-(1,10)) - ├── flags: ∅ + ├── flags: mutable ├── opening_loc: (1,0)-(1,1) = "\"" ├── parts: (length: 2) │ ├── @ EmbeddedStatementsNode (location: (1,1)-(1,7)) @@ -13,7 +13,7 @@ │ │ │ @ StatementsNode (location: (1,3)-(1,6)) │ │ │ └── body: (length: 1) │ │ │ └── @ StringNode (location: (1,3)-(1,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (1,3)-(1,4) = "'" │ │ │ ├── content_loc: (1,4)-(1,5) = "a" │ │ │ ├── closing_loc: (1,5)-(1,6) = "'" diff --git a/test/prism/snapshots/seattlerb/heredoc_nested.txt b/test/prism/snapshots/seattlerb/heredoc_nested.txt index 26d533a33d..a2322b9632 100644 --- a/test/prism/snapshots/seattlerb/heredoc_nested.txt +++ b/test/prism/snapshots/seattlerb/heredoc_nested.txt @@ -7,7 +7,7 @@ ├── flags: ∅ ├── elements: (length: 2) │ ├── @ InterpolatedStringNode (location: (1,1)-(1,4)) - │ │ ├── flags: ∅ + │ │ ├── flags: mutable │ │ ├── opening_loc: (1,1)-(1,4) = "<<A" │ │ ├── parts: (length: 3) │ │ │ ├── @ EmbeddedStatementsNode (location: (2,0)-(2,6)) @@ -16,7 +16,7 @@ │ │ │ │ │ @ StatementsNode (location: (2,2)-(2,5)) │ │ │ │ │ └── body: (length: 1) │ │ │ │ │ └── @ StringNode (location: (2,2)-(2,5)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: (2,2)-(2,5) = "<<B" │ │ │ │ │ ├── content_loc: (3,0)-(4,0) = "b\n" │ │ │ │ │ ├── closing_loc: (4,0)-(5,0) = "B\n" diff --git a/test/prism/snapshots/seattlerb/index_0_opasgn.txt b/test/prism/snapshots/seattlerb/index_0_opasgn.txt index 239a549253..322eae9907 100644 --- a/test/prism/snapshots/seattlerb/index_0_opasgn.txt +++ b/test/prism/snapshots/seattlerb/index_0_opasgn.txt @@ -21,8 +21,8 @@ ├── arguments: ∅ ├── closing_loc: (1,2)-(1,3) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,4)-(1,6) = "+=" └── value: @ CallNode (location: (1,7)-(1,8)) ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/seattlerb/masgn_colon2.txt b/test/prism/snapshots/seattlerb/masgn_colon2.txt index 73ce8a71da..a0dfe72ffc 100644 --- a/test/prism/snapshots/seattlerb/masgn_colon2.txt +++ b/test/prism/snapshots/seattlerb/masgn_colon2.txt @@ -20,10 +20,9 @@ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,6)-(1,7)) - │ │ └── name: :C - │ └── delimiter_loc: (1,4)-(1,6) = "::" + │ ├── name: :C + │ ├── delimiter_loc: (1,4)-(1,6) = "::" + │ └── name_loc: (1,6)-(1,7) = "C" ├── rest: ∅ ├── rights: (length: 0) ├── lparen_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/masgn_colon3.txt b/test/prism/snapshots/seattlerb/masgn_colon3.txt index 0cf4f8626d..f28ed7ecee 100644 --- a/test/prism/snapshots/seattlerb/masgn_colon3.txt +++ b/test/prism/snapshots/seattlerb/masgn_colon3.txt @@ -7,16 +7,14 @@ ├── lefts: (length: 2) │ ├── @ ConstantPathTargetNode (location: (1,0)-(1,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (1,0)-(1,2) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ │ └── name_loc: (1,2)-(1,3) = "A" │ └── @ ConstantPathTargetNode (location: (1,5)-(1,8)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,7)-(1,8)) - │ │ └── name: :B - │ └── delimiter_loc: (1,5)-(1,7) = "::" + │ ├── name: :B + │ ├── delimiter_loc: (1,5)-(1,7) = "::" + │ └── name_loc: (1,7)-(1,8) = "B" ├── rest: ∅ ├── rights: (length: 0) ├── lparen_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt b/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt index 7a3e9affb5..edef23044a 100644 --- a/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt +++ b/test/prism/snapshots/seattlerb/messy_op_asgn_lineno.txt @@ -24,11 +24,10 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (1,3)-(1,4)) │ │ │ │ └── name: :B - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (1,6)-(1,7)) - │ │ │ │ └── name: :C - │ │ │ └── delimiter_loc: (1,4)-(1,6) = "::" - │ │ ├── operator_loc: (1,8)-(1,10) = "*=" + │ │ │ ├── name: :C + │ │ │ ├── delimiter_loc: (1,4)-(1,6) = "::" + │ │ │ └── name_loc: (1,6)-(1,7) = "C" + │ │ ├── binary_operator_loc: (1,8)-(1,10) = "*=" │ │ ├── value: │ │ │ @ CallNode (location: (1,11)-(1,14)) │ │ │ ├── flags: ignore_visibility @@ -53,7 +52,7 @@ │ │ │ │ └── block: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ - │ │ └── operator: :* + │ │ └── binary_operator: :* │ ├── opening_loc: (1,2)-(1,3) = "(" │ └── closing_loc: (1,14)-(1,15) = ")" ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/method_call_assoc_trailing_comma.txt b/test/prism/snapshots/seattlerb/method_call_assoc_trailing_comma.txt index da10a474c3..1bb8bd0bc1 100644 --- a/test/prism/snapshots/seattlerb/method_call_assoc_trailing_comma.txt +++ b/test/prism/snapshots/seattlerb/method_call_assoc_trailing_comma.txt @@ -22,7 +22,7 @@ ├── opening_loc: (1,3)-(1,4) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,8)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,4)-(1,8)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/multiline_hash_declaration.txt b/test/prism/snapshots/seattlerb/multiline_hash_declaration.txt index 79b0ef5d23..ff28a1798b 100644 --- a/test/prism/snapshots/seattlerb/multiline_hash_declaration.txt +++ b/test/prism/snapshots/seattlerb/multiline_hash_declaration.txt @@ -12,7 +12,7 @@ │ ├── opening_loc: (1,1)-(1,2) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,2)-(3,1)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,2)-(3,1)) │ │ ├── flags: symbol_keys @@ -42,7 +42,7 @@ │ ├── opening_loc: (5,1)-(5,2) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (5,2)-(6,1)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (5,2)-(6,1)) │ │ ├── flags: symbol_keys @@ -72,7 +72,7 @@ ├── opening_loc: (8,1)-(8,2) = "(" ├── arguments: │ @ ArgumentsNode (location: (8,2)-(8,11)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (8,2)-(8,11)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt index 8e6df3e812..523ccde455 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt @@ -9,11 +9,10 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (1,0)-(1,1)) │ │ └── name: :A - │ ├── child: - │ │ @ ConstantReadNode (location: (1,3)-(1,4)) - │ │ └── name: :B - │ └── delimiter_loc: (1,1)-(1,3) = "::" - ├── operator_loc: (1,5)-(1,7) = "*=" + │ ├── name: :B + │ ├── delimiter_loc: (1,1)-(1,3) = "::" + │ └── name_loc: (1,3)-(1,4) = "B" + ├── binary_operator_loc: (1,5)-(1,7) = "*=" ├── value: │ @ CallNode (location: (1,8)-(1,11)) │ ├── flags: ignore_visibility @@ -38,4 +37,4 @@ │ │ └── block: ∅ │ ├── closing_loc: ∅ │ └── block: ∅ - └── operator: :* + └── binary_operator: :* diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt index 0daadcf6ff..b9d00edc30 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier1.txt @@ -12,8 +12,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :+ - ├── operator_loc: (1,5)-(1,7) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,5)-(1,7) = "+=" └── value: @ IntegerNode (location: (1,8)-(1,9)) ├── flags: decimal diff --git a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt index ea8603165b..c12ea3983c 100644 --- a/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt +++ b/test/prism/snapshots/seattlerb/op_asgn_primary_colon_identifier_command_call.txt @@ -12,8 +12,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :* - ├── operator_loc: (1,5)-(1,7) = "*=" + ├── binary_operator: :* + ├── binary_operator_loc: (1,5)-(1,7) = "*=" └── value: @ CallNode (location: (1,8)-(1,11)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt b/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt index 8199eb2449..84eef70b25 100644 --- a/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt +++ b/test/prism/snapshots/seattlerb/parse_line_defn_complex.txt @@ -40,13 +40,13 @@ │ │ └── block: ∅ │ ├── @ LocalVariableOperatorWriteNode (location: (3,2)-(3,8)) │ │ ├── name_loc: (3,2)-(3,3) = "y" - │ │ ├── operator_loc: (3,4)-(3,6) = "*=" + │ │ ├── binary_operator_loc: (3,4)-(3,6) = "*=" │ │ ├── value: │ │ │ @ IntegerNode (location: (3,7)-(3,8)) │ │ │ ├── flags: decimal │ │ │ └── value: 2 │ │ ├── name: :y - │ │ ├── operator: :* + │ │ ├── binary_operator: :* │ │ └── depth: 0 │ └── @ ReturnNode (location: (4,2)-(4,10)) │ ├── flags: redundant diff --git a/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt b/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt index 5c2eb2da3c..d113f2af9d 100644 --- a/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt +++ b/test/prism/snapshots/seattlerb/parse_line_op_asgn.txt @@ -5,7 +5,7 @@ └── body: (length: 2) ├── @ LocalVariableOperatorWriteNode (location: (1,6)-(2,11)) │ ├── name_loc: (1,6)-(1,9) = "foo" - │ ├── operator_loc: (1,10)-(1,12) = "+=" + │ ├── binary_operator_loc: (1,10)-(1,12) = "+=" │ ├── value: │ │ @ CallNode (location: (2,8)-(2,11)) │ │ ├── flags: variable_call, ignore_visibility @@ -18,7 +18,7 @@ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ CallNode (location: (3,6)-(3,9)) ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/seattlerb/parse_opt_call_args_assocs_comma.txt b/test/prism/snapshots/seattlerb/parse_opt_call_args_assocs_comma.txt index b767a5c17e..dc11e2ca3d 100644 --- a/test/prism/snapshots/seattlerb/parse_opt_call_args_assocs_comma.txt +++ b/test/prism/snapshots/seattlerb/parse_opt_call_args_assocs_comma.txt @@ -15,7 +15,7 @@ ├── opening_loc: (1,1)-(1,2) = "[" ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,6)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,2)-(1,6)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/pct_w_heredoc_interp_nested.txt b/test/prism/snapshots/seattlerb/pct_w_heredoc_interp_nested.txt index c4bc53b723..1b8ec69b56 100644 --- a/test/prism/snapshots/seattlerb/pct_w_heredoc_interp_nested.txt +++ b/test/prism/snapshots/seattlerb/pct_w_heredoc_interp_nested.txt @@ -13,7 +13,7 @@ │ │ ├── closing_loc: ∅ │ │ └── unescaped: "1" │ ├── @ InterpolatedStringNode (location: (1,6)-(1,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: mutable │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 1) │ │ │ └── @ EmbeddedStatementsNode (location: (1,6)-(1,12)) @@ -22,7 +22,7 @@ │ │ │ │ @ StatementsNode (location: (1,8)-(1,11)) │ │ │ │ └── body: (length: 1) │ │ │ │ └── @ StringNode (location: (1,8)-(1,11)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (1,8)-(1,11) = "<<A" │ │ │ │ ├── content_loc: (2,0)-(3,0) = "2\n" │ │ │ │ ├── closing_loc: (3,0)-(4,0) = "A\n" diff --git a/test/prism/snapshots/seattlerb/quoted_symbol_hash_arg.txt b/test/prism/snapshots/seattlerb/quoted_symbol_hash_arg.txt index 64caf51bcb..bbc19d50ef 100644 --- a/test/prism/snapshots/seattlerb/quoted_symbol_hash_arg.txt +++ b/test/prism/snapshots/seattlerb/quoted_symbol_hash_arg.txt @@ -12,7 +12,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (1,5)-(1,12)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,5)-(1,12)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/seattlerb/return_call_assocs.txt b/test/prism/snapshots/seattlerb/return_call_assocs.txt index 66b3cb2f82..8948f7879b 100644 --- a/test/prism/snapshots/seattlerb/return_call_assocs.txt +++ b/test/prism/snapshots/seattlerb/return_call_assocs.txt @@ -8,7 +8,7 @@ │ ├── keyword_loc: (1,0)-(1,6) = "return" │ └── arguments: │ @ ArgumentsNode (location: (1,7)-(1,17)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ IntegerNode (location: (1,7)-(1,8)) │ │ ├── flags: decimal @@ -34,7 +34,7 @@ │ ├── keyword_loc: (3,0)-(3,6) = "return" │ └── arguments: │ @ ArgumentsNode (location: (3,7)-(3,26)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ IntegerNode (location: (3,7)-(3,8)) │ │ ├── flags: decimal @@ -84,7 +84,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (5,9)-(5,14)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (5,9)-(5,14)) │ │ ├── flags: symbol_keys @@ -120,7 +120,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (7,9)-(7,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (7,9)-(7,12)) │ │ ├── flags: symbol_keys @@ -156,7 +156,7 @@ │ ├── opening_loc: (9,8)-(9,9) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (9,9)-(9,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (9,9)-(9,12)) │ │ ├── flags: symbol_keys @@ -192,7 +192,7 @@ ├── opening_loc: (11,8)-(11,9) = "(" ├── arguments: │ @ ArgumentsNode (location: (11,9)-(11,13)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (11,9)-(11,13)) │ ├── flags: ∅ diff --git a/test/prism/snapshots/seattlerb/ruby21_numbers.txt b/test/prism/snapshots/seattlerb/ruby21_numbers.txt index 34a3452d1b..e7eec943f8 100644 --- a/test/prism/snapshots/seattlerb/ruby21_numbers.txt +++ b/test/prism/snapshots/seattlerb/ruby21_numbers.txt @@ -12,16 +12,14 @@ │ │ ├── flags: decimal │ │ └── value: 1 │ ├── @ RationalNode (location: (1,5)-(1,7)) - │ │ └── numeric: - │ │ @ IntegerNode (location: (1,5)-(1,6)) - │ │ ├── flags: decimal - │ │ └── value: 2 + │ │ ├── flags: decimal + │ │ ├── numerator: 2 + │ │ └── denominator: 1 │ └── @ ImaginaryNode (location: (1,9)-(1,12)) │ └── numeric: │ @ RationalNode (location: (1,9)-(1,11)) - │ └── numeric: - │ @ IntegerNode (location: (1,9)-(1,10)) - │ ├── flags: decimal - │ └── value: 3 + │ ├── flags: decimal + │ ├── numerator: 3 + │ └── denominator: 1 ├── opening_loc: (1,0)-(1,1) = "[" └── closing_loc: (1,12)-(1,13) = "]" diff --git a/test/prism/snapshots/seattlerb/safe_op_asgn.txt b/test/prism/snapshots/seattlerb/safe_op_asgn.txt index 7a9fd2b7f7..ebcedd6b5e 100644 --- a/test/prism/snapshots/seattlerb/safe_op_asgn.txt +++ b/test/prism/snapshots/seattlerb/safe_op_asgn.txt @@ -20,8 +20,8 @@ ├── message_loc: (1,3)-(1,4) = "b" ├── read_name: :b ├── write_name: :b= - ├── operator: :+ - ├── operator_loc: (1,5)-(1,7) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,5)-(1,7) = "+=" └── value: @ CallNode (location: (1,8)-(1,11)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/seattlerb/str_str.txt b/test/prism/snapshots/seattlerb/str_str.txt index f3f1213a0c..97031c8a65 100644 --- a/test/prism/snapshots/seattlerb/str_str.txt +++ b/test/prism/snapshots/seattlerb/str_str.txt @@ -4,7 +4,7 @@ @ StatementsNode (location: (1,0)-(1,10)) └── body: (length: 1) └── @ InterpolatedStringNode (location: (1,0)-(1,10)) - ├── flags: ∅ + ├── flags: mutable ├── opening_loc: (1,0)-(1,1) = "\"" ├── parts: (length: 2) │ ├── @ StringNode (location: (1,1)-(1,3)) @@ -19,7 +19,7 @@ │ │ @ StatementsNode (location: (1,5)-(1,8)) │ │ └── body: (length: 1) │ │ └── @ StringNode (location: (1,5)-(1,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (1,5)-(1,6) = "'" │ │ ├── content_loc: (1,6)-(1,7) = "b" │ │ ├── closing_loc: (1,7)-(1,8) = "'" diff --git a/test/prism/snapshots/seattlerb/str_str_str.txt b/test/prism/snapshots/seattlerb/str_str_str.txt index b01f2b5794..b592d380ef 100644 --- a/test/prism/snapshots/seattlerb/str_str_str.txt +++ b/test/prism/snapshots/seattlerb/str_str_str.txt @@ -4,7 +4,7 @@ @ StatementsNode (location: (1,0)-(1,12)) └── body: (length: 1) └── @ InterpolatedStringNode (location: (1,0)-(1,12)) - ├── flags: ∅ + ├── flags: mutable ├── opening_loc: (1,0)-(1,1) = "\"" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,3)) @@ -19,7 +19,7 @@ │ │ │ @ StatementsNode (location: (1,5)-(1,8)) │ │ │ └── body: (length: 1) │ │ │ └── @ StringNode (location: (1,5)-(1,8)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (1,5)-(1,6) = "'" │ │ │ ├── content_loc: (1,6)-(1,7) = "b" │ │ │ ├── closing_loc: (1,7)-(1,8) = "'" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index dbd3a4d030..48ff0d634f 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -146,10 +146,9 @@ │ │ ├── @ FloatNode (location: (29,4)-(29,7)) │ │ │ └── value: 1.0 │ │ ├── @ RationalNode (location: (29,9)-(29,11)) - │ │ │ └── numeric: - │ │ │ @ IntegerNode (location: (29,9)-(29,10)) - │ │ │ ├── flags: decimal - │ │ │ └── value: 1 + │ │ │ ├── flags: decimal + │ │ │ ├── numerator: 1 + │ │ │ └── denominator: 1 │ │ └── @ ImaginaryNode (location: (29,13)-(29,15)) │ │ └── numeric: │ │ @ IntegerNode (location: (29,13)-(29,14)) diff --git a/test/prism/snapshots/unparser/corpus/literal/assignment.txt b/test/prism/snapshots/unparser/corpus/literal/assignment.txt index 99c8daf34c..7d3cc389c6 100644 --- a/test/prism/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/prism/snapshots/unparser/corpus/literal/assignment.txt @@ -469,18 +469,16 @@ │ ├── target: │ │ @ ConstantPathNode (location: (18,0)-(18,5)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (18,2)-(18,5)) - │ │ │ └── name: :Foo - │ │ └── delimiter_loc: (18,0)-(18,2) = "::" + │ │ ├── name: :Foo + │ │ ├── delimiter_loc: (18,0)-(18,2) = "::" + │ │ └── name_loc: (18,2)-(18,5) = "Foo" │ ├── operator_loc: (18,6)-(18,7) = "=" │ └── value: │ @ ConstantPathNode (location: (18,8)-(18,13)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (18,10)-(18,13)) - │ │ └── name: :Bar - │ └── delimiter_loc: (18,8)-(18,10) = "::" + │ ├── name: :Bar + │ ├── delimiter_loc: (18,8)-(18,10) = "::" + │ └── name_loc: (18,10)-(18,13) = "Bar" ├── @ ClassVariableWriteNode (location: (19,0)-(19,7)) │ ├── name: :@@a │ ├── name_loc: (19,0)-(19,3) = "@@a" @@ -513,14 +511,12 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (22,0)-(22,4)) │ │ │ │ └── name: :Name - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (22,6)-(22,12)) - │ │ │ │ └── name: :Spaced - │ │ │ └── delimiter_loc: (22,4)-(22,6) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (22,14)-(22,19)) - │ │ │ └── name: :CONST - │ │ └── delimiter_loc: (22,12)-(22,14) = "::" + │ │ │ ├── name: :Spaced + │ │ │ ├── delimiter_loc: (22,4)-(22,6) = "::" + │ │ │ └── name_loc: (22,6)-(22,12) = "Spaced" + │ │ ├── name: :CONST + │ │ ├── delimiter_loc: (22,12)-(22,14) = "::" + │ │ └── name_loc: (22,14)-(22,19) = "CONST" │ ├── operator_loc: (22,20)-(22,21) = "=" │ └── value: │ @ IntegerNode (location: (22,22)-(22,23)) diff --git a/test/prism/snapshots/unparser/corpus/literal/class.txt b/test/prism/snapshots/unparser/corpus/literal/class.txt index 34eb03edb3..5306888398 100644 --- a/test/prism/snapshots/unparser/corpus/literal/class.txt +++ b/test/prism/snapshots/unparser/corpus/literal/class.txt @@ -68,10 +68,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (11,6)-(11,7)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (11,9)-(11,10)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (11,7)-(11,9) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (11,7)-(11,9) = "::" + │ │ └── name_loc: (11,9)-(11,10) = "B" │ ├── inheritance_operator_loc: ∅ │ ├── superclass: ∅ │ ├── body: ∅ @@ -87,14 +86,12 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (14,6)-(14,7)) │ │ │ │ └── name: :A - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (14,9)-(14,10)) - │ │ │ │ └── name: :B - │ │ │ └── delimiter_loc: (14,7)-(14,9) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (14,12)-(14,13)) - │ │ │ └── name: :C - │ │ └── delimiter_loc: (14,10)-(14,12) = "::" + │ │ │ ├── name: :B + │ │ │ ├── delimiter_loc: (14,7)-(14,9) = "::" + │ │ │ └── name_loc: (14,9)-(14,10) = "B" + │ │ ├── name: :C + │ │ ├── delimiter_loc: (14,10)-(14,12) = "::" + │ │ └── name_loc: (14,12)-(14,13) = "C" │ ├── inheritance_operator_loc: ∅ │ ├── superclass: ∅ │ ├── body: ∅ @@ -125,10 +122,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (20,10)-(20,11)) │ │ │ └── name: :B - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (20,13)-(20,14)) - │ │ │ └── name: :C - │ │ └── delimiter_loc: (20,11)-(20,13) = "::" + │ │ ├── name: :C + │ │ ├── delimiter_loc: (20,11)-(20,13) = "::" + │ │ └── name_loc: (20,13)-(20,14) = "C" │ ├── body: ∅ │ ├── end_keyword_loc: (21,0)-(21,3) = "end" │ └── name: :A @@ -140,20 +136,18 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (23,6)-(23,7)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (23,9)-(23,10)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (23,7)-(23,9) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (23,7)-(23,9) = "::" + │ │ └── name_loc: (23,9)-(23,10) = "B" │ ├── inheritance_operator_loc: (23,11)-(23,12) = "<" │ ├── superclass: │ │ @ ConstantPathNode (location: (23,13)-(23,17)) │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (23,13)-(23,14)) │ │ │ └── name: :C - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (23,16)-(23,17)) - │ │ │ └── name: :D - │ │ └── delimiter_loc: (23,14)-(23,16) = "::" + │ │ ├── name: :D + │ │ ├── delimiter_loc: (23,14)-(23,16) = "::" + │ │ └── name_loc: (23,16)-(23,17) = "D" │ ├── body: ∅ │ ├── end_keyword_loc: (24,0)-(24,3) = "end" │ └── name: :B @@ -222,10 +216,9 @@ ├── constant_path: │ @ ConstantPathNode (location: (34,6)-(34,9)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (34,8)-(34,9)) - │ │ └── name: :A - │ └── delimiter_loc: (34,6)-(34,8) = "::" + │ ├── name: :A + │ ├── delimiter_loc: (34,6)-(34,8) = "::" + │ └── name_loc: (34,8)-(34,9) = "A" ├── inheritance_operator_loc: ∅ ├── superclass: ∅ ├── body: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/defs.txt b/test/prism/snapshots/unparser/corpus/literal/defs.txt index 431843cc19..7858877172 100644 --- a/test/prism/snapshots/unparser/corpus/literal/defs.txt +++ b/test/prism/snapshots/unparser/corpus/literal/defs.txt @@ -225,10 +225,9 @@ │ │ │ │ ├── parent: │ │ │ │ │ @ ConstantReadNode (location: (26,5)-(26,8)) │ │ │ │ │ └── name: :Foo - │ │ │ │ ├── child: - │ │ │ │ │ @ ConstantReadNode (location: (26,10)-(26,13)) - │ │ │ │ │ └── name: :Bar - │ │ │ │ └── delimiter_loc: (26,8)-(26,10) = "::" + │ │ │ │ ├── name: :Bar + │ │ │ │ ├── delimiter_loc: (26,8)-(26,10) = "::" + │ │ │ │ └── name_loc: (26,10)-(26,13) = "Bar" │ │ │ ├── call_operator_loc: (26,13)-(26,14) = "." │ │ │ ├── name: :baz │ │ │ ├── message_loc: (26,14)-(26,17) = "baz" @@ -269,10 +268,9 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (30,5)-(30,8)) │ │ │ │ └── name: :Foo - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (30,10)-(30,13)) - │ │ │ │ └── name: :Bar - │ │ │ └── delimiter_loc: (30,8)-(30,10) = "::" + │ │ │ ├── name: :Bar + │ │ │ ├── delimiter_loc: (30,8)-(30,10) = "::" + │ │ │ └── name_loc: (30,10)-(30,13) = "Bar" │ │ ├── opening_loc: (30,4)-(30,5) = "(" │ │ └── closing_loc: (30,13)-(30,14) = ")" │ ├── parameters: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index dcf00bf4e0..ddb10456bf 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -316,18 +316,17 @@ │ ├── flags: decimal │ └── value: 1 ├── @ RationalNode (location: (19,0)-(19,2)) - │ └── numeric: - │ @ IntegerNode (location: (19,0)-(19,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (20,0)-(20,4)) - │ └── numeric: - │ @ FloatNode (location: (20,0)-(20,3)) - │ └── value: 1.5 + │ ├── flags: decimal + │ ├── numerator: 3 + │ └── denominator: 2 ├── @ RationalNode (location: (21,0)-(21,4)) - │ └── numeric: - │ @ FloatNode (location: (21,0)-(21,3)) - │ └── value: 1.3 + │ ├── flags: decimal + │ ├── numerator: 13 + │ └── denominator: 10 ├── @ ImaginaryNode (location: (22,0)-(22,2)) │ └── numeric: │ @ IntegerNode (location: (22,0)-(22,1)) @@ -354,10 +353,9 @@ ├── @ ImaginaryNode (location: (27,0)-(27,3)) │ └── numeric: │ @ RationalNode (location: (27,0)-(27,2)) - │ └── numeric: - │ @ IntegerNode (location: (27,0)-(27,1)) - │ ├── flags: decimal - │ └── value: 1 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ InterpolatedStringNode (location: (28,0)-(28,11)) │ ├── flags: ∅ │ ├── opening_loc: ∅ @@ -635,7 +633,7 @@ │ │ │ @ StatementsNode (location: (53,3)-(53,11)) │ │ │ └── body: (length: 1) │ │ │ └── @ StringNode (location: (53,3)-(53,11)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (53,3)-(53,4) = "\"" │ │ │ ├── content_loc: (53,4)-(53,10) = "\\u0000" │ │ │ ├── closing_loc: (53,10)-(53,11) = "\"" @@ -707,7 +705,7 @@ │ │ │ @ StatementsNode (location: (59,4)-(59,9)) │ │ │ └── body: (length: 1) │ │ │ └── @ StringNode (location: (59,4)-(59,9)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (59,4)-(59,5) = "\"" │ │ │ ├── content_loc: (59,5)-(59,8) = "foo" │ │ │ ├── closing_loc: (59,8)-(59,9) = "\"" diff --git a/test/prism/snapshots/unparser/corpus/literal/module.txt b/test/prism/snapshots/unparser/corpus/literal/module.txt index 5dd8c03b51..6428aeea82 100644 --- a/test/prism/snapshots/unparser/corpus/literal/module.txt +++ b/test/prism/snapshots/unparser/corpus/literal/module.txt @@ -20,10 +20,9 @@ │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (4,7)-(4,8)) │ │ │ └── name: :A - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (4,10)-(4,11)) - │ │ │ └── name: :B - │ │ └── delimiter_loc: (4,8)-(4,10) = "::" + │ │ ├── name: :B + │ │ ├── delimiter_loc: (4,8)-(4,10) = "::" + │ │ └── name_loc: (4,10)-(4,11) = "B" │ ├── body: ∅ │ ├── end_keyword_loc: (5,0)-(5,3) = "end" │ └── name: :B @@ -37,14 +36,12 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (7,7)-(7,8)) │ │ │ │ └── name: :A - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (7,10)-(7,11)) - │ │ │ │ └── name: :B - │ │ │ └── delimiter_loc: (7,8)-(7,10) = "::" - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (7,13)-(7,14)) - │ │ │ └── name: :C - │ │ └── delimiter_loc: (7,11)-(7,13) = "::" + │ │ │ ├── name: :B + │ │ │ ├── delimiter_loc: (7,8)-(7,10) = "::" + │ │ │ └── name_loc: (7,10)-(7,11) = "B" + │ │ ├── name: :C + │ │ ├── delimiter_loc: (7,11)-(7,13) = "::" + │ │ └── name_loc: (7,13)-(7,14) = "C" │ ├── body: ∅ │ ├── end_keyword_loc: (8,0)-(8,3) = "end" │ └── name: :C diff --git a/test/prism/snapshots/unparser/corpus/literal/opasgn.txt b/test/prism/snapshots/unparser/corpus/literal/opasgn.txt index 8dc0849638..0761b47348 100644 --- a/test/prism/snapshots/unparser/corpus/literal/opasgn.txt +++ b/test/prism/snapshots/unparser/corpus/literal/opasgn.txt @@ -5,53 +5,53 @@ └── body: (length: 24) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,6)) │ ├── name_loc: (1,0)-(1,1) = "a" - │ ├── operator_loc: (1,2)-(1,4) = "+=" + │ ├── binary_operator_loc: (1,2)-(1,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (1,5)-(1,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (2,0)-(2,6)) │ ├── name_loc: (2,0)-(2,1) = "a" - │ ├── operator_loc: (2,2)-(2,4) = "-=" + │ ├── binary_operator_loc: (2,2)-(2,4) = "-=" │ ├── value: │ │ @ IntegerNode (location: (2,5)-(2,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :- + │ ├── binary_operator: :- │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,7)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,5) = "**=" + │ ├── binary_operator_loc: (3,2)-(3,5) = "**=" │ ├── value: │ │ @ IntegerNode (location: (3,6)-(3,7)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :** + │ ├── binary_operator: :** │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (4,0)-(4,6)) │ ├── name_loc: (4,0)-(4,1) = "a" - │ ├── operator_loc: (4,2)-(4,4) = "*=" + │ ├── binary_operator_loc: (4,2)-(4,4) = "*=" │ ├── value: │ │ @ IntegerNode (location: (4,5)-(4,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :* + │ ├── binary_operator: :* │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (5,0)-(5,6)) │ ├── name_loc: (5,0)-(5,1) = "a" - │ ├── operator_loc: (5,2)-(5,4) = "/=" + │ ├── binary_operator_loc: (5,2)-(5,4) = "/=" │ ├── value: │ │ @ IntegerNode (location: (5,5)-(5,6)) │ │ ├── flags: decimal │ │ └── value: 2 │ ├── name: :a - │ ├── operator: :/ + │ ├── binary_operator: :/ │ └── depth: 0 ├── @ LocalVariableAndWriteNode (location: (6,0)-(6,7)) │ ├── name_loc: (6,0)-(6,1) = "a" @@ -162,8 +162,8 @@ │ ├── message_loc: (10,2)-(10,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :+ - │ ├── operator_loc: (10,4)-(10,6) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (10,4)-(10,6) = "+=" │ └── value: │ @ IntegerNode (location: (10,7)-(10,8)) │ ├── flags: decimal @@ -178,8 +178,8 @@ │ ├── message_loc: (11,2)-(11,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :- - │ ├── operator_loc: (11,4)-(11,6) = "-=" + │ ├── binary_operator: :- + │ ├── binary_operator_loc: (11,4)-(11,6) = "-=" │ └── value: │ @ IntegerNode (location: (11,7)-(11,8)) │ ├── flags: decimal @@ -194,8 +194,8 @@ │ ├── message_loc: (12,2)-(12,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :** - │ ├── operator_loc: (12,4)-(12,7) = "**=" + │ ├── binary_operator: :** + │ ├── binary_operator_loc: (12,4)-(12,7) = "**=" │ └── value: │ @ IntegerNode (location: (12,8)-(12,9)) │ ├── flags: decimal @@ -210,8 +210,8 @@ │ ├── message_loc: (13,2)-(13,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :* - │ ├── operator_loc: (13,4)-(13,6) = "*=" + │ ├── binary_operator: :* + │ ├── binary_operator_loc: (13,4)-(13,6) = "*=" │ └── value: │ @ IntegerNode (location: (13,7)-(13,8)) │ ├── flags: decimal @@ -226,8 +226,8 @@ │ ├── message_loc: (14,2)-(14,3) = "b" │ ├── read_name: :b │ ├── write_name: :b= - │ ├── operator: :/ - │ ├── operator_loc: (14,4)-(14,6) = "/=" + │ ├── binary_operator: :/ + │ ├── binary_operator_loc: (14,4)-(14,6) = "/=" │ └── value: │ @ IntegerNode (location: (14,7)-(14,8)) │ ├── flags: decimal @@ -293,8 +293,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (17,3)-(17,4) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (17,5)-(17,7) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (17,5)-(17,7) = "+=" │ └── value: │ @ IntegerNode (location: (17,8)-(17,9)) │ ├── flags: decimal @@ -323,8 +323,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (18,3)-(18,4) = "]" │ ├── block: ∅ - │ ├── operator: :- - │ ├── operator_loc: (18,5)-(18,7) = "-=" + │ ├── binary_operator: :- + │ ├── binary_operator_loc: (18,5)-(18,7) = "-=" │ └── value: │ @ IntegerNode (location: (18,8)-(18,9)) │ ├── flags: decimal @@ -353,8 +353,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (19,3)-(19,4) = "]" │ ├── block: ∅ - │ ├── operator: :** - │ ├── operator_loc: (19,5)-(19,8) = "**=" + │ ├── binary_operator: :** + │ ├── binary_operator_loc: (19,5)-(19,8) = "**=" │ └── value: │ @ IntegerNode (location: (19,9)-(19,10)) │ ├── flags: decimal @@ -383,8 +383,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (20,3)-(20,4) = "]" │ ├── block: ∅ - │ ├── operator: :* - │ ├── operator_loc: (20,5)-(20,7) = "*=" + │ ├── binary_operator: :* + │ ├── binary_operator_loc: (20,5)-(20,7) = "*=" │ └── value: │ @ IntegerNode (location: (20,8)-(20,9)) │ ├── flags: decimal @@ -413,8 +413,8 @@ │ │ └── block: ∅ │ ├── closing_loc: (21,3)-(21,4) = "]" │ ├── block: ∅ - │ ├── operator: :/ - │ ├── operator_loc: (21,5)-(21,7) = "/=" + │ ├── binary_operator: :/ + │ ├── binary_operator_loc: (21,5)-(21,7) = "/=" │ └── value: │ @ IntegerNode (location: (21,8)-(21,9)) │ ├── flags: decimal @@ -501,8 +501,8 @@ ├── message_loc: (24,4)-(24,5) = "A" ├── read_name: :A ├── write_name: :A= - ├── operator: :+ - ├── operator_loc: (24,6)-(24,8) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (24,6)-(24,8) = "+=" └── value: @ IntegerNode (location: (24,9)-(24,10)) ├── flags: decimal diff --git a/test/prism/snapshots/unparser/corpus/literal/send.txt b/test/prism/snapshots/unparser/corpus/literal/send.txt index b7eb064717..3fd7f719a1 100644 --- a/test/prism/snapshots/unparser/corpus/literal/send.txt +++ b/test/prism/snapshots/unparser/corpus/literal/send.txt @@ -1251,7 +1251,7 @@ │ ├── opening_loc: (63,7)-(63,8) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (63,8)-(63,16)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (63,8)-(63,16)) │ │ ├── flags: symbol_keys @@ -1297,7 +1297,7 @@ │ ├── opening_loc: (64,7)-(64,8) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (64,8)-(64,25)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ CallNode (location: (64,8)-(64,11)) │ │ │ ├── flags: variable_call, ignore_visibility @@ -1571,7 +1571,7 @@ │ ├── opening_loc: (70,3)-(70,4) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (70,4)-(70,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (70,4)-(70,8)) │ │ ├── flags: symbol_keys @@ -1617,7 +1617,7 @@ │ ├── opening_loc: (71,5)-(71,6) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (71,6)-(71,10)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (71,6)-(71,10)) │ │ ├── flags: symbol_keys @@ -1663,7 +1663,7 @@ │ ├── opening_loc: (72,5)-(72,6) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (72,6)-(72,9)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (72,6)-(72,9)) │ │ ├── flags: ∅ @@ -2082,7 +2082,7 @@ │ ├── opening_loc: (81,1)-(81,2) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (81,2)-(81,7)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (81,2)-(81,7)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/since/32.txt b/test/prism/snapshots/unparser/corpus/literal/since/32.txt index e72be6d8b7..2b28be2fa8 100644 --- a/test/prism/snapshots/unparser/corpus/literal/since/32.txt +++ b/test/prism/snapshots/unparser/corpus/literal/since/32.txt @@ -36,7 +36,7 @@ │ │ ├── opening_loc: (2,5)-(2,6) = "(" │ │ ├── arguments: │ │ │ @ ArgumentsNode (location: (2,6)-(2,18)) - │ │ │ ├── flags: contains_keyword_splat + │ │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ │ └── arguments: (length: 2) │ │ │ ├── @ LocalVariableReadNode (location: (2,6)-(2,14)) │ │ │ │ ├── name: :argument diff --git a/test/prism/snapshots/unparser/corpus/literal/variables.txt b/test/prism/snapshots/unparser/corpus/literal/variables.txt index bf79de385c..9ae8ad1207 100644 --- a/test/prism/snapshots/unparser/corpus/literal/variables.txt +++ b/test/prism/snapshots/unparser/corpus/literal/variables.txt @@ -29,25 +29,21 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (8,0)-(8,6)) │ │ └── name: :SCOPED - │ ├── child: - │ │ @ ConstantReadNode (location: (8,8)-(8,13)) - │ │ └── name: :CONST - │ └── delimiter_loc: (8,6)-(8,8) = "::" + │ ├── name: :CONST + │ ├── delimiter_loc: (8,6)-(8,8) = "::" + │ └── name_loc: (8,8)-(8,13) = "CONST" ├── @ ConstantPathNode (location: (9,0)-(9,10)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (9,2)-(9,10)) - │ │ └── name: :TOPLEVEL - │ └── delimiter_loc: (9,0)-(9,2) = "::" + │ ├── name: :TOPLEVEL + │ ├── delimiter_loc: (9,0)-(9,2) = "::" + │ └── name_loc: (9,2)-(9,10) = "TOPLEVEL" └── @ ConstantPathNode (location: (10,0)-(10,17)) ├── parent: │ @ ConstantPathNode (location: (10,0)-(10,10)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (10,2)-(10,10)) - │ │ └── name: :TOPLEVEL - │ └── delimiter_loc: (10,0)-(10,2) = "::" - ├── child: - │ @ ConstantReadNode (location: (10,12)-(10,17)) - │ └── name: :CONST - └── delimiter_loc: (10,10)-(10,12) = "::" + │ ├── name: :TOPLEVEL + │ ├── delimiter_loc: (10,0)-(10,2) = "::" + │ └── name_loc: (10,2)-(10,10) = "TOPLEVEL" + ├── name: :CONST + ├── delimiter_loc: (10,10)-(10,12) = "::" + └── name_loc: (10,12)-(10,17) = "CONST" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index ef666890be..448207f5d3 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -4,14 +4,13 @@ @ StatementsNode (location: (1,0)-(14,10)) └── body: (length: 14) ├── @ RationalNode (location: (1,0)-(1,4)) - │ └── numeric: - │ @ FloatNode (location: (1,0)-(1,3)) - │ └── value: 1.0 + │ ├── flags: decimal + │ ├── numerator: 1 + │ └── denominator: 1 ├── @ RationalNode (location: (2,0)-(2,3)) - │ └── numeric: - │ @ IntegerNode (location: (2,0)-(2,2)) - │ ├── flags: decimal - │ └── value: 0 + │ ├── flags: decimal + │ ├── numerator: 0 + │ └── denominator: 1 ├── @ IntegerNode (location: (3,0)-(3,3)) │ ├── flags: hexadecimal │ └── value: 1 diff --git a/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt b/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt index e100dd8ecb..7dd26a38dc 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/opasgn.txt @@ -44,8 +44,8 @@ │ └── closing_loc: (1,10)-(1,11) = "\"" ├── closing_loc: (1,11)-(1,12) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,13)-(1,15) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,13)-(1,15) = "+=" └── value: @ InterpolatedStringNode (location: (1,16)-(1,25)) ├── flags: ∅ diff --git a/test/prism/snapshots/whitequark/args_args_assocs.txt b/test/prism/snapshots/whitequark/args_args_assocs.txt index 6297f212d8..d257a885ce 100644 --- a/test/prism/snapshots/whitequark/args_args_assocs.txt +++ b/test/prism/snapshots/whitequark/args_args_assocs.txt @@ -12,7 +12,7 @@ │ ├── opening_loc: (1,3)-(1,4) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,4)-(1,18)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 2) │ │ ├── @ CallNode (location: (1,4)-(1,7)) │ │ │ ├── flags: variable_call, ignore_visibility @@ -51,7 +51,7 @@ ├── opening_loc: (3,3)-(3,4) = "(" ├── arguments: │ @ ArgumentsNode (location: (3,4)-(3,18)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ CallNode (location: (3,4)-(3,7)) │ │ ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/whitequark/args_args_assocs_comma.txt b/test/prism/snapshots/whitequark/args_args_assocs_comma.txt index 969514a511..2d986dd90a 100644 --- a/test/prism/snapshots/whitequark/args_args_assocs_comma.txt +++ b/test/prism/snapshots/whitequark/args_args_assocs_comma.txt @@ -22,7 +22,7 @@ ├── opening_loc: (1,3)-(1,4) = "[" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,18)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ CallNode (location: (1,4)-(1,7)) │ │ ├── flags: variable_call, ignore_visibility diff --git a/test/prism/snapshots/whitequark/args_assocs_comma.txt b/test/prism/snapshots/whitequark/args_assocs_comma.txt index b1b9fbeefe..64a25bbc45 100644 --- a/test/prism/snapshots/whitequark/args_assocs_comma.txt +++ b/test/prism/snapshots/whitequark/args_assocs_comma.txt @@ -22,7 +22,7 @@ ├── opening_loc: (1,3)-(1,4) = "[" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,13)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,4)-(1,13)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/whitequark/bug_cmdarg.txt b/test/prism/snapshots/whitequark/bug_cmdarg.txt index 509dd7e818..32d05746a7 100644 --- a/test/prism/snapshots/whitequark/bug_cmdarg.txt +++ b/test/prism/snapshots/whitequark/bug_cmdarg.txt @@ -12,7 +12,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (1,7)-(1,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,7)-(1,15)) │ │ ├── flags: symbol_keys @@ -62,7 +62,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (5,2)-(5,26)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (5,2)-(5,26)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/whitequark/casgn_scoped.txt b/test/prism/snapshots/whitequark/casgn_scoped.txt index 4e3fd6fe44..42b90be061 100644 --- a/test/prism/snapshots/whitequark/casgn_scoped.txt +++ b/test/prism/snapshots/whitequark/casgn_scoped.txt @@ -9,10 +9,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (1,0)-(1,3)) │ │ └── name: :Bar - │ ├── child: - │ │ @ ConstantReadNode (location: (1,5)-(1,8)) - │ │ └── name: :Foo - │ └── delimiter_loc: (1,3)-(1,5) = "::" + │ ├── name: :Foo + │ ├── delimiter_loc: (1,3)-(1,5) = "::" + │ └── name_loc: (1,5)-(1,8) = "Foo" ├── operator_loc: (1,9)-(1,10) = "=" └── value: @ IntegerNode (location: (1,11)-(1,13)) diff --git a/test/prism/snapshots/whitequark/casgn_toplevel.txt b/test/prism/snapshots/whitequark/casgn_toplevel.txt index 11facfefb3..070d90a46b 100644 --- a/test/prism/snapshots/whitequark/casgn_toplevel.txt +++ b/test/prism/snapshots/whitequark/casgn_toplevel.txt @@ -7,10 +7,9 @@ ├── target: │ @ ConstantPathNode (location: (1,0)-(1,5)) │ ├── parent: ∅ - │ ├── child: - │ │ @ ConstantReadNode (location: (1,2)-(1,5)) - │ │ └── name: :Foo - │ └── delimiter_loc: (1,0)-(1,2) = "::" + │ ├── name: :Foo + │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ └── name_loc: (1,2)-(1,5) = "Foo" ├── operator_loc: (1,6)-(1,7) = "=" └── value: @ IntegerNode (location: (1,8)-(1,10)) diff --git a/test/prism/snapshots/whitequark/complex.txt b/test/prism/snapshots/whitequark/complex.txt index e688585a5f..bc748db09b 100644 --- a/test/prism/snapshots/whitequark/complex.txt +++ b/test/prism/snapshots/whitequark/complex.txt @@ -10,9 +10,9 @@ ├── @ ImaginaryNode (location: (3,0)-(3,6)) │ └── numeric: │ @ RationalNode (location: (3,0)-(3,5)) - │ └── numeric: - │ @ FloatNode (location: (3,0)-(3,4)) - │ └── value: 42.1 + │ ├── flags: decimal + │ ├── numerator: 421 + │ └── denominator: 10 ├── @ ImaginaryNode (location: (5,0)-(5,3)) │ └── numeric: │ @ IntegerNode (location: (5,0)-(5,2)) @@ -21,7 +21,6 @@ └── @ ImaginaryNode (location: (7,0)-(7,4)) └── numeric: @ RationalNode (location: (7,0)-(7,3)) - └── numeric: - @ IntegerNode (location: (7,0)-(7,2)) - ├── flags: decimal - └── value: 42 + ├── flags: decimal + ├── numerator: 42 + └── denominator: 1 diff --git a/test/prism/snapshots/whitequark/const_op_asgn.txt b/test/prism/snapshots/whitequark/const_op_asgn.txt index 4985f3e54b..71df6208d2 100644 --- a/test/prism/snapshots/whitequark/const_op_asgn.txt +++ b/test/prism/snapshots/whitequark/const_op_asgn.txt @@ -7,41 +7,39 @@ │ ├── target: │ │ @ ConstantPathNode (location: (1,0)-(1,3)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (1,0)-(1,2) = "::" - │ ├── operator_loc: (1,4)-(1,6) = "+=" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ │ └── name_loc: (1,2)-(1,3) = "A" + │ ├── binary_operator_loc: (1,4)-(1,6) = "+=" │ ├── value: │ │ @ IntegerNode (location: (1,7)-(1,8)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ ConstantOperatorWriteNode (location: (3,0)-(3,6)) │ ├── name: :A │ ├── name_loc: (3,0)-(3,1) = "A" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (3,5)-(3,6)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ ConstantPathOperatorWriteNode (location: (5,0)-(5,9)) │ ├── target: │ │ @ ConstantPathNode (location: (5,0)-(5,4)) │ │ ├── parent: │ │ │ @ ConstantReadNode (location: (5,0)-(5,1)) │ │ │ └── name: :B - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (5,3)-(5,4)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (5,1)-(5,3) = "::" - │ ├── operator_loc: (5,5)-(5,7) = "+=" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (5,1)-(5,3) = "::" + │ │ └── name_loc: (5,3)-(5,4) = "A" + │ ├── binary_operator_loc: (5,5)-(5,7) = "+=" │ ├── value: │ │ @ IntegerNode (location: (5,8)-(5,9)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :+ + │ └── binary_operator: :+ ├── @ DefNode (location: (7,0)-(7,21)) │ ├── name: :x │ ├── name_loc: (7,4)-(7,5) = "x" @@ -54,10 +52,9 @@ │ │ ├── target: │ │ │ @ ConstantPathNode (location: (7,7)-(7,10)) │ │ │ ├── parent: ∅ - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (7,9)-(7,10)) - │ │ │ │ └── name: :A - │ │ │ └── delimiter_loc: (7,7)-(7,9) = "::" + │ │ │ ├── name: :A + │ │ │ ├── delimiter_loc: (7,7)-(7,9) = "::" + │ │ │ └── name_loc: (7,9)-(7,10) = "A" │ │ ├── operator_loc: (7,11)-(7,14) = "||=" │ │ └── value: │ │ @ IntegerNode (location: (7,15)-(7,16)) @@ -83,10 +80,9 @@ │ │ @ ConstantPathNode (location: (9,7)-(9,14)) │ │ ├── parent: │ │ │ @ SelfNode (location: (9,7)-(9,11)) - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (9,13)-(9,14)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (9,11)-(9,13) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (9,11)-(9,13) = "::" + │ │ └── name_loc: (9,13)-(9,14) = "A" │ ├── operator_loc: (9,15)-(9,18) = "||=" │ └── value: │ @ IntegerNode (location: (9,19)-(9,20)) diff --git a/test/prism/snapshots/whitequark/const_scoped.txt b/test/prism/snapshots/whitequark/const_scoped.txt index 1e2bccef96..83af4b187b 100644 --- a/test/prism/snapshots/whitequark/const_scoped.txt +++ b/test/prism/snapshots/whitequark/const_scoped.txt @@ -7,7 +7,6 @@ ├── parent: │ @ ConstantReadNode (location: (1,0)-(1,3)) │ └── name: :Bar - ├── child: - │ @ ConstantReadNode (location: (1,5)-(1,8)) - │ └── name: :Foo - └── delimiter_loc: (1,3)-(1,5) = "::" + ├── name: :Foo + ├── delimiter_loc: (1,3)-(1,5) = "::" + └── name_loc: (1,5)-(1,8) = "Foo" diff --git a/test/prism/snapshots/whitequark/const_toplevel.txt b/test/prism/snapshots/whitequark/const_toplevel.txt index b54b069d06..3d7df5defc 100644 --- a/test/prism/snapshots/whitequark/const_toplevel.txt +++ b/test/prism/snapshots/whitequark/const_toplevel.txt @@ -5,7 +5,6 @@ └── body: (length: 1) └── @ ConstantPathNode (location: (1,0)-(1,5)) ├── parent: ∅ - ├── child: - │ @ ConstantReadNode (location: (1,2)-(1,5)) - │ └── name: :Foo - └── delimiter_loc: (1,0)-(1,2) = "::" + ├── name: :Foo + ├── delimiter_loc: (1,0)-(1,2) = "::" + └── name_loc: (1,2)-(1,5) = "Foo" diff --git a/test/prism/snapshots/whitequark/cpath.txt b/test/prism/snapshots/whitequark/cpath.txt index b892525646..e801456bf7 100644 --- a/test/prism/snapshots/whitequark/cpath.txt +++ b/test/prism/snapshots/whitequark/cpath.txt @@ -9,10 +9,9 @@ │ ├── constant_path: │ │ @ ConstantPathNode (location: (1,7)-(1,12)) │ │ ├── parent: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (1,9)-(1,12)) - │ │ │ └── name: :Foo - │ │ └── delimiter_loc: (1,7)-(1,9) = "::" + │ │ ├── name: :Foo + │ │ ├── delimiter_loc: (1,7)-(1,9) = "::" + │ │ └── name_loc: (1,9)-(1,12) = "Foo" │ ├── body: ∅ │ ├── end_keyword_loc: (1,14)-(1,17) = "end" │ └── name: :Foo @@ -24,10 +23,9 @@ │ ├── parent: │ │ @ ConstantReadNode (location: (3,7)-(3,10)) │ │ └── name: :Bar - │ ├── child: - │ │ @ ConstantReadNode (location: (3,12)-(3,15)) - │ │ └── name: :Foo - │ └── delimiter_loc: (3,10)-(3,12) = "::" + │ ├── name: :Foo + │ ├── delimiter_loc: (3,10)-(3,12) = "::" + │ └── name_loc: (3,12)-(3,15) = "Foo" ├── body: ∅ ├── end_keyword_loc: (3,17)-(3,20) = "end" └── name: :Foo diff --git a/test/prism/snapshots/whitequark/dedenting_heredoc.txt b/test/prism/snapshots/whitequark/dedenting_heredoc.txt index acb79e83d2..67896b2415 100644 --- a/test/prism/snapshots/whitequark/dedenting_heredoc.txt +++ b/test/prism/snapshots/whitequark/dedenting_heredoc.txt @@ -15,7 +15,7 @@ │ │ ├── flags: ∅ │ │ └── arguments: (length: 1) │ │ └── @ InterpolatedStringNode (location: (1,2)-(1,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: mutable │ │ ├── opening_loc: (1,2)-(1,8) = "<<~\"E\"" │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (2,0)-(3,0)) @@ -30,7 +30,7 @@ │ │ │ │ │ @ StatementsNode (location: (3,4)-(3,9)) │ │ │ │ │ └── body: (length: 1) │ │ │ │ │ └── @ StringNode (location: (3,4)-(3,9)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: (3,4)-(3,5) = "\"" │ │ │ │ │ ├── content_loc: (3,5)-(3,8) = " y" │ │ │ │ │ ├── closing_loc: (3,8)-(3,9) = "\"" diff --git a/test/prism/snapshots/whitequark/forwarded_argument_with_kwrestarg.txt b/test/prism/snapshots/whitequark/forwarded_argument_with_kwrestarg.txt index c06818a98b..acaf9c052d 100644 --- a/test/prism/snapshots/whitequark/forwarded_argument_with_kwrestarg.txt +++ b/test/prism/snapshots/whitequark/forwarded_argument_with_kwrestarg.txt @@ -36,7 +36,7 @@ │ ├── opening_loc: (1,26)-(1,27) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,27)-(1,39)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 2) │ │ ├── @ LocalVariableReadNode (location: (1,27)-(1,35)) │ │ │ ├── name: :argument diff --git a/test/prism/snapshots/whitequark/forwarded_kwrestarg.txt b/test/prism/snapshots/whitequark/forwarded_kwrestarg.txt index adaf111fd7..b4235fb20a 100644 --- a/test/prism/snapshots/whitequark/forwarded_kwrestarg.txt +++ b/test/prism/snapshots/whitequark/forwarded_kwrestarg.txt @@ -33,7 +33,7 @@ │ ├── opening_loc: (1,16)-(1,17) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,17)-(1,19)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,17)-(1,19)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/whitequark/forwarded_kwrestarg_with_additional_kwarg.txt b/test/prism/snapshots/whitequark/forwarded_kwrestarg_with_additional_kwarg.txt index a03421455c..33779abcc1 100644 --- a/test/prism/snapshots/whitequark/forwarded_kwrestarg_with_additional_kwarg.txt +++ b/test/prism/snapshots/whitequark/forwarded_kwrestarg_with_additional_kwarg.txt @@ -33,7 +33,7 @@ │ ├── opening_loc: (1,16)-(1,17) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,17)-(1,35)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,17)-(1,35)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/whitequark/keyword_argument_omission.txt b/test/prism/snapshots/whitequark/keyword_argument_omission.txt index 62e8fecf4e..446b45b56b 100644 --- a/test/prism/snapshots/whitequark/keyword_argument_omission.txt +++ b/test/prism/snapshots/whitequark/keyword_argument_omission.txt @@ -12,7 +12,7 @@ ├── opening_loc: (1,3)-(1,4) = "(" ├── arguments: │ @ ArgumentsNode (location: (1,4)-(1,10)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,4)-(1,10)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/whitequark/kwoptarg_with_kwrestarg_and_forwarded_args.txt b/test/prism/snapshots/whitequark/kwoptarg_with_kwrestarg_and_forwarded_args.txt index 6d0bdfb817..db281e2f0d 100644 --- a/test/prism/snapshots/whitequark/kwoptarg_with_kwrestarg_and_forwarded_args.txt +++ b/test/prism/snapshots/whitequark/kwoptarg_with_kwrestarg_and_forwarded_args.txt @@ -39,7 +39,7 @@ │ ├── opening_loc: (1,20)-(1,21) = "(" │ ├── arguments: │ │ @ ArgumentsNode (location: (1,21)-(1,23)) - │ │ ├── flags: contains_keyword_splat + │ │ ├── flags: contains_keywords, contains_keyword_splat │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,21)-(1,23)) │ │ ├── flags: ∅ diff --git a/test/prism/snapshots/whitequark/masgn_const.txt b/test/prism/snapshots/whitequark/masgn_const.txt index 56169deb17..ddfccb0d8b 100644 --- a/test/prism/snapshots/whitequark/masgn_const.txt +++ b/test/prism/snapshots/whitequark/masgn_const.txt @@ -7,10 +7,9 @@ │ ├── lefts: (length: 2) │ │ ├── @ ConstantPathTargetNode (location: (1,0)-(1,3)) │ │ │ ├── parent: ∅ - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (1,2)-(1,3)) - │ │ │ │ └── name: :A - │ │ │ └── delimiter_loc: (1,0)-(1,2) = "::" + │ │ │ ├── name: :A + │ │ │ ├── delimiter_loc: (1,0)-(1,2) = "::" + │ │ │ └── name_loc: (1,2)-(1,3) = "A" │ │ └── @ LocalVariableTargetNode (location: (1,5)-(1,8)) │ │ ├── name: :foo │ │ └── depth: 0 @@ -28,10 +27,9 @@ │ ├── @ ConstantPathTargetNode (location: (3,0)-(3,7)) │ │ ├── parent: │ │ │ @ SelfNode (location: (3,0)-(3,4)) - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (3,6)-(3,7)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (3,4)-(3,6) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (3,4)-(3,6) = "::" + │ │ └── name_loc: (3,6)-(3,7) = "A" │ └── @ LocalVariableTargetNode (location: (3,9)-(3,12)) │ ├── name: :foo │ └── depth: 0 diff --git a/test/prism/snapshots/whitequark/newline_in_hash_argument.txt b/test/prism/snapshots/whitequark/newline_in_hash_argument.txt index d5e89d87f9..7ef006645b 100644 --- a/test/prism/snapshots/whitequark/newline_in_hash_argument.txt +++ b/test/prism/snapshots/whitequark/newline_in_hash_argument.txt @@ -102,7 +102,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (10,8)-(11,1)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (10,8)-(11,1)) │ │ ├── flags: symbol_keys @@ -141,7 +141,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (13,8)-(14,1)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (13,8)-(14,1)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/whitequark/op_asgn.txt b/test/prism/snapshots/whitequark/op_asgn.txt index 8f24a35ad0..f726617904 100644 --- a/test/prism/snapshots/whitequark/op_asgn.txt +++ b/test/prism/snapshots/whitequark/op_asgn.txt @@ -20,8 +20,8 @@ │ ├── message_loc: (1,4)-(1,5) = "A" │ ├── read_name: :A │ ├── write_name: :A= - │ ├── operator: :+ - │ ├── operator_loc: (1,6)-(1,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (1,6)-(1,8) = "+=" │ └── value: │ @ IntegerNode (location: (1,9)-(1,10)) │ ├── flags: decimal @@ -43,8 +43,8 @@ │ ├── message_loc: (3,4)-(3,5) = "a" │ ├── read_name: :a │ ├── write_name: :a= - │ ├── operator: :+ - │ ├── operator_loc: (3,6)-(3,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (3,6)-(3,8) = "+=" │ └── value: │ @ IntegerNode (location: (3,9)-(3,10)) │ ├── flags: decimal @@ -66,8 +66,8 @@ ├── message_loc: (5,5)-(5,6) = "a" ├── read_name: :a ├── write_name: :a= - ├── operator: :+ - ├── operator_loc: (5,7)-(5,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (5,7)-(5,9) = "+=" └── value: @ IntegerNode (location: (5,10)-(5,11)) ├── flags: decimal diff --git a/test/prism/snapshots/whitequark/op_asgn_cmd.txt b/test/prism/snapshots/whitequark/op_asgn_cmd.txt index 109c2fd2ed..d2d86b1bf9 100644 --- a/test/prism/snapshots/whitequark/op_asgn_cmd.txt +++ b/test/prism/snapshots/whitequark/op_asgn_cmd.txt @@ -20,8 +20,8 @@ │ ├── message_loc: (1,4)-(1,5) = "A" │ ├── read_name: :A │ ├── write_name: :A= - │ ├── operator: :+ - │ ├── operator_loc: (1,6)-(1,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (1,6)-(1,8) = "+=" │ └── value: │ @ CallNode (location: (1,9)-(1,14)) │ ├── flags: ignore_visibility @@ -63,8 +63,8 @@ │ ├── message_loc: (3,4)-(3,5) = "a" │ ├── read_name: :a │ ├── write_name: :a= - │ ├── operator: :+ - │ ├── operator_loc: (3,6)-(3,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (3,6)-(3,8) = "+=" │ └── value: │ @ CallNode (location: (3,9)-(3,14)) │ ├── flags: ignore_visibility @@ -103,11 +103,10 @@ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (5,5)-(5,6)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (5,3)-(5,5) = "::" - │ ├── operator_loc: (5,7)-(5,9) = "+=" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (5,3)-(5,5) = "::" + │ │ └── name_loc: (5,5)-(5,6) = "A" + │ ├── binary_operator_loc: (5,7)-(5,9) = "+=" │ ├── value: │ │ @ CallNode (location: (5,10)-(5,15)) │ │ ├── flags: ignore_visibility @@ -132,7 +131,7 @@ │ │ │ └── block: ∅ │ │ ├── closing_loc: ∅ │ │ └── block: ∅ - │ └── operator: :+ + │ └── binary_operator: :+ └── @ CallOperatorWriteNode (location: (7,0)-(7,15)) ├── flags: ∅ ├── receiver: @@ -150,8 +149,8 @@ ├── message_loc: (7,5)-(7,6) = "a" ├── read_name: :a ├── write_name: :a= - ├── operator: :+ - ├── operator_loc: (7,7)-(7,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (7,7)-(7,9) = "+=" └── value: @ CallNode (location: (7,10)-(7,15)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/whitequark/op_asgn_index.txt b/test/prism/snapshots/whitequark/op_asgn_index.txt index fe077fae13..b302abdafe 100644 --- a/test/prism/snapshots/whitequark/op_asgn_index.txt +++ b/test/prism/snapshots/whitequark/op_asgn_index.txt @@ -30,8 +30,8 @@ │ └── value: 1 ├── closing_loc: (1,8)-(1,9) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,10)-(1,12) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,10)-(1,12) = "+=" └── value: @ IntegerNode (location: (1,13)-(1,14)) ├── flags: decimal diff --git a/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt b/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt index 87082aad94..319ed1a51a 100644 --- a/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt +++ b/test/prism/snapshots/whitequark/op_asgn_index_cmd.txt @@ -30,8 +30,8 @@ │ └── value: 1 ├── closing_loc: (1,8)-(1,9) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (1,10)-(1,12) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (1,10)-(1,12) = "+=" └── value: @ CallNode (location: (1,13)-(1,18)) ├── flags: ignore_visibility diff --git a/test/prism/snapshots/whitequark/parser_bug_525.txt b/test/prism/snapshots/whitequark/parser_bug_525.txt index a69b8a207f..3a31a97cdc 100644 --- a/test/prism/snapshots/whitequark/parser_bug_525.txt +++ b/test/prism/snapshots/whitequark/parser_bug_525.txt @@ -12,7 +12,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (1,3)-(1,11)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 1) │ └── @ KeywordHashNode (location: (1,3)-(1,11)) │ ├── flags: symbol_keys diff --git a/test/prism/snapshots/whitequark/rational.txt b/test/prism/snapshots/whitequark/rational.txt index 90bbd17929..e8c8eed508 100644 --- a/test/prism/snapshots/whitequark/rational.txt +++ b/test/prism/snapshots/whitequark/rational.txt @@ -4,11 +4,10 @@ @ StatementsNode (location: (1,0)-(3,3)) └── body: (length: 2) ├── @ RationalNode (location: (1,0)-(1,5)) - │ └── numeric: - │ @ FloatNode (location: (1,0)-(1,4)) - │ └── value: 42.1 + │ ├── flags: decimal + │ ├── numerator: 421 + │ └── denominator: 10 └── @ RationalNode (location: (3,0)-(3,3)) - └── numeric: - @ IntegerNode (location: (3,0)-(3,2)) - ├── flags: decimal - └── value: 42 + ├── flags: decimal + ├── numerator: 42 + └── denominator: 1 diff --git a/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt b/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt index b269104f30..840e5a4fc0 100644 --- a/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt +++ b/test/prism/snapshots/whitequark/rescue_mod_op_assign.txt @@ -5,7 +5,7 @@ └── body: (length: 1) └── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,22)) ├── name_loc: (1,0)-(1,3) = "foo" - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator_loc: (1,4)-(1,6) = "+=" ├── value: │ @ RescueModifierNode (location: (1,7)-(1,22)) │ ├── expression: @@ -32,5 +32,5 @@ │ ├── closing_loc: ∅ │ └── block: ∅ ├── name: :foo - ├── operator: :+ + ├── binary_operator: :+ └── depth: 0 diff --git a/test/prism/snapshots/whitequark/ruby_bug_11380.txt b/test/prism/snapshots/whitequark/ruby_bug_11380.txt index b7a7ef9a98..d5ec10b06c 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11380.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11380.txt @@ -12,7 +12,7 @@ ├── opening_loc: ∅ ├── arguments: │ @ ArgumentsNode (location: (1,2)-(1,21)) - │ ├── flags: ∅ + │ ├── flags: contains_keywords │ └── arguments: (length: 2) │ ├── @ LambdaNode (location: (1,2)-(1,15)) │ │ ├── locals: [] diff --git a/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt b/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt index 93418e6448..831d57e30d 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11873_a.txt @@ -225,9 +225,9 @@ │ │ │ ├── closing_loc: (7,7)-(7,8) = ")" │ │ │ └── block: ∅ │ │ └── @ RationalNode (location: (7,10)-(7,14)) - │ │ └── numeric: - │ │ @ FloatNode (location: (7,10)-(7,13)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (7,15)-(7,21)) @@ -519,9 +519,9 @@ │ │ │ ├── closing_loc: (17,8)-(17,9) = ")" │ │ │ └── block: ∅ │ │ └── @ RationalNode (location: (17,11)-(17,15)) - │ │ └── numeric: - │ │ @ FloatNode (location: (17,11)-(17,14)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (17,16)-(17,22)) @@ -833,9 +833,9 @@ │ │ │ ├── opening_loc: (27,3)-(27,4) = "{" │ │ │ └── closing_loc: (27,7)-(27,8) = "}" │ │ └── @ RationalNode (location: (27,10)-(27,14)) - │ │ └── numeric: - │ │ @ FloatNode (location: (27,10)-(27,13)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (27,15)-(27,21)) @@ -1152,9 +1152,9 @@ │ │ │ ├── opening_loc: (37,3)-(37,4) = "{" │ │ │ └── closing_loc: (37,8)-(37,9) = "}" │ │ └── @ RationalNode (location: (37,11)-(37,15)) - │ │ └── numeric: - │ │ @ FloatNode (location: (37,11)-(37,14)) - │ │ └── value: 1.0 + │ │ ├── flags: decimal + │ │ ├── numerator: 1 + │ │ └── denominator: 1 │ ├── closing_loc: ∅ │ └── block: │ @ BlockNode (location: (37,16)-(37,22)) diff --git a/test/prism/snapshots/whitequark/ruby_bug_12073.txt b/test/prism/snapshots/whitequark/ruby_bug_12073.txt index 9db30f6fec..2b4d3eab2a 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_12073.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_12073.txt @@ -21,7 +21,7 @@ │ ├── opening_loc: ∅ │ ├── arguments: │ │ @ ArgumentsNode (location: (1,9)-(1,13)) - │ │ ├── flags: ∅ + │ │ ├── flags: contains_keywords │ │ └── arguments: (length: 1) │ │ └── @ KeywordHashNode (location: (1,9)-(1,13)) │ │ ├── flags: symbol_keys @@ -75,10 +75,9 @@ │ │ │ ├── parent: │ │ │ │ @ ConstantReadNode (location: (3,21)-(3,22)) │ │ │ │ └── name: :A - │ │ │ ├── child: - │ │ │ │ @ ConstantReadNode (location: (3,24)-(3,25)) - │ │ │ │ └── name: :B - │ │ │ └── delimiter_loc: (3,22)-(3,24) = "::" + │ │ │ ├── name: :B + │ │ │ ├── delimiter_loc: (3,22)-(3,24) = "::" + │ │ │ └── name_loc: (3,24)-(3,25) = "B" │ │ └── @ StringNode (location: (3,27)-(3,29)) │ │ ├── flags: ∅ │ │ ├── opening_loc: (3,27)-(3,28) = "'" diff --git a/test/prism/snapshots/whitequark/ruby_bug_12402.txt b/test/prism/snapshots/whitequark/ruby_bug_12402.txt index df5aea00c3..4935007f8a 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_12402.txt @@ -5,7 +5,7 @@ └── body: (length: 14) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,27)) │ ├── name_loc: (1,0)-(1,3) = "foo" - │ ├── operator_loc: (1,4)-(1,6) = "+=" + │ ├── binary_operator_loc: (1,4)-(1,6) = "+=" │ ├── value: │ │ @ RescueModifierNode (location: (1,7)-(1,27)) │ │ ├── expression: @@ -36,11 +36,11 @@ │ │ └── rescue_expression: │ │ @ NilNode (location: (1,24)-(1,27)) │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,28)) │ ├── name_loc: (3,0)-(3,3) = "foo" - │ ├── operator_loc: (3,4)-(3,6) = "+=" + │ ├── binary_operator_loc: (3,4)-(3,6) = "+=" │ ├── value: │ │ @ RescueModifierNode (location: (3,7)-(3,28)) │ │ ├── expression: @@ -71,7 +71,7 @@ │ │ └── rescue_expression: │ │ @ NilNode (location: (3,25)-(3,28)) │ ├── name: :foo - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableWriteNode (location: (5,0)-(5,26)) │ ├── name: :foo @@ -151,8 +151,8 @@ │ ├── message_loc: (9,4)-(9,5) = "C" │ ├── read_name: :C │ ├── write_name: :C= - │ ├── operator: :+ - │ ├── operator_loc: (9,6)-(9,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (9,6)-(9,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (9,9)-(9,29)) │ ├── expression: @@ -192,8 +192,8 @@ │ ├── message_loc: (11,4)-(11,5) = "C" │ ├── read_name: :C │ ├── write_name: :C= - │ ├── operator: :+ - │ ├── operator_loc: (11,6)-(11,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (11,6)-(11,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (11,9)-(11,30)) │ ├── expression: @@ -233,8 +233,8 @@ │ ├── message_loc: (13,4)-(13,5) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (13,6)-(13,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (13,6)-(13,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (13,9)-(13,29)) │ ├── expression: @@ -274,8 +274,8 @@ │ ├── message_loc: (15,4)-(15,5) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (15,6)-(15,8) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (15,6)-(15,8) = "+=" │ └── value: │ @ RescueModifierNode (location: (15,9)-(15,30)) │ ├── expression: @@ -312,10 +312,9 @@ │ │ │ @ LocalVariableReadNode (location: (17,0)-(17,3)) │ │ │ ├── name: :foo │ │ │ └── depth: 0 - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (17,5)-(17,6)) - │ │ │ └── name: :C - │ │ └── delimiter_loc: (17,3)-(17,5) = "::" + │ │ ├── name: :C + │ │ ├── delimiter_loc: (17,3)-(17,5) = "::" + │ │ └── name_loc: (17,5)-(17,6) = "C" │ ├── operator_loc: (17,7)-(17,10) = "||=" │ └── value: │ @ RescueModifierNode (location: (17,11)-(17,31)) @@ -353,10 +352,9 @@ │ │ │ @ LocalVariableReadNode (location: (19,0)-(19,3)) │ │ │ ├── name: :foo │ │ │ └── depth: 0 - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (19,5)-(19,6)) - │ │ │ └── name: :C - │ │ └── delimiter_loc: (19,3)-(19,5) = "::" + │ │ ├── name: :C + │ │ ├── delimiter_loc: (19,3)-(19,5) = "::" + │ │ └── name_loc: (19,5)-(19,6) = "C" │ ├── operator_loc: (19,7)-(19,10) = "||=" │ └── value: │ @ RescueModifierNode (location: (19,11)-(19,32)) @@ -397,8 +395,8 @@ │ ├── message_loc: (21,5)-(21,6) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (21,7)-(21,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (21,7)-(21,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (21,10)-(21,30)) │ ├── expression: @@ -438,8 +436,8 @@ │ ├── message_loc: (23,5)-(23,6) = "m" │ ├── read_name: :m │ ├── write_name: :m= - │ ├── operator: :+ - │ ├── operator_loc: (23,7)-(23,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (23,7)-(23,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (23,10)-(23,31)) │ ├── expression: @@ -486,8 +484,8 @@ │ │ └── value: 0 │ ├── closing_loc: (25,5)-(25,6) = "]" │ ├── block: ∅ - │ ├── operator: :+ - │ ├── operator_loc: (25,7)-(25,9) = "+=" + │ ├── binary_operator: :+ + │ ├── binary_operator_loc: (25,7)-(25,9) = "+=" │ └── value: │ @ RescueModifierNode (location: (25,10)-(25,30)) │ ├── expression: @@ -534,8 +532,8 @@ │ └── value: 0 ├── closing_loc: (27,5)-(27,6) = "]" ├── block: ∅ - ├── operator: :+ - ├── operator_loc: (27,7)-(27,9) = "+=" + ├── binary_operator: :+ + ├── binary_operator_loc: (27,7)-(27,9) = "+=" └── value: @ RescueModifierNode (location: (27,10)-(27,31)) ├── expression: diff --git a/test/prism/snapshots/whitequark/ruby_bug_12669.txt b/test/prism/snapshots/whitequark/ruby_bug_12669.txt index 86b021351b..6151143ba8 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_12669.txt @@ -5,11 +5,11 @@ └── body: (length: 4) ├── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,18)) │ ├── name_loc: (1,0)-(1,1) = "a" - │ ├── operator_loc: (1,2)-(1,4) = "+=" + │ ├── binary_operator_loc: (1,2)-(1,4) = "+=" │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (1,5)-(1,18)) │ │ ├── name_loc: (1,5)-(1,6) = "b" - │ │ ├── operator_loc: (1,7)-(1,9) = "+=" + │ │ ├── binary_operator_loc: (1,7)-(1,9) = "+=" │ │ ├── value: │ │ │ @ CallNode (location: (1,10)-(1,18)) │ │ │ ├── flags: ignore_visibility @@ -31,14 +31,14 @@ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ │ │ ├── name: :b - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableOperatorWriteNode (location: (3,0)-(3,17)) │ ├── name_loc: (3,0)-(3,1) = "a" - │ ├── operator_loc: (3,2)-(3,4) = "+=" + │ ├── binary_operator_loc: (3,2)-(3,4) = "+=" │ ├── value: │ │ @ LocalVariableWriteNode (location: (3,5)-(3,17)) │ │ ├── name: :b @@ -66,7 +66,7 @@ │ │ │ └── block: ∅ │ │ └── operator_loc: (3,7)-(3,8) = "=" │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 ├── @ LocalVariableWriteNode (location: (5,0)-(5,17)) │ ├── name: :a @@ -75,7 +75,7 @@ │ ├── value: │ │ @ LocalVariableOperatorWriteNode (location: (5,4)-(5,17)) │ │ ├── name_loc: (5,4)-(5,5) = "b" - │ │ ├── operator_loc: (5,6)-(5,8) = "+=" + │ │ ├── binary_operator_loc: (5,6)-(5,8) = "+=" │ │ ├── value: │ │ │ @ CallNode (location: (5,9)-(5,17)) │ │ │ ├── flags: ignore_visibility @@ -97,7 +97,7 @@ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ │ │ ├── name: :b - │ │ ├── operator: :+ + │ │ ├── binary_operator: :+ │ │ └── depth: 0 │ └── operator_loc: (5,2)-(5,3) = "=" └── @ LocalVariableWriteNode (location: (7,0)-(7,16)) diff --git a/test/prism/snapshots/whitequark/send_attr_asgn.txt b/test/prism/snapshots/whitequark/send_attr_asgn.txt index 392ae5587e..1d09daa4a6 100644 --- a/test/prism/snapshots/whitequark/send_attr_asgn.txt +++ b/test/prism/snapshots/whitequark/send_attr_asgn.txt @@ -69,10 +69,9 @@ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ └── block: ∅ - │ │ ├── child: - │ │ │ @ ConstantReadNode (location: (5,5)-(5,6)) - │ │ │ └── name: :A - │ │ └── delimiter_loc: (5,3)-(5,5) = "::" + │ │ ├── name: :A + │ │ ├── delimiter_loc: (5,3)-(5,5) = "::" + │ │ └── name_loc: (5,5)-(5,6) = "A" │ ├── operator_loc: (5,7)-(5,8) = "=" │ └── value: │ @ IntegerNode (location: (5,9)-(5,10)) diff --git a/test/prism/snapshots/whitequark/var_op_asgn.txt b/test/prism/snapshots/whitequark/var_op_asgn.txt index f423a62dee..f20f612fa2 100644 --- a/test/prism/snapshots/whitequark/var_op_asgn.txt +++ b/test/prism/snapshots/whitequark/var_op_asgn.txt @@ -6,30 +6,30 @@ ├── @ ClassVariableOperatorWriteNode (location: (1,0)-(1,11)) │ ├── name: :@@var │ ├── name_loc: (1,0)-(1,5) = "@@var" - │ ├── operator_loc: (1,6)-(1,8) = "|=" + │ ├── binary_operator_loc: (1,6)-(1,8) = "|=" │ ├── value: │ │ @ IntegerNode (location: (1,9)-(1,11)) │ │ ├── flags: decimal │ │ └── value: 10 - │ └── operator: :| + │ └── binary_operator: :| ├── @ InstanceVariableOperatorWriteNode (location: (3,0)-(3,7)) │ ├── name: :@a │ ├── name_loc: (3,0)-(3,2) = "@a" - │ ├── operator_loc: (3,3)-(3,5) = "|=" + │ ├── binary_operator_loc: (3,3)-(3,5) = "|=" │ ├── value: │ │ @ IntegerNode (location: (3,6)-(3,7)) │ │ ├── flags: decimal │ │ └── value: 1 - │ └── operator: :| + │ └── binary_operator: :| ├── @ LocalVariableOperatorWriteNode (location: (5,0)-(5,6)) │ ├── name_loc: (5,0)-(5,1) = "a" - │ ├── operator_loc: (5,2)-(5,4) = "+=" + │ ├── binary_operator_loc: (5,2)-(5,4) = "+=" │ ├── value: │ │ @ IntegerNode (location: (5,5)-(5,6)) │ │ ├── flags: decimal │ │ └── value: 1 │ ├── name: :a - │ ├── operator: :+ + │ ├── binary_operator: :+ │ └── depth: 0 └── @ DefNode (location: (7,0)-(7,23)) ├── name: :a @@ -42,12 +42,12 @@ │ └── @ ClassVariableOperatorWriteNode (location: (7,7)-(7,18)) │ ├── name: :@@var │ ├── name_loc: (7,7)-(7,12) = "@@var" - │ ├── operator_loc: (7,13)-(7,15) = "|=" + │ ├── binary_operator_loc: (7,13)-(7,15) = "|=" │ ├── value: │ │ @ IntegerNode (location: (7,16)-(7,18)) │ │ ├── flags: decimal │ │ └── value: 10 - │ └── operator: :| + │ └── binary_operator: :| ├── locals: [] ├── def_keyword_loc: (7,0)-(7,3) = "def" ├── operator_loc: ∅ diff --git a/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt b/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt index d56c099c7e..0bfa06d5b7 100644 --- a/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt +++ b/test/prism/snapshots/whitequark/var_op_asgn_cmd.txt @@ -5,7 +5,7 @@ └── body: (length: 1) └── @ LocalVariableOperatorWriteNode (location: (1,0)-(1,12)) ├── name_loc: (1,0)-(1,3) = "foo" - ├── operator_loc: (1,4)-(1,6) = "+=" + ├── binary_operator_loc: (1,4)-(1,6) = "+=" ├── value: │ @ CallNode (location: (1,7)-(1,12)) │ ├── flags: ignore_visibility @@ -24,5 +24,5 @@ │ ├── closing_loc: ∅ │ └── block: ∅ ├── name: :foo - ├── operator: :+ + ├── binary_operator: :+ └── depth: 0 diff --git a/test/prism/static_inspect_test.rb b/test/prism/static_inspect_test.rb index 8df2fd241e..cc8ed28c95 100644 --- a/test/prism/static_inspect_test.rb +++ b/test/prism/static_inspect_test.rb @@ -2,8 +2,6 @@ require_relative "test_helper" -return if Prism::BACKEND == :FFI - module Prism class StaticInspectTest < TestCase def test_false @@ -84,7 +82,8 @@ module Prism private def static_inspect(source, **options) - Debug.static_inspect(source, **options) + warnings = Prism.parse("{ #{source} => 1, #{source} => 1 }", **options).warnings + warnings.last.message[/^key (.+) is duplicated and overwritten on line \d/, 1] end end end diff --git a/test/prism/unescape_test.rb b/test/prism/unescape_test.rb index 2a352c5234..3f78a59b11 100644 --- a/test/prism/unescape_test.rb +++ b/test/prism/unescape_test.rb @@ -38,7 +38,7 @@ module Prism end def prism(escape) - result = Prism.parse(code(escape)) + result = Prism.parse(code(escape), encoding: "binary") if result.success? yield result.value.statements.body.first diff --git a/test/prism/warnings_test.rb b/test/prism/warnings_test.rb index d01db01a0e..7ad704918a 100644 --- a/test/prism/warnings_test.rb +++ b/test/prism/warnings_test.rb @@ -23,6 +23,23 @@ module Prism assert_warning("a /b/", "wrap regexp in parentheses") end + def test_binary_operator + [ + [:**, "argument prefix"], + [:*, "argument prefix"], + [:<<, "here document"], + [:&, "argument prefix"], + [:+, "unary operator"], + [:-, "unary operator"], + [:/, "regexp literal"], + [:%, "string literal"] + ].each do |(operator, warning)| + assert_warning("puts 1 #{operator}0", warning) + assert_warning("puts :a #{operator}0", warning) + assert_warning("m = 1; puts m #{operator}0", warning) + end + end + def test_equal_in_conditional assert_warning("if a = 1; end; a = a", "should be ==") end @@ -48,7 +65,7 @@ module Prism end def test_duplicated_when_clause - assert_warning("case 1; when 1, 1; end", "clause with line") + assert_warning("case 1; when 1, 1; end", "when' clause") end def test_float_out_of_range @@ -64,6 +81,15 @@ module Prism assert_warning("if true\nelsif\nfalse; end", "end of line") end + def test_shareable_constant_value + assert_warning("foo # shareable_constant_value: none", "ignored") + assert_warning("\v # shareable_constant_value: none", "ignored") + + refute_warning("# shareable_constant_value: none") + refute_warning(" # shareable_constant_value: none") + refute_warning("\t\t# shareable_constant_value: none") + end + def test_string_in_predicate assert_warning("if 'foo'; end", "string") assert_warning("if \"\#{foo}\"; end", "string") diff --git a/test/reline/test_config.rb b/test/reline/test_config.rb index e9f0a9449d..03e4178f32 100644 --- a/test/reline/test_config.rb +++ b/test/reline/test_config.rb @@ -275,6 +275,78 @@ class Reline::Config::Test < Reline::TestCase assert_equal "INPUTRC:1: unmatched endif", e.message end + def test_if_with_mode + @config.read_lines(<<~LINES.lines) + $if mode=emacs + "\C-e": history-search-backward # comment + $else + "\C-f": history-search-forward + $endif + LINES + + assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + end + + def test_else + @config.read_lines(<<~LINES.lines) + $if mode=vi + "\C-e": history-search-backward # comment + $else + "\C-f": history-search-forward + $endif + LINES + + assert_equal({[6] => :history_search_forward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + end + + def test_if_with_invalid_mode + @config.read_lines(<<~LINES.lines) + $if mode=vim + "\C-e": history-search-backward + $else + "\C-f": history-search-forward # comment + $endif + LINES + + assert_equal({[6] => :history_search_forward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + end + + def test_mode_label_differs_from_keymap_label + @config.read_lines(<<~LINES.lines) + # Sets mode_label and keymap_label to vi + set editing-mode vi + # Change keymap_label to emacs. mode_label is still vi. + set keymap emacs + # condition=true because current mode_label is vi + $if mode=vi + # sets keybinding to current keymap_label=emacs + "\C-e": history-search-backward + $endif + LINES + assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + end + + def test_if_without_else_condition + @config.read_lines(<<~LINES.lines) + set editing-mode vi + $if mode=vi + "\C-e": history-search-backward + $endif + LINES + + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:emacs]) + assert_equal({[5] => :history_search_backward}, @config.instance_variable_get(:@additional_key_bindings)[:vi_insert]) + assert_equal({}, @config.instance_variable_get(:@additional_key_bindings)[:vi_command]) + end + def test_default_key_bindings @config.add_default_key_binding('abcd'.bytes, 'EFGH'.bytes) @config.read_lines(<<~'LINES'.lines) @@ -345,6 +417,18 @@ class Reline::Config::Test < Reline::TestCase assert_equal expected, @config.key_bindings end + def test_key_bindings_with_reset + # @config.reset is called after each readline. + # inputrc file is read once, so key binding shouldn't be cleared by @config.reset + @config.add_default_key_binding('default'.bytes, 'DEFAULT'.bytes) + @config.read_lines(<<~'LINES'.lines) + "additional": "ADDITIONAL" + LINES + @config.reset + expected = { 'default'.bytes => 'DEFAULT'.bytes, 'additional'.bytes => 'ADDITIONAL'.bytes } + assert_equal expected, @config.key_bindings + end + def test_history_size @config.read_lines(<<~LINES.lines) set history-size 5000 @@ -375,6 +459,17 @@ class Reline::Config::Test < Reline::TestCase ENV['INPUTRC'] = inputrc_backup end + def test_inputrc_raw_value + @config.read_lines(<<~'LINES'.lines) + set editing-mode vi ignored-string + set vi-ins-mode-string aaa aaa + set vi-cmd-mode-string bbb ccc # comment + LINES + assert_equal :vi_insert, @config.instance_variable_get(:@editing_mode_label) + assert_equal 'aaa aaa', @config.vi_ins_mode_string + assert_equal 'bbb ccc # comment', @config.vi_cmd_mode_string + end + def test_inputrc_with_utf8 # This file is encoded by UTF-8 so this heredoc string is also UTF-8. @config.read_lines(<<~'LINES'.lines) @@ -458,4 +553,3 @@ class Reline::Config::Test < Reline::TestCase ENV['HOME'] = home_backup end end - diff --git a/test/reline/test_key_actor_emacs.rb b/test/reline/test_key_actor_emacs.rb index a9baf9ad37..013ca2f7b3 100644 --- a/test/reline/test_key_actor_emacs.rb +++ b/test/reline/test_key_actor_emacs.rb @@ -787,9 +787,22 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase input_keys('b') input_keys("\C-i", false) assert_line_around_cursor('foo_ba', '') + input_keys("\C-h") + input_key_by_symbol(:complete) + assert_line_around_cursor('foo_ba', '') + input_keys("\C-h", false) + input_key_by_symbol(:menu_complete) + assert_line_around_cursor('foo_bar', '') + input_key_by_symbol(:menu_complete) + assert_line_around_cursor('foo_baz', '') + input_keys("\C-h", false) + input_key_by_symbol(:menu_complete_backward) + assert_line_around_cursor('foo_baz', '') + input_key_by_symbol(:menu_complete_backward) + assert_line_around_cursor('foo_bar', '') end - def test_autocompletion_with_upward_navigation + def test_autocompletion @config.autocompletion = true @line_editor.completion_proc = proc { |word| %w{ @@ -806,31 +819,14 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase assert_line_around_cursor('Readline', '') input_keys("\C-i", false) assert_line_around_cursor('Regexp', '') - @line_editor.input_key(Reline::Key.new(:completion_journey_up, :completion_journey_up, false)) + input_key_by_symbol(:completion_journey_up) assert_line_around_cursor('Readline', '') - ensure - @config.autocompletion = false - end - - def test_autocompletion_with_upward_navigation_and_menu_complete_backward - @config.autocompletion = true - @line_editor.completion_proc = proc { |word| - %w{ - Readline - Regexp - RegexpError - }.map { |i| - i.encode(@encoding) - } - } - input_keys('Re') - assert_line_around_cursor('Re', '') - input_keys("\C-i", false) - assert_line_around_cursor('Readline', '') - input_keys("\C-i", false) + input_key_by_symbol(:complete) assert_line_around_cursor('Regexp', '') - @line_editor.input_key(Reline::Key.new(:menu_complete_backward, :menu_complete_backward, false)) + input_key_by_symbol(:menu_complete_backward) assert_line_around_cursor('Readline', '') + input_key_by_symbol(:menu_complete) + assert_line_around_cursor('Regexp', '') ensure @config.autocompletion = false end @@ -1441,4 +1437,72 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase @line_editor.__send__(:vi_editing_mode, nil) assert(@config.editing_mode_is?(:vi_insert)) end + + def test_undo + input_keys("\C-_", false) + assert_line_around_cursor('', '') + input_keys("aあb\C-h\C-h\C-h", false) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあb', '') + input_keys("\C-_", false) + assert_line_around_cursor('aあ', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('', '') + end + + def test_undo_with_cursor_position + input_keys("abc\C-b\C-h", false) + assert_line_around_cursor('a', 'c') + input_keys("\C-_", false) + assert_line_around_cursor('ab', 'c') + input_keys("あいう\C-b\C-h", false) + assert_line_around_cursor('abあ', 'うc') + input_keys("\C-_", false) + assert_line_around_cursor('abあい', 'うc') + end + + def test_undo_with_multiline + @line_editor.multiline_on + @line_editor.confirm_multiline_termination_proc = proc {} + input_keys("1\n2\n3", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(2) + assert_line_around_cursor('3', '') + input_keys("\C-p\C-h\C-h", false) + assert_whole_lines(["1", "3"]) + assert_line_index(0) + assert_line_around_cursor('1', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "", "3"]) + assert_line_index(1) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2", "3"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2", ""]) + assert_line_index(2) + assert_line_around_cursor('', '') + input_keys("\C-_", false) + assert_whole_lines(["1", "2"]) + assert_line_index(1) + assert_line_around_cursor('2', '') + end + + def test_undo_with_many_times + str = "a" + "b" * 100 + input_keys(str, false) + 100.times { input_keys("\C-_", false) } + assert_line_around_cursor('a', '') + input_keys("\C-_", false) + assert_line_around_cursor('a', '') + end end diff --git a/test/reline/test_line_editor.rb b/test/reline/test_line_editor.rb index bf688ac3c6..7a38ecd596 100644 --- a/test/reline/test_line_editor.rb +++ b/test/reline/test_line_editor.rb @@ -112,6 +112,36 @@ class Reline::LineEditor end end + def test_multibyte + base = [0, 12, '一二三一二三'] + left = [0, 3, 'LLL'] + right = [9, 3, 'RRR'] + front = [3, 6, 'FFFFFF'] + # 一 FFFFFF 三 + # 一二三一二三 + assert_output '[COL_2]二三一二' do + @line_editor.render_line_differential([base, front], [base, nil]) + end + + # LLLFFFFFF 三 + # LLL 三一二三 + assert_output '[COL_3] 三一二' do + @line_editor.render_line_differential([base, left, front], [base, left, nil]) + end + + # 一 FFFFFFRRR + # 一二三一 RRR + assert_output '[COL_2]二三一 ' do + @line_editor.render_line_differential([base, right, front], [base, right, nil]) + end + + # LLLFFFFFFRRR + # LLL 三一 RRR + assert_output '[COL_3] 三一 ' do + @line_editor.render_line_differential([base, left, right, front], [base, left, right, nil]) + end + end + def test_complicated state_a = [nil, [19, 7, 'bbbbbbb'], [15, 8, 'cccccccc'], [10, 5, 'ddddd'], [18, 4, 'eeee'], [1, 3, 'fff'], [17, 2, 'gg'], [7, 1, 'h']] state_b = [[5, 9, 'aaaaaaaaa'], nil, [15, 8, 'cccccccc'], nil, [18, 4, 'EEEE'], [25, 4, 'ffff'], [17, 2, 'gg'], [2, 2, 'hh']] diff --git a/test/reline/test_reline.rb b/test/reline/test_reline.rb index 40c880c11f..a20a5c9f44 100644 --- a/test/reline/test_reline.rb +++ b/test/reline/test_reline.rb @@ -378,10 +378,28 @@ class Reline::Test < Reline::TestCase assert_equal("Reline::GeneralIO", out.chomp) end + def test_require_reline_should_not_trigger_winsize + pend if win? + lib = File.expand_path("../../lib", __dir__) + code = <<~RUBY + require "io/console" + def STDIN.tty?; true; end + def STDOUT.tty?; true; end + def STDIN.winsize; raise; end + require("reline") && p(Reline.core.io_gate) + RUBY + out = IO.popen([{}, Reline.test_rubybin, "-I#{lib}", "-e", code], &:read) + assert_equal("Reline::ANSI", out.chomp) + end + + def win? + /mswin|mingw/.match?(RUBY_PLATFORM) + end + def get_reline_encoding if encoding = Reline.core.encoding encoding - elsif RUBY_PLATFORM =~ /mswin|mingw/ + elsif win? Encoding::UTF_8 else Encoding::default_external diff --git a/test/reline/test_unicode.rb b/test/reline/test_unicode.rb index a9d82e2c56..deba4d4681 100644 --- a/test/reline/test_unicode.rb +++ b/test/reline/test_unicode.rb @@ -46,8 +46,8 @@ class Reline::Unicode::Test < Reline::TestCase def test_take_range assert_equal 'cdef', Reline::Unicode.take_range('abcdefghi', 2, 4) assert_equal 'あde', Reline::Unicode.take_range('abあdef', 2, 4) - assert_equal 'zerocdef', Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4) - assert_equal 'bzerocde', Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4) + assert_equal "\1zero\2cdef", Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4) + assert_equal "b\1zero\2cde", Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4) assert_equal "\e[31mcd\e[42mef", Reline::Unicode.take_range("\e[31mabcd\e[42mefg", 2, 4) assert_equal "\e]0;1\acd", Reline::Unicode.take_range("ab\e]0;1\acd", 2, 3) assert_equal 'いう', Reline::Unicode.take_range('あいうえお', 2, 4) @@ -67,4 +67,26 @@ class Reline::Unicode::Test < Reline::TestCase assert_equal 10, Reline::Unicode.calculate_width('あいうえお') assert_equal 10, Reline::Unicode.calculate_width('あいうえお', true) end + + def test_take_mbchar_range + assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4) + assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, padding: true) + assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_begin: true) + assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_end: true) + assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4) + assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, padding: true) + assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_begin: true) + assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_end: true) + assert_equal ['う', 4, 2], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4) + assert_equal [' う ', 3, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, padding: true) + assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true) + assert_equal ['うえ', 4, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true) + assert_equal ['いう ', 2, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true, padding: true) + assert_equal [' うえ', 3, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true, padding: true) + assert_equal [' うえお ', 3, 10], Reline::Unicode.take_mbchar_range('あいうえお', 3, 10, padding: true) + assert_equal [" \e[41mうえお\e[0m ", 3, 10], Reline::Unicode.take_mbchar_range("あい\e[41mうえお", 3, 10, padding: true) + assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true) + assert_equal ["\e[31mc\1ABC\2d\e[0mef", 2, 4], Reline::Unicode.take_mbchar_range("\e[31mabc\1ABC\2d\e[0mefghi", 2, 4) + assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true) + end end diff --git a/test/reline/test_within_pipe.rb b/test/reline/test_within_pipe.rb index a42ca755fc..4f05255301 100644 --- a/test/reline/test_within_pipe.rb +++ b/test/reline/test_within_pipe.rb @@ -23,7 +23,6 @@ class Reline::WithinPipeTest < Reline::TestCase @reader.close @output_writer.close @config.reset - @config.reset_default_key_bindings Reline.test_reset end diff --git a/test/reline/yamatanooroti/multiline_repl b/test/reline/yamatanooroti/multiline_repl index eba410f6dd..8b82be60f4 100755 --- a/test/reline/yamatanooroti/multiline_repl +++ b/test/reline/yamatanooroti/multiline_repl @@ -12,7 +12,7 @@ opt = OptionParser.new opt.on('--dynamic-prompt') { Reline.prompt_proc = proc { |lines| lines.each_with_index.map { |l, i| - '[%04d]> ' % i + "\e[1m[%04d]>\e[m " % i } } } @@ -222,7 +222,7 @@ rescue end begin - prompt = ENV['RELINE_TEST_PROMPT'] || 'prompt> ' + prompt = ENV['RELINE_TEST_PROMPT'] || "\e[1mprompt>\e[m " puts 'Multiline REPL.' while code = Reline.readmultiline(prompt, true) { |code| TerminationChecker.terminated?(code) } case code.chomp diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index 7b96e7b6a6..37a1c1a193 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -543,15 +543,10 @@ begin EOC end - def test_enable_bracketed_paste + def test_bracketed_paste omit if Reline.core.io_gate.win? - write_inputrc <<~LINES - set enable-bracketed-paste on - LINES start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') - write("\e[200~,") - write("def hoge\n 3\nend") - write("\e[200~.") + write("\e[200~def hoge\r\t3\rend\e[201~") close assert_screen(<<~EOC) Multiline REPL. @@ -561,6 +556,19 @@ begin EOC end + def test_bracketed_paste_with_undo + omit if Reline.core.io_gate.win? + start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') + write("abc") + write("\e[200~def hoge\r\t3\rend\e[201~") + write("\C-_") + close + assert_screen(<<~EOC) + Multiline REPL. + prompt> abc + EOC + end + def test_backspace_until_returns_to_initial start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') write("ABC") @@ -945,18 +953,30 @@ begin EOC end - def test_with_newline + def test_eof_with_newline omit if Reline.core.io_gate.win? cmd = %Q{ruby -e 'print(%Q{abc def \\e\\r})' | ruby -I#{@pwd}/lib -rreline -e 'p Reline.readline(%{> })'} start_terminal(40, 50, ['bash', '-c', cmd]) sleep 1 - close + close rescue nil assert_screen(<<~'EOC') > abc def "abc def " EOC end + def test_eof_without_newline + omit if Reline.core.io_gate.win? + cmd = %Q{ruby -e 'print(%{hello})' | ruby -I#{@pwd}/lib -rreline -e 'p Reline.readline(%{> })'} + start_terminal(40, 50, ['bash', '-c', cmd]) + sleep 1 + close rescue nil + assert_screen(<<~'EOC') + > hello + "hello" + EOC + end + def test_em_set_mark_and_em_exchange_mark start_terminal(10, 50, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') write("aaa bbb ccc ddd\M-b\M-b\M-\x20\M-b\C-x\C-xX\C-x\C-xY") @@ -1173,7 +1193,36 @@ begin assert_screen(<<~'EOC') Multiline REPL. prompt> St - r String + r + String + Struct + EOC + end + + def test_autocomplete_target_at_end_of_line + start_terminal(20, 20, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --autocomplete}, startup_message: 'Multiline REPL.') + write(' ') + write('Str') + write("\C-i") + close + assert_screen(<<~'EOC') + Multiline REPL. + prompt> Str + ing String + Struct + EOC + end + + def test_autocomplete_completed_input_is_wrapped + start_terminal(20, 20, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --autocomplete}, startup_message: 'Multiline REPL.') + write(' ') + write('Str') + write("\C-i") + close + assert_screen(<<~'EOC') + Multiline REPL. + prompt> Stri + ng String Struct EOC end diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb index cbae6e7ed5..a5c7401968 100644 --- a/test/ripper/test_parser_events.rb +++ b/test/ripper/test_parser_events.rb @@ -269,15 +269,27 @@ class TestRipper::ParserEvents < Test::Unit::TestCase def test_assign_error_backref thru_assign_error = false result = - parse('$` = 1', :on_assign_error) {thru_assign_error = true} + parse('$& = 1', :on_assign_error) {thru_assign_error = true} assert_equal true, thru_assign_error - assert_equal '[assign(assign_error(var_field($`)),1)]', result + assert_equal '[assign(assign_error(var_field($&)),1)]', result thru_assign_error = false result = - parse('$`, _ = 1', :on_assign_error) {thru_assign_error = true} + parse('$&, _ = 1', :on_assign_error) {thru_assign_error = true} assert_equal true, thru_assign_error - assert_equal '[massign([assign_error(var_field($`)),var_field(_)],1)]', result + assert_equal '[massign([assign_error(var_field($&)),var_field(_)],1)]', result + + thru_assign_error = false + result = + parse('$& += 1', :on_assign_error) {thru_assign_error = true} + assert_equal true, thru_assign_error + assert_equal '[assign_error(opassign(var_field($&),+=,1))]', result + + thru_assign_error = false + result = + parse('$& += cmd 1, 2', :on_assign_error) {thru_assign_error = true} + assert_equal true, thru_assign_error + assert_equal '[assign_error(opassign(var_field($&),+=,command(cmd,[1,2])))]', result end def test_assign_error_const_qualified @@ -1682,8 +1694,8 @@ class TestRipper::ParserEvents < Test::Unit::TestCase else end STR - assert_match(/duplicated 'when' clause/, fmt) - assert_equal([3], args) + assert_match(/duplicates 'when' clause/, fmt) + assert_equal([4, 3], args) end def test_warn_duplicated_hash_keys diff --git a/test/ripper/test_ripper.rb b/test/ripper/test_ripper.rb index 6061496d9c..f129a41be9 100644 --- a/test/ripper/test_ripper.rb +++ b/test/ripper/test_ripper.rb @@ -170,6 +170,14 @@ end end; end + def test_sexp_no_memory_leak + assert_no_memory_leak(%w(-rripper), "", "#{<<~'end;'}", rss: true) + 1_000_000.times do + Ripper.sexp("") + end + end; + end + class TestInput < self Input = Struct.new(:lines) do def gets diff --git a/test/ruby/test_allocation.rb b/test/ruby/test_allocation.rb index 48348c0fbd..fbe0548899 100644 --- a/test/ruby/test_allocation.rb +++ b/test/ruby/test_allocation.rb @@ -25,6 +25,10 @@ class TestAllocation < Test::Unit::TestCase empty_array = empty_array = [] empty_hash = empty_hash = {} array1 = array1 = [1] + r2k_array = r2k_array = [Hash.ruby2_keywords_hash(a: 3)] + r2k_array1 = r2k_array1 = [1, Hash.ruby2_keywords_hash(a: 3)] + r2k_empty_array = r2k_empty_array = [Hash.ruby2_keywords_hash({})] + r2k_empty_array1 = r2k_empty_array1 = [1, Hash.ruby2_keywords_hash({})] hash1 = hash1 = {a: 2} nill = nill = nil block = block = lambda{} @@ -95,6 +99,8 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 0, "none(*empty_array, *empty_array#{block})") check_allocations(0, 1, "none(**empty_hash, **empty_hash#{block})") check_allocations(1, 1, "none(*empty_array, *empty_array, **empty_hash, **empty_hash#{block})") + + check_allocations(0, 0, "none(*r2k_empty_array#{block})") RUBY end @@ -114,6 +120,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "required(**hash1, **empty_hash#{block})") check_allocations(1, 0, "required(*array1, *empty_array, **empty_hash#{block})") + check_allocations(0, 0, "required(*r2k_empty_array1#{block})") + check_allocations(0, 1, "required(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required(*empty_array, **hash1, **empty_hash#{block})") RUBY @@ -135,6 +144,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "optional(**hash1, **empty_hash#{block})") check_allocations(1, 0, "optional(*array1, *empty_array, **empty_hash#{block})") + check_allocations(0, 0, "optional(*r2k_empty_array#{block})") + check_allocations(0, 0, "optional(*r2k_empty_array1#{block})") + check_allocations(0, 1, "optional(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "optional(*empty_array, **hash1, **empty_hash#{block})") RUBY @@ -166,6 +179,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat(**hash1, **empty_hash#{block})") check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "splat(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*r2k_empty_array#{block})") + check_allocations(1, 0, "splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat(*r2k_array#{block})") + check_allocations(1, 1, "splat(*r2k_array1#{block})") RUBY end @@ -195,6 +213,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "req_splat(**hash1, **empty_hash#{block})") check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "req_splat(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "req_splat(*r2k_array#{block})") + check_allocations(1, 1, "req_splat(*r2k_array1#{block})") RUBY end @@ -224,6 +246,10 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_post(**hash1, **empty_hash#{block})") check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") check_allocations(1, 1, "splat_post(*empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_post(*r2k_array#{block})") + check_allocations(1, 1, "splat_post(*r2k_array1#{block})") RUBY end @@ -251,6 +277,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword(*empty_array, **hash1, **empty_hash#{block})") @@ -281,6 +310,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword_splat(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword_splat(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword_splat(*empty_array, **hash1, **empty_hash#{block})") @@ -311,6 +343,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*r2k_array#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, **hash1, **empty_hash#{block})") @@ -350,6 +385,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(*r2k_empty_array1#{block})") + check_allocations(1, 1, "required_and_keyword(*r2k_array1#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") @@ -397,6 +435,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_and_keyword(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "splat_and_keyword(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "splat_and_keyword(*array1, **nil#{block})") + + check_allocations(1, 0, "splat_and_keyword(*r2k_empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword(*r2k_array#{block})") + check_allocations(1, 0, "splat_and_keyword(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_and_keyword(*r2k_array1#{block})") RUBY end @@ -433,6 +476,9 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*r2k_array1#{block})") + # Currently allocates 1 array unnecessarily due to splatarray true check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") @@ -480,6 +526,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*r2k_array1#{block})") RUBY end @@ -521,6 +572,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array1#{block})") RUBY end @@ -562,6 +618,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_empty_array1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*r2k_array1#{block})") RUBY end @@ -603,6 +664,11 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array1#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array1#{block})") RUBY end @@ -644,6 +710,61 @@ class TestAllocation < Test::Unit::TestCase check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_empty_array1#{block})") + check_allocations(1, 1, "argument_forwarding(*r2k_array1#{block})") + RUBY + end + + def test_ruby2_keywords + check_allocations(<<~RUBY) + def self.r2k(*a#{block}); end + singleton_class.send(:ruby2_keywords, :r2k) + + check_allocations(1, 1, "r2k(1, a: 2#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "r2k(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "r2k(1, **nil#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **hash1#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "r2k(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "r2k(1, *empty_array#{block})") + check_allocations(1, 1, "r2k(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(*array1, a: 2#{block})") + + check_allocations(1, 0, "r2k(*array1, **nill#{block})") + check_allocations(1, 1, "r2k(*array1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, **hash1#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "r2k(*array1, *empty_array#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "r2k(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "r2k(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "r2k(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "r2k(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "r2k(*array1, **nil#{block})") + + check_allocations(1, 0, "r2k(*r2k_empty_array#{block})") + check_allocations(1, 1, "r2k(*r2k_array#{block})") + unless defined?(RubyVM::YJIT.enabled?) && RubyVM::YJIT.enabled? + # YJIT may or may not allocate depending on arch? + check_allocations(1, 0, "r2k(*r2k_empty_array1#{block})") + check_allocations(1, 1, "r2k(*r2k_array1#{block})") + end RUBY end diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb index 29da607fc5..45c6b63963 100644 --- a/test/ruby/test_ast.rb +++ b/test/ruby/test_ast.rb @@ -337,6 +337,8 @@ class TestAst < Test::Unit::TestCase end def test_node_id_for_location + omit if compiling_with_prism? + exception = begin raise rescue => e @@ -356,6 +358,8 @@ class TestAst < Test::Unit::TestCase end def test_of_proc_and_method + omit if compiling_with_prism? + proc = Proc.new { 1 + 2 } method = self.method(__method__) @@ -385,6 +389,8 @@ class TestAst < Test::Unit::TestCase end def test_of_backtrace_location + omit if compiling_with_prism? + backtrace_location, lineno = sample_backtrace_location node = RubyVM::AbstractSyntaxTree.of(backtrace_location) assert_instance_of(RubyVM::AbstractSyntaxTree::Node, node) @@ -396,6 +402,8 @@ class TestAst < Test::Unit::TestCase end def test_of_proc_and_method_under_eval + omit if compiling_with_prism? + keep_script_lines_back = RubyVM.keep_script_lines RubyVM.keep_script_lines = false @@ -425,6 +433,7 @@ class TestAst < Test::Unit::TestCase end def test_of_proc_and_method_under_eval_with_keep_script_lines + omit if compiling_with_prism? pend if ENV['RUBY_ISEQ_DUMP_DEBUG'] # TODO keep_script_lines_back = RubyVM.keep_script_lines @@ -456,6 +465,8 @@ class TestAst < Test::Unit::TestCase end def test_of_backtrace_location_under_eval + omit if compiling_with_prism? + keep_script_lines_back = RubyVM.keep_script_lines RubyVM.keep_script_lines = false @@ -474,6 +485,7 @@ class TestAst < Test::Unit::TestCase end def test_of_backtrace_location_under_eval_with_keep_script_lines + omit if compiling_with_prism? pend if ENV['RUBY_ISEQ_DUMP_DEBUG'] # TODO keep_script_lines_back = RubyVM.keep_script_lines @@ -736,6 +748,8 @@ dummy end def test_keep_script_lines_for_of + omit if compiling_with_prism? + proc = Proc.new { 1 + 2 } method = self.method(__method__) @@ -1238,6 +1252,24 @@ dummy end end + def test_memory_leak + assert_no_memory_leak([], "#{<<~"begin;"}", "\n#{<<~'end;'}", rss: true) + begin; + 1_000_000.times do + eval("") + end + end; + end + + private + + # We can't revisit instruction sequences to find node ids if the prism + # compiler was used instead of the parse.y compiler. In that case, we'll omit + # some tests. + def compiling_with_prism? + RubyVM::InstructionSequence.compile("").to_a[4][:parser] == :prism + end + def assert_error_tolerant(src, expected, keep_tokens: false) begin verbose_bak, $VERBOSE = $VERBOSE, false diff --git a/test/ruby/test_file.rb b/test/ruby/test_file.rb index aa10566bfa..9fa96f8a7c 100644 --- a/test/ruby/test_file.rb +++ b/test/ruby/test_file.rb @@ -460,6 +460,39 @@ class TestFile < Test::Unit::TestCase end end + def test_initialize + Dir.mktmpdir(__method__.to_s) do |tmpdir| + path = File.join(tmpdir, "foo") + + assert_raise(Errno::ENOENT) {File.new(path)} + f = File.new(path, "w") + f.write("FOO\n") + f.close + f = File.new(path) + data = f.read + f.close + assert_equal("FOO\n", data) + + f = File.new(path, File::WRONLY) + f.write("BAR\n") + f.close + f = File.new(path, File::RDONLY) + data = f.read + f.close + assert_equal("BAR\n", data) + + data = File.open(path) {|file| + File.new(file.fileno, mode: File::RDONLY, autoclose: false).read + } + assert_equal("BAR\n", data) + + data = File.open(path) {|file| + File.new(file.fileno, File::RDONLY, autoclose: false).read + } + assert_equal("BAR\n", data) + end + end + def test_file_open_newline_option Dir.mktmpdir(__method__.to_s) do |tmpdir| path = File.join(tmpdir, "foo") diff --git a/test/ruby/test_io.rb b/test/ruby/test_io.rb index 476d9f882f..ce81286c4d 100644 --- a/test/ruby/test_io.rb +++ b/test/ruby/test_io.rb @@ -2929,6 +2929,15 @@ class TestIO < Test::Unit::TestCase f.close assert_equal("FOO\n", File.read(t.path)) + + fd = IO.sysopen(t.path) + %w[w r+ w+ a+].each do |mode| + assert_raise(Errno::EINVAL, "#{mode} [ruby-dev:38571]") {IO.new(fd, mode)} + end + f = IO.new(fd, "r") + data = f.read + f.close + assert_equal("FOO\n", data) } end diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb index d2a39e673f..df4b6651b0 100644 --- a/test/ruby/test_iseq.rb +++ b/test/ruby/test_iseq.rb @@ -347,11 +347,17 @@ class TestISeq < Test::Unit::TestCase end end assert_equal([m1, e1.message], [m2, e2.message], feature11951) - message = e1.message.each_line - message.with_index(1) do |line, i| - next if /^ / =~ line - assert_send([line, :start_with?, __FILE__], - proc {message.map {|l, j| (i == j ? ">" : " ") + l}.join("")}) + + if e1.message.lines[0] == "#{__FILE__}:#{line}: syntax errors found\n" + # Prism lays out the error messages in line with the source, so the + # following assertions do not make sense in that context. + else + message = e1.message.each_line + message.with_index(1) do |line, i| + next if /^ / =~ line + assert_send([line, :start_with?, __FILE__], + proc {message.map {|l, j| (i == j ? ">" : " ") + l}.join("")}) + end end end diff --git a/test/ruby/test_marshal.rb b/test/ruby/test_marshal.rb index 79d9577737..bcd8892f23 100644 --- a/test/ruby/test_marshal.rb +++ b/test/ruby/test_marshal.rb @@ -570,13 +570,19 @@ class TestMarshal < Test::Unit::TestCase def test_class_ivar assert_raise(TypeError) {Marshal.load("\x04\x08Ic\x1bTestMarshal::TestClass\x06:\x0e@ivar_bug\"\x08bug")} assert_raise(TypeError) {Marshal.load("\x04\x08IM\x1bTestMarshal::TestClass\x06:\x0e@ivar_bug\"\x08bug")} - assert_not_operator(TestClass, :instance_variable_defined?, :@bug) + assert_not_operator(TestClass, :instance_variable_defined?, :@ivar_bug) + + assert_raise(TypeError) {Marshal.load("\x04\x08[\x07c\x1bTestMarshal::TestClassI@\x06\x06:\x0e@ivar_bug\"\x08bug")} + assert_not_operator(TestClass, :instance_variable_defined?, :@ivar_bug) end def test_module_ivar assert_raise(TypeError) {Marshal.load("\x04\x08Im\x1cTestMarshal::TestModule\x06:\x0e@ivar_bug\"\x08bug")} assert_raise(TypeError) {Marshal.load("\x04\x08IM\x1cTestMarshal::TestModule\x06:\x0e@ivar_bug\"\x08bug")} - assert_not_operator(TestModule, :instance_variable_defined?, :@bug) + assert_not_operator(TestModule, :instance_variable_defined?, :@ivar_bug) + + assert_raise(TypeError) {Marshal.load("\x04\x08[\x07m\x1cTestMarshal::TestModuleI@\x06\x06:\x0e@ivar_bug\"\x08bug")} + assert_not_operator(TestModule, :instance_variable_defined?, :@ivar_bug) end class TestForRespondToFalse diff --git a/test/ruby/test_pack.rb b/test/ruby/test_pack.rb index 1ce46e8916..a0c66c188b 100644 --- a/test/ruby/test_pack.rb +++ b/test/ruby/test_pack.rb @@ -895,4 +895,22 @@ EXPECTED } assert_equal [nil], "a".unpack("C", offset: 1) end + + def test_monkey_pack + assert_separately([], <<-'end;') + $-w = false + class Array + alias :old_pack :pack + def pack _; "oh no"; end + end + + v = [2 ** 15].pack('n') + + class Array + alias :pack :old_pack + end + + assert_equal "oh no", v + end; + end end diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb index fe649cddb9..3857f3cc17 100644 --- a/test/ruby/test_parse.rb +++ b/test/ruby/test_parse.rb @@ -18,7 +18,7 @@ class TestParse < Test::Unit::TestCase end def test_else_without_rescue - assert_syntax_error(<<-END, %r":#{__LINE__+2}: else without rescue"o, [__FILE__, __LINE__+1]) + assert_syntax_error(<<-END, %r"(:#{__LINE__+2}:|#{__LINE__+2} \|.+?\n.+?\^~.+?;) else without rescue"o, [__FILE__, __LINE__+1]) begin else 42 @@ -555,34 +555,42 @@ class TestParse < Test::Unit::TestCase mesg = 'from the backslash through the invalid char' e = assert_syntax_error('"\xg1"', /hex escape/) - assert_equal(' ^~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{1234"', 'unterminated Unicode escape') - assert_equal(' ^'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{xxxx}"', 'invalid Unicode escape') - assert_equal(' ^'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\u{xxxx', 'Unicode escape') - assert_pattern_list([ - /.*: invalid Unicode escape\n.*\n/, - / \^/, - /\n/, - /.*: unterminated Unicode escape\n.*\n/, - / \^/, - /\n/, - /.*: unterminated string.*\n.*\n/, - / \^\n/, - ], e.message) + if e.message.lines.first == "#{__FILE__}:#{__LINE__ - 1}: syntax errors found\n" + assert_pattern_list([ + /\s+\| \^ unterminated string;.+\n/, + /\s+\| \^ unterminated Unicode escape\n/, + /\s+\| \^ invalid Unicode escape sequence\n/, + ], e.message.lines[2..-1].join) + else + assert_pattern_list([ + /.*: invalid Unicode escape\n.*\n/, + / \^/, + /\n/, + /.*: unterminated Unicode escape\n.*\n/, + / \^/, + /\n/, + /.*: unterminated string.*\n.*\n/, + / \^\n/, + ], e.message) + end e = assert_syntax_error('"\M1"', /escape character syntax/) - assert_equal(' ^~~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~~(?!~)/, e.message.lines.last, mesg) e = assert_syntax_error('"\C1"', /escape character syntax/) - assert_equal(' ^~~'"\n", e.message.lines.last, mesg) + assert_match(/(^|\| ) \^~~(?!~)/, e.message.lines.last, mesg) src = '"\xD0\u{90'"\n""000000000000000000000000" - assert_syntax_error(src, /:#{__LINE__}: unterminated/o) + assert_syntax_error(src, /(:#{__LINE__}:|> #{__LINE__} \|.+) unterminated/om) assert_syntax_error('"\u{100000000}"', /invalid Unicode escape/) assert_equal("", eval('"\u{}"')) @@ -605,22 +613,22 @@ class TestParse < Test::Unit::TestCase assert_syntax_error("\"\\C-\\M-\x01\"", 'Invalid escape character syntax') e = assert_syntax_error('"\c\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\c\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\C-\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\C-\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\M-\u0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error('"\M-\U0000"', 'Invalid escape character syntax') - assert_equal(' ^~~~~'"\n", e.message.lines.last) + assert_match(/(^|\| ) \^~~~~(?!~)/, e.message.lines.last) e = assert_syntax_error(%["\\C-\u3042"], 'Invalid escape character syntax') - assert_match(/^\s \^(?# \\ ) ~(?# C ) ~(?# - ) ~+(?# U+3042 )$/x, e.message.lines.last) + assert_match(/(^|\|\s)\s \^(?# \\ ) ~(?# C ) ~(?# - ) ~+(?# U+3042 )($|\s)/x, e.message.lines.last) assert_not_include(e.message, "invalid multibyte char") end @@ -875,7 +883,8 @@ x = __ENCODING__ $test_parse_foobarbazqux = nil assert_equal(nil, $&) assert_equal(nil, eval('alias $& $preserve_last_match')) - assert_syntax_error('a = $#', /as a global variable name\na = \$\#\n \^~$/) + assert_syntax_error('a = $#', /as a global variable name/) + assert_syntax_error('a = $#', /a = \$\#\n(^|.+?\| ) \^~(?!~)/) end def test_invalid_instance_variable @@ -1259,8 +1268,8 @@ x = __ENCODING__ assert_syntax_error("def f r:def d; def f 0end", /unexpected/) end; - assert_syntax_error("def\nf(000)end", /^ \^~~/) - assert_syntax_error("def\nf(&0)end", /^ \^/) + assert_syntax_error("def\nf(000)end", /(^|\| ) \^~~/) + assert_syntax_error("def\nf(&0)end", /(^|\| ) \^/) end def test_method_location_in_rescue @@ -1336,17 +1345,21 @@ x = __ENCODING__ end def test_unexpected_token_after_numeric - assert_syntax_error('0000xyz', /^ \^~~\Z/) - assert_syntax_error('1.2i1.1', /^ \^~~\Z/) - assert_syntax_error('1.2.3', /^ \^~\Z/) + assert_syntax_error('0000xyz', /(^|\| ) \^~~(?!~)/) + assert_syntax_error('1.2i1.1', /(^|\| ) \^~~(?!~)/) + assert_syntax_error('1.2.3', /(^|\| ) \^~(?!~)/) assert_syntax_error('1.', /unexpected end-of-input/) assert_syntax_error('1e', /expecting end-of-input/) end def test_truncated_source_line - e = assert_syntax_error("'0123456789012345678901234567890123456789' abcdefghijklmnopqrstuvwxyz0123456789 0123456789012345678901234567890123456789", + lineno = __LINE__ + 1 + e = assert_syntax_error("'0123456789012345678901234567890123456789' abcdefghijklmnopqrstuvwxyz0123456789 123456789012345678901234567890123456789", /unexpected local variable or method/) + line = e.message.lines[1] + line.delete_prefix!("> #{lineno} | ") if line.start_with?(">") + assert_operator(line, :start_with?, "...") assert_operator(line, :end_with?, "...\n") end @@ -1390,11 +1403,11 @@ x = __ENCODING__ end def test_unexpected_eof - assert_syntax_error('unless', /^ \^\Z/) + assert_syntax_error('unless', /(^|\| ) \^(?!~)/) end def test_location_of_invalid_token - assert_syntax_error('class xxx end', /^ \^~~\Z/) + assert_syntax_error('class xxx end', /(^|\| ) \^~~(?!~)/) end def test_whitespace_warning diff --git a/test/ruby/test_pattern_matching.rb b/test/ruby/test_pattern_matching.rb index db6ad06b82..cfe3bd1e19 100644 --- a/test/ruby/test_pattern_matching.rb +++ b/test/ruby/test_pattern_matching.rb @@ -1331,7 +1331,7 @@ END end assert_block do - case {} + case C.new({}) in {} C.keys == nil end diff --git a/test/ruby/test_rubyvm.rb b/test/ruby/test_rubyvm.rb index d729aa5af8..053a914a8a 100644 --- a/test/ruby/test_rubyvm.rb +++ b/test/ruby/test_rubyvm.rb @@ -32,6 +32,7 @@ class TestRubyVM < Test::Unit::TestCase end def test_keep_script_lines + omit if compiling_with_prism? pend if ENV['RUBY_ISEQ_DUMP_DEBUG'] # TODO prev_conf = RubyVM.keep_script_lines @@ -68,4 +69,12 @@ class TestRubyVM < Test::Unit::TestCase ensure RubyVM.keep_script_lines = prev_conf end + + private + + # RubyVM.keep_script_lines does not mean anything in the context of prism, so + # we should omit tests that are looking for that functionality. + def compiling_with_prism? + RubyVM::InstructionSequence.compile("").to_a[4][:parser] == :prism + end end diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 44162f06cb..8f94ec71fc 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -506,10 +506,6 @@ class TestSyntax < Test::Unit::TestCase end def test_warn_balanced - warning = <<WARN -test:1: warning: '%s' after local variable or literal is interpreted as binary operator -test:1: warning: even though it seems like %s -WARN [ [:**, "argument prefix"], [:*, "argument prefix"], @@ -523,7 +519,9 @@ WARN all_assertions do |a| ["puts 1 #{op}0", "puts :a #{op}0", "m = 1; puts m #{op}0"].each do |src| a.for(src) do - assert_warning(warning % [op, syn], src) do + warning = /'#{Regexp.escape(op)}' after local variable or literal is interpreted as binary operator.+?even though it seems like #{syn}/m + + assert_warning(warning, src) do assert_valid_syntax(src, "test", verbose: true) end end @@ -715,8 +713,8 @@ WARN end def test_duplicated_when - w = 'warning: duplicated \'when\' clause with line 3 is ignored' - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + w = ->(line) { "warning: 'when' clause on line #{line} duplicates 'when' clause on line 3 and is ignored" } + assert_warning(/#{w[3]}.+#{w[4]}.+#{w[4]}.+#{w[5]}.+#{w[5]}/m) { eval %q{ case 1 when 1, 1 @@ -725,7 +723,7 @@ WARN end } } - assert_warning(/#{w}/) {#/3: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m){ + assert_warning(/#{w[3]}.+#{w[4]}.+#{w[5]}.+#{w[5]}/m) { a = a = 1 eval %q{ case 1 @@ -735,7 +733,7 @@ WARN end } } - assert_warning(/3: #{w}/m) { + assert_warning(/#{w[3]}/) { eval %q{ case 1 when __LINE__, __LINE__ @@ -744,7 +742,7 @@ WARN end } } - assert_warning(/3: #{w}/m) { + assert_warning(/#{w[3]}/) { eval %q{ case 1 when __FILE__, __FILE__ @@ -756,7 +754,7 @@ WARN end def test_duplicated_when_check_option - w = /duplicated \'when\' clause with line 3 is ignored/ + w = /'when' clause on line 4 duplicates 'when' clause on line 3 and is ignored/ assert_in_out_err(%[-wc], "#{<<~"begin;"}\n#{<<~'end;'}", ["Syntax OK"], w) begin; case 1 @@ -889,6 +887,16 @@ e" assert_dedented_heredoc(expect, result) end + def test_dedented_heredoc_with_leading_blank_line + # the blank line has six leading spaces + result = " \n" \ + " b\n" + expect = " \n" \ + "b\n" + assert_dedented_heredoc(expect, result) + end + + def test_dedented_heredoc_with_blank_more_indented_line_escaped result = " a\n" \ "\\ \\ \\ \\ \\ \\ \n" \ @@ -996,7 +1004,7 @@ eom end def test_dedented_heredoc_concatenation - assert_equal("\n0\n1", eval("<<~0 '1'\n \n0\#{}\n0")) + assert_equal(" \n0\n1", eval("<<~0 '1'\n \n0\#{}\n0")) end def test_heredoc_mixed_encoding @@ -1238,6 +1246,20 @@ eom assert_syntax_error("a&.x,=0", /multiple assignment destination/) end + def test_safe_call_in_for_variable + assert_valid_syntax("for x&.bar in []; end") + assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}") + begin; + foo = nil + for foo&.bar in [1]; end + assert_nil(foo) + + foo = Struct.new(:bar).new + for foo&.bar in [1]; end + assert_equal(1, foo.bar) + end; + end + def test_no_warning_logop_literal assert_warning("") do eval("true||raise;nil") @@ -1577,7 +1599,7 @@ eom end def test_syntax_error_at_newline - expected = "\n ^" + expected = /(\n|\| ) \^/ assert_syntax_error("%[abcdef", expected) assert_syntax_error("%[abcdef\n", expected) end @@ -1755,8 +1777,8 @@ eom assert_equal("instance ok", k.new.rescued("ok")) # Current technical limitation: cannot prepend "private" or something for command endless def - error = /syntax error, unexpected string literal/ - error2 = /syntax error, unexpected local variable or method/ + error = /(syntax error,|\^~*) unexpected string literal/ + error2 = /(syntax error,|\^~*) unexpected local variable or method/ assert_syntax_error('private def foo = puts "Hello"', error) assert_syntax_error('private def foo() = puts "Hello"', error) assert_syntax_error('private def foo(x) = puts x', error2) @@ -1900,7 +1922,7 @@ eom ] end assert_valid_syntax('proc {def foo(_);end;it}') - assert_syntax_error('p { [it **2] }', /unexpected \*\*arg/) + assert_syntax_error('p { [it **2] }', /unexpected \*\*/) end def test_value_expr_in_condition diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index f97306717d..7014843bba 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -76,8 +76,6 @@ class Gem::TestCase < Test::Unit::TestCase attr_accessor :uri # :nodoc: - @@tempdirs = [] - def assert_activate(expected, *specs) specs.each do |spec| case spec @@ -451,9 +449,7 @@ class Gem::TestCase < Test::Unit::TestCase Dir.chdir @current_dir - FileUtils.rm_rf @tempdir - - restore_env + ENV.replace(@orig_env) Gem::ConfigFile.send :remove_const, :SYSTEM_WIDE_CONFIG_FILE Gem::ConfigFile.send :const_set, :SYSTEM_WIDE_CONFIG_FILE, @@ -481,12 +477,9 @@ class Gem::TestCase < Test::Unit::TestCase @back_ui.close - refute_directory_exists @tempdir, "may be still in use" - ghosts = @@tempdirs.filter_map do |test_name, tempdir| - test_name if File.exist?(tempdir) - end - @@tempdirs << [method_name, @tempdir] - assert_empty ghosts + FileUtils.rm_rf @tempdir + + refute_directory_exists @tempdir, "#{@tempdir} used by test #{method_name} is still in use" end def credential_setup @@ -541,6 +534,16 @@ class Gem::TestCase < Test::Unit::TestCase ENV["BUNDLE_GEMFILE"] = File.join(@tempdir, "Gemfile") end + def with_env(overrides, &block) + orig_env = ENV.to_h + ENV.replace(overrides) + begin + block.call + ensure + ENV.replace(orig_env) + end + end + ## # A git_gem is used with a gem dependencies file. The gem created here # has no files, just a gem specification for the given +name+ and +version+. @@ -1526,23 +1529,6 @@ Also, a list: PUBLIC_KEY = nil PUBLIC_CERT = nil end if Gem::HAVE_OPENSSL - - private - - def restore_env - unless Gem.win_platform? - ENV.replace(@orig_env) - return - end - - # Fallback logic for Windows below to workaround - # https://bugs.ruby-lang.org/issues/16798. Can be dropped once all - # supported rubies include the fix for that. - - ENV.clear - - @orig_env.each {|k, v| ENV[k] = v } - end end # https://github.com/seattlerb/minitest/blob/13c48a03d84a2a87855a4de0c959f96800100357/lib/minitest/mock.rb#L192 diff --git a/test/rubygems/test_bundled_ca.rb b/test/rubygems/test_bundled_ca.rb index 50e621f22b..a737185681 100644 --- a/test/rubygems/test_bundled_ca.rb +++ b/test/rubygems/test_bundled_ca.rb @@ -33,7 +33,7 @@ class TestGemBundledCA < Gem::TestCase http.verify_mode = OpenSSL::SSL::VERIFY_PEER http.cert_store = bundled_certificate_store http.get("/") - rescue Errno::ENOENT, Errno::ETIMEDOUT, SocketError, Net::OpenTimeout + rescue Errno::ENOENT, Errno::ETIMEDOUT, SocketError, Gem::Net::OpenTimeout pend "#{host} seems offline, I can't tell whether ssl would work." rescue OpenSSL::SSL::SSLError => e # Only fail for certificate verification errors diff --git a/test/rubygems/test_gem.rb b/test/rubygems/test_gem.rb index 244b7749a5..40a473f8d6 100644 --- a/test/rubygems/test_gem.rb +++ b/test/rubygems/test_gem.rb @@ -516,7 +516,10 @@ class TestGem < Gem::TestCase Gem.clear_paths - assert_nil Gem::Specification.send(:class_variable_get, :@@all) + with_env("GEM_HOME" => "foo", "GEM_PATH" => "bar") do + assert_equal("foo", Gem.dir) + assert_equal("bar", Gem.path.first) + end end def test_self_configuration @@ -1281,7 +1284,6 @@ class TestGem < Gem::TestCase def test_self_try_activate_missing_extensions spec = util_spec "ext", "1" do |s| s.extensions = %w[ext/extconf.rb] - s.mark_version s.installed_by_version = v("2.2") end diff --git a/test/rubygems/test_gem_ci_detector.rb b/test/rubygems/test_gem_ci_detector.rb index 3caefce97d..a28ee49f4b 100644 --- a/test/rubygems/test_gem_ci_detector.rb +++ b/test/rubygems/test_gem_ci_detector.rb @@ -3,7 +3,7 @@ require_relative "helper" require "rubygems" -class TestCiDetector < Test::Unit::TestCase +class TestCiDetector < Gem::TestCase def test_ci? with_env("FOO" => "bar") { assert_equal(false, Gem::CIDetector.ci?) } with_env("CI" => "true") { assert_equal(true, Gem::CIDetector.ci?) } @@ -29,16 +29,4 @@ class TestCiDetector < Test::Unit::TestCase assert_equal(["dsari", "taskcluster"], Gem::CIDetector.ci_strings) end end - - private - - def with_env(overrides, &block) - @orig_env = ENV.to_h - ENV.replace(overrides) - begin - block.call - ensure - ENV.replace(@orig_env) - end - end end diff --git a/test/rubygems/test_gem_commands_rebuild_command.rb b/test/rubygems/test_gem_commands_rebuild_command.rb index 5e8c797e2d..3b7927c44e 100644 --- a/test/rubygems/test_gem_commands_rebuild_command.rb +++ b/test/rubygems/test_gem_commands_rebuild_command.rb @@ -105,7 +105,7 @@ class TestGemCommandsRebuildCommand < Gem::TestCase assert_equal old_spec.name, new_spec.name assert_equal old_spec.summary, new_spec.summary - reproduced + [reproduced, original] end def test_build_is_reproducible @@ -134,12 +134,21 @@ class TestGemCommandsRebuildCommand < Gem::TestCase # also testing that `gem rebuild` overrides the value. ENV["SOURCE_DATE_EPOCH"] = Time.new(2007, 8, 9, 10, 11, 12).to_s - rebuild_gem_file = util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) + rebuild_gem_file, saved_gem_file = + util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) rebuild_contents = File.read(rebuild_gem_file) assert_equal build_contents, rebuild_contents ensure ENV["SOURCE_DATE_EPOCH"] = epoch + if rebuild_gem_file + File.unlink(rebuild_gem_file) + dir = File.dirname(rebuild_gem_file) + Dir.rmdir(dir) + File.unlink(saved_gem_file) + Dir.rmdir(File.dirname(saved_gem_file)) + Dir.rmdir(File.dirname(dir)) + end end end diff --git a/test/rubygems/test_gem_commands_setup_command.rb b/test/rubygems/test_gem_commands_setup_command.rb index 43f695f147..8eedb6c03a 100644 --- a/test/rubygems/test_gem_commands_setup_command.rb +++ b/test/rubygems/test_gem_commands_setup_command.rb @@ -159,6 +159,23 @@ class TestGemCommandsSetupCommand < Gem::TestCase end end + def test_destdir_flag_regenerates_binstubs + # install to destdir + destdir = File.join(@tempdir, "foo") + gem_bin_path = gem_install "destdir-only-gem", install_dir: destdir + + # change binstub manually + write_file gem_bin_path do |io| + io.puts "I changed it!" + end + + @cmd.options[:destdir] = destdir + @cmd.options[:prefix] = "/" + @cmd.execute + + assert_match(/\A#!/, File.read(gem_bin_path)) + end + def test_files_in assert_equal %w[rubygems.rb rubygems/requirement.rb rubygems/ssl_certs/rubygems.org/foo.pem], @cmd.files_in("lib").sort @@ -412,7 +429,7 @@ class TestGemCommandsSetupCommand < Gem::TestCase end end - def gem_install(name) + def gem_install(name, **options) gem = util_spec name do |s| s.executables = [name] s.files = %W[bin/#{name}] @@ -420,8 +437,8 @@ class TestGemCommandsSetupCommand < Gem::TestCase write_file File.join @tempdir, "bin", name do |f| f.puts "#!/usr/bin/ruby" end - install_gem gem - File.join @gemhome, "bin", name + install_gem gem, **options + File.join options[:install_dir] || @gemhome, "bin", name end def gem_install_with_plugin(name) diff --git a/test/rubygems/test_gem_package_tar_header.rb b/test/rubygems/test_gem_package_tar_header.rb index 4469750f9a..a3f95bb770 100644 --- a/test/rubygems/test_gem_package_tar_header.rb +++ b/test/rubygems/test_gem_package_tar_header.rb @@ -99,6 +99,31 @@ class TestGemPackageTarHeader < Gem::Package::TarTestCase assert_empty @tar_header end + def test_empty + @tar_header = Gem::Package::TarHeader.from(StringIO.new(Gem::Package::TarHeader::EMPTY_HEADER)) + + assert_empty @tar_header + assert_equal Gem::Package::TarHeader.new( + checksum: 0, + devmajor: 0, + devminor: 0, + empty: true, + gid: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + mtime: 0, + name: "", + prefix: "", + size: 0, + typeflag: "0", + uid: 0, + uname: "", + version: 0, + ), @tar_header + end + def test_equals2 assert_equal @tar_header, @tar_header assert_equal @tar_header, @tar_header.dup diff --git a/test/rubygems/test_gem_platform.rb b/test/rubygems/test_gem_platform.rb index e4bf882317..00e48498c6 100644 --- a/test/rubygems/test_gem_platform.rb +++ b/test/rubygems/test_gem_platform.rb @@ -145,6 +145,9 @@ class TestGemPlatform < Gem::TestCase "x86_64-openbsd3.9" => ["x86_64", "openbsd", "3.9"], "x86_64-openbsd4.0" => ["x86_64", "openbsd", "4.0"], "x86_64-openbsd" => ["x86_64", "openbsd", nil], + "wasm32-wasi" => ["wasm32", "wasi", nil], + "wasm32-wasip1" => ["wasm32", "wasi", nil], + "wasm32-wasip2" => ["wasm32", "wasi", nil], } test_cases.each do |arch, expected| diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 9e05649f7c..9395e34f75 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -56,7 +56,6 @@ end s.add_dependency "jabber4r", "> 0.0.0" s.add_dependency "pqa", ["> 0.4", "<= 0.6"] - s.mark_version s.files = %w[lib/code.rb] end end @@ -69,7 +68,6 @@ end s.license = "MIT" s.platform = platform - s.mark_version s.files = %w[lib/code.rb] s.installed_by_version = v("2.2") end @@ -96,7 +94,6 @@ end s.requirements << "A working computer" s.license = "MIT" - s.mark_version s.files = %w[lib/code.rb] end @@ -970,7 +967,10 @@ dependencies: [] def test_self_stubs_for_lazy_loading Gem.loaded_specs.clear - Gem::Specification.class_variable_set(:@@stubs, nil) + + specification_record = Gem::Specification.specification_record + + specification_record.instance_variable_set(:@stubs, nil) dir_standard_specs = File.join Gem.dir, "specifications" @@ -978,9 +978,9 @@ dependencies: [] save_gemspec("b-1", "1", dir_standard_specs) {|s| s.name = "b" } assert_equal ["a-1"], Gem::Specification.stubs_for("a").map(&:full_name) - assert_equal 1, Gem::Specification.class_variable_get(:@@stubs_by_name).length + assert_equal 1, specification_record.instance_variable_get(:@stubs_by_name).length assert_equal ["b-1"], Gem::Specification.stubs_for("b").map(&:full_name) - assert_equal 2, Gem::Specification.class_variable_get(:@@stubs_by_name).length + assert_equal 2, specification_record.instance_variable_get(:@stubs_by_name).length assert_equal( Gem::Specification.stubs_for("a").map(&:object_id), @@ -989,7 +989,7 @@ dependencies: [] Gem.loaded_specs.delete "a" Gem.loaded_specs.delete "b" - Gem::Specification.class_variable_set(:@@stubs, nil) + specification_record.instance_variable_set(:@stubs, nil) end def test_self_stubs_for_no_lazy_loading_after_all_specs_setup @@ -3187,7 +3187,7 @@ or set it to nil if you don't want to specify a license. end def test_removed_methods - assert_equal Gem::Specification::REMOVED_METHODS, [:rubyforge_project=] + assert_equal Gem::Specification::REMOVED_METHODS, [:rubyforge_project=, :mark_version] end def test_validate_removed_rubyforge_project @@ -3480,12 +3480,17 @@ Did you mean 'Ruby'? util_setup_validate @a1.rubygems_version = "3" - e = assert_raise Gem::InvalidSpecificationException do + + use_ui @ui do @a1.validate end - assert_equal "expected RubyGems version #{Gem::VERSION}, was 3", - e.message + expected = <<~EXPECTED + #{w}: expected RubyGems version #{Gem::VERSION}, was 3 + #{w}: See https://guides.rubygems.org/specification-reference/ for help + EXPECTED + + assert_equal expected, @ui.error end def test_validate_specification_version diff --git a/test/rubygems/test_gem_uninstaller.rb b/test/rubygems/test_gem_uninstaller.rb index 9e0c1aa3d8..aa5ab0ed67 100644 --- a/test/rubygems/test_gem_uninstaller.rb +++ b/test/rubygems/test_gem_uninstaller.rb @@ -188,22 +188,81 @@ class TestGemUninstaller < Gem::InstallerTestCase refute File.exist?(plugin_path), "plugin not removed" end - def test_remove_plugins_with_install_dir + def test_uninstall_with_install_dir_removes_plugins write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| io.write "# do nothing" end @spec.files += %w[lib/rubygems_plugin.rb] - Gem::Installer.at(Gem::Package.build(@spec), force: true).install + package = Gem::Package.build(@spec) + + Gem::Installer.at(package, force: true).install plugin_path = File.join Gem.plugindir, "a_plugin.rb" assert File.exist?(plugin_path), "plugin not written" - Dir.mkdir "#{@gemhome}2" - Gem::Uninstaller.new(nil, install_dir: "#{@gemhome}2").remove_plugins @spec + install_dir = "#{@gemhome}2" + + Gem::Installer.at(package, force: true, install_dir: install_dir).install + + install_dir_plugin_path = File.join install_dir, "plugins/a_plugin.rb" + assert File.exist?(install_dir_plugin_path), "plugin not written" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, executables: true, install_dir: install_dir).uninstall assert File.exist?(plugin_path), "plugin unintentionally removed" + refute File.exist?(install_dir_plugin_path), "plugin not removed" + end + + def test_uninstall_with_install_dir_regenerates_plugins + write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| + io.write "# do nothing" + end + + @spec.files += %w[lib/rubygems_plugin.rb] + + install_dir = "#{@gemhome}2" + + package = Gem::Package.build(@spec) + + spec_v9 = @spec.dup + spec_v9.version = "9" + package_v9 = Gem::Package.build(spec_v9) + + Gem::Installer.at(package, force: true, install_dir: install_dir).install + Gem::Installer.at(package_v9, force: true, install_dir: install_dir).install + + install_dir_plugin_path = File.join install_dir, "plugins/a_plugin.rb" + assert File.exist?(install_dir_plugin_path), "plugin not written" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, version: "9", executables: true, install_dir: install_dir).uninstall + assert File.exist?(install_dir_plugin_path), "plugin unintentionally removed" + + Gem::Specification.dirs = [install_dir] + Gem::Uninstaller.new(@spec.name, executables: true, install_dir: install_dir).uninstall + refute File.exist?(install_dir_plugin_path), "plugin not removed" + end + + def test_remove_plugins_user_installed + write_file File.join(@tempdir, "lib", "rubygems_plugin.rb") do |io| + io.write "# do nothing" + end + + @spec.files += %w[lib/rubygems_plugin.rb] + + Gem::Installer.at(Gem::Package.build(@spec), force: true, user_install: true).install + + plugin_path = File.join Gem.user_dir, "plugins/a_plugin.rb" + assert File.exist?(plugin_path), "plugin not written" + + Gem::Specification.dirs = [Gem.dir, Gem.user_dir] + + Gem::Uninstaller.new(@spec.name, executables: true, force: true, user_install: true).uninstall + + refute File.exist?(plugin_path), "plugin not removed" end def test_regenerate_plugins_for @@ -370,7 +429,7 @@ create_makefile '#{@spec.name}' end def test_uninstall_user_install - @user_spec = Gem::Specification.find_by_name "b" + Gem::Specification.dirs = [Gem.user_dir] uninstaller = Gem::Uninstaller.new(@user_spec.name, executables: true, @@ -394,6 +453,32 @@ create_makefile '#{@spec.name}' assert_same uninstaller, @post_uninstall_hook_arg end + def test_uninstall_user_install_with_symlinked_home + pend "Symlinks not supported or not enabled" unless symlink_supported? + + Gem::Specification.dirs = [Gem.user_dir] + + symlinked_home = File.join(@tempdir, "new-home") + FileUtils.ln_s(Gem.user_home, symlinked_home) + + ENV["HOME"] = symlinked_home + Gem.instance_variable_set(:@user_home, nil) + Gem.instance_variable_set(:@data_home, nil) + + uninstaller = Gem::Uninstaller.new(@user_spec.name, + executables: true, + user_install: true, + force: true) + + gem_dir = File.join @user_spec.gem_dir + + assert_path_exist gem_dir + + uninstaller.uninstall + + assert_path_not_exist gem_dir + end + def test_uninstall_wrong_repo Dir.mkdir "#{@gemhome}2" Gem.use_paths "#{@gemhome}2", [@gemhome] diff --git a/test/rubygems/test_webauthn_poller.rb b/test/rubygems/test_webauthn_poller.rb index 23290d8ea1..fd24081758 100644 --- a/test/rubygems/test_webauthn_poller.rb +++ b/test/rubygems/test_webauthn_poller.rb @@ -13,7 +13,7 @@ class WebauthnPollerTest < Gem::TestCase @fetcher = Gem::FakeFetcher.new Gem::RemoteFetcher.fetcher = @fetcher @credentials = { - email: "email@example.com", + identifier: "email@example.com", password: "password", } end @@ -121,4 +121,14 @@ class WebauthnPollerTest < Gem::TestCase assert_equal error.message, "Security device verification failed: The token in the link you used has either expired or been used already." end + + def test_poll_for_otp_missing_credentials + @credentials = { password: "password" } + + error = assert_raise Gem::WebauthnVerificationError do + Gem::GemcutterUtilities::WebauthnPoller.new({}, @host).poll_for_otp(@webauthn_url, @credentials) + end + + assert_equal error.message, "Security device verification failed: Provided missing credentials" + end end diff --git a/test/test_delegate.rb b/test/test_delegate.rb index 57480b18ea..f7bedf37fb 100644 --- a/test/test_delegate.rb +++ b/test/test_delegate.rb @@ -3,14 +3,6 @@ require 'test/unit' require 'delegate' class TestDelegateClass < Test::Unit::TestCase - module PP - def mu_pp(obj) - str = super - str = "#<#{obj.class}: #{str}>" if Delegator === obj - str - end - end - module M attr_reader :m end @@ -215,7 +207,6 @@ class TestDelegateClass < Test::Unit::TestCase end def test_eql? - extend PP s0 = SimpleDelegator.new("foo") s1 = SimpleDelegator.new("bar") s2 = SimpleDelegator.new("foo") diff --git a/test/win32/test_registry.rb b/test/win32/test_registry.rb new file mode 100644 index 0000000000..02cafc09b0 --- /dev/null +++ b/test/win32/test_registry.rb @@ -0,0 +1,97 @@ +if /mswin|mingw|cygwin/ =~ RUBY_PLATFORM + begin + require 'win32/registry' + rescue LoadError + else + require 'test/unit' + end +end + +if defined?(Win32::Registry) + class TestWin32Registry < Test::Unit::TestCase + COMPUTERNAME = 'SYSTEM\\CurrentControlSet\\Control\\ComputerName\\ComputerName' + VOLATILE_ENVIRONMENT = 'Volatile Environment' + + def test_predefined + assert_predefined_key Win32::Registry::HKEY_CLASSES_ROOT + assert_predefined_key Win32::Registry::HKEY_CURRENT_USER + assert_predefined_key Win32::Registry::HKEY_LOCAL_MACHINE + assert_predefined_key Win32::Registry::HKEY_USERS + assert_predefined_key Win32::Registry::HKEY_PERFORMANCE_DATA + assert_predefined_key Win32::Registry::HKEY_PERFORMANCE_TEXT + assert_predefined_key Win32::Registry::HKEY_PERFORMANCE_NLSTEXT + assert_predefined_key Win32::Registry::HKEY_CURRENT_CONFIG + assert_predefined_key Win32::Registry::HKEY_DYN_DATA + end + + def test_class_open + name1, keys1 = Win32::Registry.open(Win32::Registry::HKEY_LOCAL_MACHINE, "SYSTEM") do |reg| + assert_predicate reg, :open? + [reg.name, reg.keys] + end + name2, keys2 = Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM") do |reg| + assert_predicate reg, :open? + [reg.name, reg.keys] + end + assert_equal name1, name2 + assert_equal keys1, keys2 + end + + def test_read + computername = ENV['COMPUTERNAME'] + Win32::Registry::HKEY_LOCAL_MACHINE.open(COMPUTERNAME) do |reg| + assert_equal computername, reg['ComputerName'] + assert_equal [Win32::Registry::REG_SZ, computername], reg.read('ComputerName') + assert_raise(TypeError) {reg.read('ComputerName', Win32::Registry::REG_DWORD)} + end + end + + def test_create + desired = Win32::Registry::KEY_ALL_ACCESS + option = Win32::Registry::REG_OPTION_VOLATILE + Win32::Registry::HKEY_CURRENT_USER.open(VOLATILE_ENVIRONMENT, desired) do |reg| + v = self.class.unused_value(reg) + begin + reg.create(v, desired, option) {} + ensure + reg.delete_key(v, true) + end + end + end + + def test_write + desired = Win32::Registry::KEY_ALL_ACCESS + Win32::Registry::HKEY_CURRENT_USER.open(VOLATILE_ENVIRONMENT, desired) do |reg| + v = self.class.unused_value(reg) + begin + reg.write_s(v, "data") + assert_equal [Win32::Registry::REG_SZ, "data"], reg.read(v) + reg.write_i(v, 0x5fe79027) + assert_equal [Win32::Registry::REG_DWORD, 0x5fe79027], reg.read(v) + ensure + reg.delete(v) + end + end + end + + private + + def assert_predefined_key(key) + assert_kind_of Win32::Registry, key + assert_predicate key, :open? + assert_not_predicate key, :created? + end + + class << self + def unused_value(reg, prefix = "Test_", limit = 100, fail: true) + limit.times do + v = + rand(0x100000).to_s(36) + reg.read(v) + rescue + return v + end + omit "Unused value not found in #{reg}" if fail + end + end + end +end diff --git a/test/zlib/test_zlib.rb b/test/zlib/test_zlib.rb index ae4adc21fe..15e5bd852f 100644 --- a/test/zlib/test_zlib.rb +++ b/test/zlib/test_zlib.rb @@ -991,6 +991,25 @@ if defined? Zlib assert_raise(ArgumentError) { f.read(-1) } assert_equal(str, f.read) end + + Zlib::GzipReader.open(t.path) do |f| + s = "".b + + assert_raise(ArgumentError) { f.read(-1, s) } + + assert_same s, f.read(1, s) + assert_equal "\xE3".b, s + + assert_same s, f.read(2, s) + assert_equal "\x81\x82".b, s + + assert_same s, f.read(6, s) + assert_equal "\u3044\u3046".b, s + + assert_nil f.read(1, s) + assert_equal "".b, s + assert_predicate f, :eof? + end } end @@ -1005,10 +1024,14 @@ if defined? Zlib Zlib::GzipReader.open(t.path) do |f| s = "".dup - f.readpartial(3, s) + assert_same s, f.readpartial(3, s) assert("foo".start_with?(s)) assert_raise(ArgumentError) { f.readpartial(-1) } + + assert_same s, f.readpartial(3, s) + + assert_predicate f, :eof? end } end @@ -197,6 +197,10 @@ static inline void blocking_region_end(rb_thread_t *th, struct rb_blocking_regio if (blocking_region_begin(th, &__region, (ubf), (ubfarg), fail_if_interrupted) || \ /* always return true unless fail_if_interrupted */ \ !only_if_constant(fail_if_interrupted, TRUE)) { \ + /* Important that this is inlined into the macro, and not part of \ + * blocking_region_begin - see bug #20493 */ \ + RB_VM_SAVE_MACHINE_CONTEXT(th); \ + thread_sched_to_waiting(TH_SCHED(th), th); \ exec; \ blocking_region_end(th, &__region); \ }; \ @@ -1482,9 +1486,6 @@ blocking_region_begin(rb_thread_t *th, struct rb_blocking_region_buffer *region, rb_ractor_blocking_threads_inc(th->ractor, __FILE__, __LINE__); RUBY_DEBUG_LOG("thread_id:%p", (void *)th->nt->thread_id); - - RB_VM_SAVE_MACHINE_CONTEXT(th); - thread_sched_to_waiting(TH_SCHED(th), th); return TRUE; } else { @@ -1767,7 +1768,7 @@ rb_thread_mn_schedulable(VALUE thval) VALUE rb_thread_io_blocking_call(rb_blocking_function_t *func, void *data1, int fd, int events) { - rb_execution_context_t * volatile ec = GET_EC(); + rb_execution_context_t *volatile ec = GET_EC(); rb_thread_t *th = rb_ec_thread_ptr(ec); RUBY_DEBUG_LOG("th:%u fd:%d ev:%d", rb_th_serial(th), fd, events); @@ -1795,6 +1796,7 @@ rb_thread_io_blocking_call(rb_blocking_function_t *func, void *data1, int fd, in saved_errno = errno; }, ubf_select, waiting_fd.th, FALSE); + th = rb_ec_thread_ptr(ec); if (events && blocking_call_retryable_p((int)val, saved_errno) && thread_io_wait_events(th, fd, events, NULL)) { @@ -1804,6 +1806,7 @@ rb_thread_io_blocking_call(rb_blocking_function_t *func, void *data1, int fd, in } EC_POP_TAG(); + th = rb_ec_thread_ptr(ec); th->mn_schedulable = prev_mn_schedulable; } /* @@ -1893,6 +1896,8 @@ rb_thread_call_with_gvl(void *(*func)(void *), void *data1) /* leave from Ruby world: You can not access Ruby values, etc. */ int released = blocking_region_begin(th, brb, prev_unblock.func, prev_unblock.arg, FALSE); RUBY_ASSERT_ALWAYS(released); + RB_VM_SAVE_MACHINE_CONTEXT(th); + thread_sched_to_waiting(TH_SCHED(th), th); return r; } @@ -4205,9 +4210,10 @@ rb_fd_set(int fd, rb_fdset_t *set) #endif static int -wait_retryable(int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) +wait_retryable(volatile int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) { - if (*result < 0) { + int r = *result; + if (r < 0) { switch (errnum) { case EINTR: #ifdef ERESTART @@ -4221,7 +4227,7 @@ wait_retryable(int *result, int errnum, rb_hrtime_t *rel, rb_hrtime_t end) } return FALSE; } - else if (*result == 0) { + else if (r == 0) { /* check for spurious wakeup */ if (rel) { return !hrtime_update_expire(rel, end); @@ -4259,11 +4265,12 @@ static VALUE do_select(VALUE p) { struct select_set *set = (struct select_set *)p; - int result = 0; + volatile int result = 0; int lerrno; rb_hrtime_t *to, rel, end = 0; timeout_prepare(&to, &rel, &end, set->timeout); + volatile rb_hrtime_t endtime = end; #define restore_fdset(dst, src) \ ((dst) ? rb_fd_dup(dst, src) : (void)0) #define do_select_update() \ @@ -4279,15 +4286,15 @@ do_select(VALUE p) struct timeval tv; if (!RUBY_VM_INTERRUPTED(set->th->ec)) { - result = native_fd_select(set->max, - set->rset, set->wset, set->eset, - rb_hrtime2timeval(&tv, to), set->th); + result = native_fd_select(set->max, + set->rset, set->wset, set->eset, + rb_hrtime2timeval(&tv, to), set->th); if (result < 0) lerrno = errno; } }, ubf_select, set->th, TRUE); RUBY_VM_CHECK_INTS_BLOCKING(set->th->ec); /* may raise */ - } while (wait_retryable(&result, lerrno, to, end) && do_select_update()); + } while (wait_retryable(&result, lerrno, to, endtime) && do_select_update()); if (result < 0) { errno = lerrno; @@ -4349,6 +4356,23 @@ rb_thread_fd_select(int max, rb_fdset_t * read, rb_fdset_t * write, rb_fdset_t * # define POLLERR_SET (0) #endif +static int +wait_for_single_fd_blocking_region(rb_thread_t *th, struct pollfd *fds, nfds_t nfds, + rb_hrtime_t *const to, volatile int *lerrno) +{ + struct timespec ts; + volatile int result = 0; + + *lerrno = 0; + BLOCKING_REGION(th, { + if (!RUBY_VM_INTERRUPTED(th->ec)) { + result = ppoll(fds, nfds, rb_hrtime2timespec(&ts, to), 0); + if (result < 0) *lerrno = errno; + } + }, ubf_select, th, TRUE); + return result; +} + /* * returns a mask of events */ @@ -4360,7 +4384,7 @@ rb_thread_wait_for_single_fd(int fd, int events, struct timeval *timeout) .events = (short)events, .revents = 0, }}; - int result = 0; + volatile int result = 0; nfds_t nfds; struct waiting_fd wfd; enum ruby_tag_type state; @@ -4384,17 +4408,8 @@ rb_thread_wait_for_single_fd(int fd, int events, struct timeval *timeout) RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); timeout_prepare(&to, &rel, &end, timeout); do { - nfds = 1; - - lerrno = 0; - BLOCKING_REGION(wfd.th, { - struct timespec ts; - - if (!RUBY_VM_INTERRUPTED(wfd.th->ec)) { - result = ppoll(fds, nfds, rb_hrtime2timespec(&ts, to), 0); - if (result < 0) lerrno = errno; - } - }, ubf_select, wfd.th, TRUE); + nfds = numberof(fds); + result = wait_for_single_fd_blocking_region(wfd.th, fds, nfds, to, &lerrno); RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); } while (wait_retryable(&result, lerrno, to, end)); diff --git a/thread_pthread.c b/thread_pthread.c index 82b5e362cc..b9421559f2 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -2569,10 +2569,7 @@ ubf_wakeup_thread(rb_thread_t *th) { RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); - int r = pthread_kill(th->nt->thread_id, SIGVTALRM); - if (r != 0) { - rb_bug_errno("pthread_kill", r); - } + pthread_kill(th->nt->thread_id, SIGVTALRM); } static void diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index 3ba27f6d64..8730f0fb3a 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -11,7 +11,8 @@ module BundledGem "time", # net-ftp "singleton", # prime "ipaddr", # rinda - "forwardable" # prime, rinda + "forwardable", # prime, rinda + "strscan" # rexml ] module_function diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index d758b5fb02..2b0856b822 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -37,6 +37,26 @@ module Test class PendedError < AssertionFailedError; end + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + module Order class NoSort def initialize(seed) @@ -1778,15 +1798,7 @@ module Test end def location e # :nodoc: - last_before_assertion = "" - - return '<empty>' unless e&.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(?:Test::Unit::(?:Core)?Assertions#)?(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index b4f1dbc176..aad422f7e7 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -768,7 +768,14 @@ EOT e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb index 7f50be873c..3eb92f8ef4 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb @@ -9,6 +9,28 @@ module Lrama @user_code = nil @precedence_sym = nil end + + def resolve_user_code(bindings) + return unless user_code + + var_to_arg = {} + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + user_code.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return user_code + end end end end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb index eb119c005e..38f0fca4ea 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb @@ -2,12 +2,13 @@ module Lrama class Grammar class ParameterizingRule class Rule - attr_reader :name, :parameters, :rhs_list, :required_parameters_count, :is_inline + attr_reader :name, :parameters, :rhs_list, :required_parameters_count, :tag, :is_inline - def initialize(name, parameters, rhs_list, is_inline: false) + def initialize(name, parameters, rhs_list, tag: nil, is_inline: false) @name = name @parameters = parameters @rhs_list = rhs_list + @tag = tag @is_inline = is_inline @required_parameters_count = parameters.count end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb index 4ba92bc804..ccb41e67f8 100644 --- a/tool/lrama/lib/lrama/grammar/rule_builder.rb +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -130,12 +130,12 @@ module Lrama @replaced_rhs << lhs_token parameterizing_rule_resolver.created_lhs_list << lhs_token parameterizing_rule.rhs_list.each do |r| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, lhs_tag: token.lhs_tag) + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, lhs_tag: token.lhs_tag || parameterizing_rule.tag) rule_builder.lhs = lhs_token r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } rule_builder.line = line rule_builder.precedence_sym = r.precedence_sym - rule_builder.user_code = r.user_code + rule_builder.user_code = r.resolve_user_code(bindings) rule_builder.complete_input rule_builder.setup_rules(parameterizing_rule_resolver) @rule_builders_for_parameterizing_rules << rule_builder diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index 24db131c70..04603105b4 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -732,43 +732,43 @@ end ##### State transition tables begin ### racc_action_table = [ - 98, 51, 99, 163, 88, 79, 51, 51, 179, 163, - 79, 79, 51, 162, 179, 156, 79, 165, 157, 51, - 3, 50, 180, 165, 70, 51, 8, 50, 180, 79, + 98, 51, 99, 163, 88, 79, 51, 51, 180, 163, + 79, 79, 51, 162, 180, 156, 79, 165, 157, 51, + 3, 50, 181, 165, 70, 51, 8, 50, 181, 79, 75, 51, 6, 50, 7, 161, 82, 47, 51, 51, 50, 50, 89, 82, 82, 166, 41, 51, 100, 50, - 181, 166, 82, 51, 48, 50, 181, 23, 25, 26, + 182, 166, 82, 51, 48, 50, 182, 23, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 47, 51, 51, 50, 50, 93, 79, 196, - 51, 51, 50, 50, 79, 196, 51, 51, 50, 50, - 79, 196, 23, 25, 26, 27, 28, 29, 30, 31, + 37, 38, 47, 51, 51, 50, 50, 93, 79, 197, + 51, 51, 50, 50, 79, 197, 51, 51, 50, 50, + 79, 197, 23, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 9, 51, 54, 50, 14, 15, 16, 17, 18, 19, 54, 54, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 51, 51, - 50, 50, 79, 196, 51, 51, 50, 50, 79, 196, - 51, 51, 50, 50, 79, 196, 51, 51, 50, 50, + 50, 50, 79, 197, 51, 51, 50, 50, 79, 197, + 51, 51, 50, 50, 79, 197, 51, 51, 50, 50, 79, 79, 51, 51, 50, 50, 79, 79, 51, 51, - 50, 50, 79, 79, 51, 51, 50, 206, 79, 79, - 51, 51, 206, 206, 79, 79, 51, 51, 50, 50, - 79, 186, 187, 188, 96, 186, 187, 188, 96, 216, - 220, 227, 217, 217, 217, 51, 51, 50, 50, 186, - 187, 188, 57, 58, 59, 60, 61, 62, 63, 64, + 50, 50, 79, 79, 51, 51, 50, 207, 79, 79, + 51, 51, 207, 207, 79, 79, 51, 51, 50, 50, + 79, 187, 188, 189, 96, 187, 188, 189, 96, 217, + 221, 229, 218, 218, 218, 51, 51, 50, 50, 187, + 188, 189, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 90, 94, 96, 101, 101, 101, 103, 109, 113, 114, 117, 117, 117, 117, 120, 47, 124, 125, 127, 129, 130, 131, 132, 133, 136, 140, 141, 142, 143, 146, 147, 148, 150, 160, 168, 170, 171, - 172, 173, 174, 175, 176, 177, 146, 183, 191, 192, - 160, 160, 203, 210, 211, 177, 214, 215, 210, 225, - 226, 96, 96, 210 ] + 172, 173, 174, 176, 177, 178, 146, 184, 192, 193, + 200, 160, 204, 176, 211, 160, 215, 216, 178, 176, + 226, 176, 228, 96, 96, 176 ] racc_action_check = [ - 49, 145, 49, 145, 39, 145, 159, 182, 159, 182, - 159, 182, 200, 144, 200, 139, 200, 145, 139, 33, - 1, 33, 159, 182, 33, 34, 3, 34, 200, 34, + 49, 145, 49, 145, 39, 145, 159, 183, 159, 183, + 159, 183, 201, 144, 201, 139, 201, 145, 139, 33, + 1, 33, 159, 183, 33, 34, 3, 34, 201, 34, 34, 35, 2, 35, 2, 144, 35, 9, 36, 37, 36, 37, 39, 36, 37, 145, 7, 38, 49, 38, - 159, 182, 38, 15, 14, 15, 200, 9, 9, 9, + 159, 183, 38, 15, 14, 15, 201, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 42, 69, 172, 69, 172, 42, 172, 172, 173, 70, 173, 70, 173, 173, 174, 81, 174, 81, @@ -776,22 +776,22 @@ racc_action_check = [ 42, 42, 42, 42, 42, 42, 42, 4, 82, 16, 82, 4, 4, 4, 4, 4, 4, 17, 18, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 193, 109, - 193, 109, 193, 193, 197, 111, 197, 111, 197, 197, - 198, 117, 198, 117, 198, 198, 74, 75, 74, 75, + 4, 4, 4, 4, 4, 4, 4, 4, 194, 109, + 194, 109, 194, 194, 198, 111, 198, 111, 198, 198, + 199, 117, 199, 117, 199, 199, 74, 75, 74, 75, 74, 75, 114, 116, 114, 116, 114, 116, 137, 166, - 137, 166, 137, 166, 181, 183, 181, 183, 181, 183, - 203, 215, 203, 215, 203, 215, 217, 119, 217, 119, - 217, 164, 164, 164, 164, 178, 178, 178, 178, 207, - 213, 222, 207, 213, 222, 134, 138, 134, 138, 208, - 208, 208, 19, 20, 23, 25, 26, 27, 28, 29, + 137, 166, 137, 166, 182, 184, 182, 184, 182, 184, + 204, 216, 204, 216, 204, 216, 218, 119, 218, 119, + 218, 164, 164, 164, 164, 179, 179, 179, 179, 208, + 214, 223, 208, 214, 223, 134, 138, 134, 138, 209, + 209, 209, 19, 20, 23, 25, 26, 27, 28, 29, 30, 31, 32, 40, 45, 46, 53, 55, 56, 57, 68, 72, 73, 80, 85, 86, 87, 88, 89, 95, 96, 102, 104, 105, 106, 107, 108, 112, 120, 121, 122, 123, 124, 125, 126, 128, 141, 149, 151, 152, 153, 154, 155, 156, 157, 158, 161, 163, 167, 169, - 175, 177, 179, 185, 189, 199, 204, 206, 216, 219, - 221, 225, 226, 228 ] + 175, 178, 180, 186, 190, 200, 205, 207, 213, 217, + 220, 221, 222, 226, 228, 230 ] racc_action_pointer = [ nil, 20, 22, 26, 98, nil, nil, 39, nil, 33, @@ -809,18 +809,18 @@ racc_action_pointer = [ 243, 207, 239, 249, 206, 201, 252, nil, 253, nil, nil, nil, nil, nil, 202, nil, nil, 165, 203, -26, nil, 210, nil, nil, -10, -2, nil, nil, nil, 237, - nil, 238, 239, 240, 241, 242, 221, 259, 220, 3, + nil, 238, 239, 240, 241, 242, 255, 259, 220, 3, nil, 220, nil, 227, 143, nil, 166, 248, nil, 249, - nil, nil, 71, 77, 83, 224, nil, 225, 147, 232, - nil, 171, 4, 172, nil, 265, nil, nil, nil, 272, - nil, nil, nil, 135, nil, nil, nil, 141, 147, 230, - 9, nil, nil, 177, 274, nil, 237, 158, 161, nil, - nil, nil, nil, 159, nil, 178, 270, 183, nil, 259, - nil, 260, 160, nil, nil, 230, 231, nil, 275, nil, - nil ] + nil, nil, 71, 77, 83, 228, nil, nil, 225, 147, + 232, nil, 171, 4, 172, nil, 265, nil, nil, nil, + 272, nil, nil, nil, 135, nil, nil, nil, 141, 147, + 229, 9, nil, nil, 177, 274, nil, 237, 158, 161, + nil, nil, nil, 233, 159, nil, 178, 271, 183, nil, + 260, 273, 262, 160, nil, nil, 232, nil, 233, nil, + 277, nil, nil ] racc_action_default = [ - -2, -138, -8, -138, -138, -3, -4, -138, 231, -138, + -2, -138, -8, -138, -138, -3, -4, -138, 233, -138, -9, -10, -11, -12, -138, -138, -138, -138, -138, -138, -138, -24, -25, -138, -29, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, -138, @@ -835,75 +835,75 @@ racc_action_default = [ -138, -138, -138, -138, -104, -138, -138, -86, -138, -22, -27, -31, -34, -37, -52, -55, -74, -77, -90, -138, -58, -62, -6, -125, -100, -101, -105, -121, -83, -138, - -19, -138, -138, -138, -138, -138, -138, -138, -57, -60, + -19, -138, -138, -138, -138, -138, -136, -138, -57, -60, -63, -104, -103, -94, -120, -109, -138, -138, -87, -138, - -23, -28, -138, -138, -138, -62, -59, -62, -120, -94, - -67, -138, -102, -138, -106, -136, -113, -114, -115, -138, - -112, -84, -20, -32, -131, -133, -134, -35, -38, -56, - -61, -64, -65, -138, -138, -70, -94, -138, -116, -107, - -137, -110, -132, -138, -68, -138, -136, -138, -118, -138, - -66, -138, -138, -108, -117, -120, -120, -119, -136, -69, - -111 ] + -23, -28, -138, -138, -138, -138, -137, -59, -62, -120, + -94, -67, -138, -102, -138, -106, -136, -113, -114, -115, + -138, -112, -84, -20, -32, -131, -133, -134, -35, -38, + -62, -61, -64, -65, -138, -138, -70, -94, -138, -116, + -107, -110, -132, -56, -138, -68, -138, -136, -138, -118, + -138, -136, -138, -138, -108, -117, -120, -66, -120, -119, + -136, -69, -111 ] racc_goto_table = [ - 76, 74, 69, 52, 209, 95, 110, 118, 185, 145, - 158, 207, 1, 2, 119, 212, 4, 43, 42, 212, - 212, 72, 202, 84, 84, 84, 84, 80, 85, 86, - 87, 213, 53, 55, 56, 223, 193, 197, 198, 111, - 115, 76, 116, 222, 199, 118, 182, 230, 110, 138, - 92, 91, 218, 5, 40, 122, 10, 72, 72, 11, - 12, 13, 49, 97, 118, 128, 169, 104, 151, 84, - 84, 110, 105, 152, 106, 153, 107, 154, 134, 108, - 76, 137, 115, 155, 68, 73, 112, 135, 139, 121, - 200, 204, 221, 126, 167, 102, 149, 72, 144, 72, - 189, 219, 123, 115, nil, 84, nil, 84, nil, nil, - nil, 164, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 72, 184, nil, 178, 84, nil, nil, nil, - nil, nil, 190, nil, nil, nil, nil, 201, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 205, 164, 208, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 178, nil, nil, 208, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 208, nil, 224, 228, 229 ] + 76, 95, 69, 52, 74, 158, 110, 175, 118, 119, + 145, 208, 1, 212, 186, 2, 43, 212, 212, 4, + 42, 72, 91, 84, 84, 84, 84, 5, 40, 203, + 122, 214, 80, 85, 86, 87, 10, 210, 11, 111, + 115, 76, 12, 223, 138, 116, 118, 183, 110, 92, + 53, 55, 56, 194, 198, 199, 13, 72, 72, 219, + 49, 97, 128, 169, 213, 118, 104, 151, 224, 84, + 84, 110, 227, 105, 152, 106, 153, 107, 134, 154, + 76, 232, 115, 108, 137, 155, 68, 73, 112, 135, + 139, 121, 201, 205, 222, 126, 167, 72, 102, 72, + 149, 144, 190, 115, 220, 84, 123, 84, nil, nil, + nil, 164, nil, nil, nil, nil, nil, nil, nil, 185, + nil, nil, 72, nil, nil, 179, 84, nil, nil, nil, + nil, nil, 191, nil, 202, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 206, 164, + 209, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 179, nil, nil, + 209, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, 230, 209, 231, 225 ] racc_goto_check = [ - 42, 48, 33, 35, 60, 43, 34, 55, 44, 59, - 39, 45, 1, 2, 54, 64, 3, 56, 4, 64, - 64, 35, 44, 35, 35, 35, 35, 32, 32, 32, - 32, 45, 15, 15, 15, 60, 21, 21, 21, 33, - 42, 42, 48, 45, 39, 55, 59, 60, 34, 54, - 56, 5, 44, 6, 7, 8, 9, 35, 35, 10, - 11, 12, 13, 14, 55, 16, 17, 18, 19, 35, - 35, 34, 22, 23, 24, 25, 26, 27, 33, 28, - 42, 48, 42, 29, 30, 31, 36, 37, 38, 40, - 41, 46, 47, 50, 51, 52, 53, 35, 58, 35, - 61, 62, 63, 42, nil, 35, nil, 35, nil, nil, - nil, 42, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 35, 43, nil, 42, 35, nil, nil, nil, - nil, nil, 42, nil, nil, nil, nil, 43, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 42, 42, 42, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 42, nil, nil, 42, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 42, nil, 42, 43, 43 ] + 43, 44, 33, 35, 49, 40, 34, 39, 56, 55, + 60, 46, 1, 64, 45, 2, 57, 64, 64, 3, + 4, 35, 5, 35, 35, 35, 35, 6, 7, 45, + 8, 46, 32, 32, 32, 32, 9, 39, 10, 33, + 43, 43, 11, 46, 55, 49, 56, 60, 34, 57, + 15, 15, 15, 21, 21, 21, 12, 35, 35, 45, + 13, 14, 16, 17, 40, 56, 18, 19, 39, 35, + 35, 34, 39, 22, 23, 24, 25, 26, 33, 27, + 43, 39, 43, 28, 49, 29, 30, 31, 36, 37, + 38, 41, 42, 47, 48, 51, 52, 35, 53, 35, + 54, 59, 61, 43, 62, 35, 63, 35, nil, nil, + nil, 43, nil, nil, nil, nil, nil, nil, nil, 44, + nil, nil, 35, nil, nil, 43, 35, nil, nil, nil, + nil, nil, 43, nil, 44, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 43, 43, + 43, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 43, nil, nil, + 43, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, 44, 43, 44, 43 ] racc_goto_pointer = [ - nil, 12, 13, 14, 9, 9, 51, 48, -35, 52, - 55, 56, 57, 47, 14, 16, -38, -84, 9, -61, - nil, -136, 13, -57, 14, -56, 15, -55, 17, -50, - 51, 51, -8, -31, -63, -12, 14, -25, -32, -131, - 0, -87, -34, -41, -156, -172, -89, -122, -33, nil, - -8, -54, 41, -31, -68, -74, 8, nil, -26, -115, - -181, -65, -110, 9, -178 ] + nil, 12, 15, 17, 11, -20, 25, 22, -60, 32, + 34, 38, 52, 45, 12, 34, -41, -87, 8, -62, + nil, -119, 14, -56, 15, -55, 16, -53, 21, -48, + 53, 53, -3, -31, -63, -12, 16, -23, -30, -149, + -136, 2, -86, -34, -45, -150, -173, -88, -121, -30, + nil, -6, -52, 44, -27, -73, -73, 7, nil, -23, + -114, -63, -107, 13, -181 ] racc_goto_default = [ nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 45, nil, nil, nil, nil, nil, nil, nil, nil, nil, 24, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 71, 77, nil, nil, nil, nil, - 46, 159, 195, nil, nil, nil, nil, nil, nil, 78, - nil, nil, nil, nil, 81, 83, nil, 44, nil, nil, - nil, nil, nil, nil, 194 ] + nil, 46, 159, 196, nil, nil, nil, nil, nil, nil, + 78, nil, nil, nil, nil, 81, 83, nil, 44, nil, + nil, nil, nil, nil, 195 ] racc_reduce_table = [ 0, 0, :racc_error, @@ -962,21 +962,21 @@ racc_reduce_table = [ 1, 87, :_reduce_53, 2, 87, :_reduce_54, 3, 88, :_reduce_55, - 7, 65, :_reduce_56, + 8, 65, :_reduce_56, 5, 66, :_reduce_57, 1, 92, :_reduce_58, 3, 92, :_reduce_59, - 1, 93, :_reduce_60, - 3, 93, :_reduce_61, - 0, 95, :_reduce_62, - 1, 95, :_reduce_63, - 3, 95, :_reduce_64, - 3, 95, :_reduce_65, - 5, 95, :_reduce_66, - 0, 100, :_reduce_67, - 0, 101, :_reduce_68, - 7, 95, :_reduce_69, - 3, 95, :_reduce_70, + 1, 94, :_reduce_60, + 3, 94, :_reduce_61, + 0, 96, :_reduce_62, + 1, 96, :_reduce_63, + 3, 96, :_reduce_64, + 3, 96, :_reduce_65, + 6, 96, :_reduce_66, + 0, 101, :_reduce_67, + 0, 102, :_reduce_68, + 7, 96, :_reduce_69, + 3, 96, :_reduce_70, 0, 90, :_reduce_none, 1, 90, :_reduce_none, 0, 91, :_reduce_none, @@ -984,51 +984,51 @@ racc_reduce_table = [ 1, 85, :_reduce_75, 2, 85, :_reduce_76, 3, 85, :_reduce_77, - 1, 102, :_reduce_78, - 2, 102, :_reduce_79, - 1, 96, :_reduce_none, - 1, 96, :_reduce_none, - 0, 104, :_reduce_82, - 0, 105, :_reduce_83, + 1, 103, :_reduce_78, + 2, 103, :_reduce_79, + 1, 97, :_reduce_none, + 1, 97, :_reduce_none, + 0, 105, :_reduce_82, + 0, 106, :_reduce_83, 6, 69, :_reduce_84, - 0, 106, :_reduce_85, - 0, 107, :_reduce_86, + 0, 107, :_reduce_85, + 0, 108, :_reduce_86, 5, 69, :_reduce_87, 1, 86, :_reduce_88, 2, 86, :_reduce_89, 3, 86, :_reduce_90, - 1, 108, :_reduce_91, - 2, 108, :_reduce_92, - 1, 109, :_reduce_none, + 1, 109, :_reduce_91, + 2, 109, :_reduce_92, + 1, 110, :_reduce_none, 1, 89, :_reduce_94, 1, 89, :_reduce_95, 1, 58, :_reduce_none, 2, 58, :_reduce_none, - 1, 110, :_reduce_none, - 2, 110, :_reduce_none, - 4, 111, :_reduce_100, - 1, 112, :_reduce_101, - 3, 112, :_reduce_102, - 2, 112, :_reduce_none, - 0, 113, :_reduce_104, - 1, 113, :_reduce_105, - 3, 113, :_reduce_106, - 4, 113, :_reduce_107, - 6, 113, :_reduce_108, + 1, 111, :_reduce_none, + 2, 111, :_reduce_none, + 4, 112, :_reduce_100, + 1, 113, :_reduce_101, + 3, 113, :_reduce_102, + 2, 113, :_reduce_none, + 0, 114, :_reduce_104, + 1, 114, :_reduce_105, + 3, 114, :_reduce_106, + 4, 114, :_reduce_107, + 6, 114, :_reduce_108, 0, 115, :_reduce_109, 0, 116, :_reduce_110, - 8, 113, :_reduce_111, - 3, 113, :_reduce_112, - 1, 98, :_reduce_113, - 1, 98, :_reduce_114, - 1, 98, :_reduce_115, - 1, 99, :_reduce_116, - 3, 99, :_reduce_117, - 2, 99, :_reduce_118, - 4, 99, :_reduce_119, - 0, 97, :_reduce_none, - 3, 97, :_reduce_121, - 1, 94, :_reduce_none, + 8, 114, :_reduce_111, + 3, 114, :_reduce_112, + 1, 99, :_reduce_113, + 1, 99, :_reduce_114, + 1, 99, :_reduce_115, + 1, 100, :_reduce_116, + 3, 100, :_reduce_117, + 2, 100, :_reduce_118, + 4, 100, :_reduce_119, + 0, 98, :_reduce_none, + 3, 98, :_reduce_121, + 1, 95, :_reduce_none, 0, 59, :_reduce_none, 0, 117, :_reduce_124, 3, 59, :_reduce_125, @@ -1041,13 +1041,13 @@ racc_reduce_table = [ 2, 75, :_reduce_132, 1, 118, :_reduce_none, 1, 118, :_reduce_none, - 1, 103, :_reduce_135, - 0, 114, :_reduce_none, - 1, 114, :_reduce_none ] + 1, 104, :_reduce_135, + 0, 93, :_reduce_none, + 1, 93, :_reduce_none ] racc_reduce_n = 138 -racc_shift_n = 231 +racc_shift_n = 233 racc_token_table = { false => 0, @@ -1220,6 +1220,7 @@ Racc_token_to_s_table = [ "int_opt", "alias", "rule_args", + "tag_opt", "rule_rhs_list", "id_colon", "rule_rhs", @@ -1241,7 +1242,6 @@ Racc_token_to_s_table = [ "rules", "rhs_list", "rhs", - "tag_opt", "@21", "@22", "@23", @@ -1666,7 +1666,7 @@ module_eval(<<'.,.,', 'parser.y', 232) module_eval(<<'.,.,', 'parser.y', 236) def _reduce_56(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[6]) + rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) @grammar.add_parameterizing_rule(rule) result @@ -1757,7 +1757,7 @@ module_eval(<<'.,.,', 'parser.y', 280) module_eval(<<'.,.,', 'parser.y', 286) def _reduce_66(val, _values, result) builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3]) + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) result = builder result diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 6b1976c731..ef840ce435 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,3 +1,3 @@ module Lrama - VERSION = "0.6.7".freeze + VERSION = "0.6.9".freeze end diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..171455549e --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADER([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/rjit/bindgen.rb b/tool/rjit/bindgen.rb index fb6653ed9c..2fc4576216 100755 --- a/tool/rjit/bindgen.rb +++ b/tool/rjit/bindgen.rb @@ -540,6 +540,7 @@ generator = BindingGenerator.new( rb_vm_opt_newarray_min rb_vm_opt_newarray_max rb_vm_opt_newarray_hash + rb_vm_opt_newarray_pack rb_vm_setinstancevariable rb_vm_splat_array rjit_full_cfunc_return diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 709b495572..1e19c102b8 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -50,4 +50,17 @@ class TestAssertion < Test::Unit::TestCase assert_pattern_list(pattern_list, actual, message) end end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end end diff --git a/universal_parser.c b/universal_parser.c index fa30fcd221..c5e557ca30 100644 --- a/universal_parser.c +++ b/universal_parser.c @@ -59,7 +59,7 @@ #undef st_lookup #define st_lookup rb_parser_st_lookup -#define rb_encoding void +#define rb_encoding const void #undef xmalloc #define xmalloc p->config->malloc @@ -168,6 +168,7 @@ #define rb_ascii8bit_encoding p->config->ascii8bit_encoding #define rb_enc_codelen p->config->enc_codelen #define rb_enc_mbcput p->config->enc_mbcput +#define rb_enc_mbclen p->config->enc_mbclen #define rb_enc_find_index p->config->enc_find_index #define rb_enc_from_index p->config->enc_from_index #define rb_enc_isspace p->config->enc_isspace @@ -1480,14 +1480,14 @@ rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const I tmp_node.nd_body = 0; tmp_node.nd_args = 0; - VALUE vast = rb_ruby_ast_new(RNODE(&tmp_node)); + VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node)); if (base_iseq) { - iseq = rb_iseq_new(vast, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL); + iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL); } else { VALUE tempstr = rb_fstring_lit("<temp>"); - iseq = rb_iseq_new_top(vast, tempstr, tempstr, tempstr, NULL); + iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL); } tmp_node.nd_tbl = 0; /* reset table */ ALLOCV_END(idtmp); @@ -2278,6 +2278,7 @@ vm_redefinition_bop_for_id(ID mid) OP(NilP, NIL_P); OP(Cmp, CMP); OP(Default, DEFAULT); + OP(Pack, PACK); #undef OP } return -1; @@ -3077,6 +3078,7 @@ ruby_vm_destruct(rb_vm_t *vm) rb_vm_postponed_job_free(); rb_id_table_free(vm->constant_cache); + st_free_table(vm->unused_block_warning_table); if (th) { xfree(th->nt); @@ -684,15 +684,20 @@ setup_parameters_complex(rb_execution_context_t * const ec, const rb_iseq_t * co if (RB_TYPE_P(rest_last, T_HASH) && FL_TEST_RAW(rest_last, RHASH_PASS_AS_KEYWORDS)) { // def f(**kw); a = [..., kw]; g(*a) splat_flagged_keyword_hash = rest_last; - rest_last = rb_hash_dup(rest_last); + if (!RHASH_EMPTY_P(rest_last) || (ISEQ_BODY(iseq)->param.flags.has_kwrest)) { + rest_last = rb_hash_dup(rest_last); + } kw_flag |= VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT; // Unset rest_dupped set by anon_rest as we may need to modify splat in this case args->rest_dupped = false; if (ignore_keyword_hash_p(rest_last, iseq, &kw_flag, &converted_keyword_hash)) { - arg_rest_dup(args); - rb_ary_pop(args->rest); + if (ISEQ_BODY(iseq)->param.flags.has_rest || arg_setup_type != arg_setup_method) { + // Only duplicate/modify splat array if it will be used + arg_rest_dup(args); + rb_ary_pop(args->rest); + } given_argc--; kw_flag &= ~(VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT); } @@ -1209,11 +1209,11 @@ typedef enum { RUBY_SYMBOL_EXPORT_BEGIN /* node -> iseq */ -rb_iseq_t *rb_iseq_new (const VALUE vast, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, enum rb_iseq_type); -rb_iseq_t *rb_iseq_new_top (const VALUE vast, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent); -rb_iseq_t *rb_iseq_new_main (const VALUE vast, VALUE path, VALUE realpath, const rb_iseq_t *parent, int opt); -rb_iseq_t *rb_iseq_new_eval (const VALUE vast, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth); -rb_iseq_t *rb_iseq_new_with_opt(const VALUE vast, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth, +rb_iseq_t *rb_iseq_new (const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, enum rb_iseq_type); +rb_iseq_t *rb_iseq_new_top (const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent); +rb_iseq_t *rb_iseq_new_main (const VALUE ast_value, VALUE path, VALUE realpath, const rb_iseq_t *parent, int opt); +rb_iseq_t *rb_iseq_new_eval (const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth); +rb_iseq_t *rb_iseq_new_with_opt(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, int first_lineno, const rb_iseq_t *parent, int isolated_depth, enum rb_iseq_type, const rb_compile_option_t*, VALUE script_lines); @@ -1629,6 +1629,10 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, const rb_iseq_t *const parent = vm_block_iseq(base_block); const rb_iseq_t *iseq = parent; VALUE name = rb_fstring_lit("<compiled>"); + + // Conditionally enable coverage depending on the current mode: + int coverage_enabled = ((rb_get_coverage_mode() & COVERAGE_TARGET_EVAL) != 0) ? 1 : 0; + if (!fname) { fname = rb_source_location(&line); } @@ -1638,10 +1642,12 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, } else { fname = get_eval_default_path(); + coverage_enabled = 0; } pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, line); + result.node.coverage_enabled = coverage_enabled; // Cout scopes, one for each parent iseq, plus one for our local scope int scopes_count = 0; @@ -1703,6 +1709,7 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, RUBY_ASSERT(parent_scope != NULL); pm_options_scope_t *options_scope = &result.options.scopes[scopes_count - scopes_index - 1]; + parent_scope->coverage_enabled = coverage_enabled; parent_scope->parser = &result.parser; parent_scope->index_lookup_table = st_init_numtable(); @@ -1753,7 +1760,7 @@ eval_make_iseq(VALUE src, VALUE fname, int line, const VALUE parser = rb_parser_new(); const rb_iseq_t *const parent = vm_block_iseq(base_block); rb_iseq_t *iseq = NULL; - VALUE vast; + VALUE ast_value; rb_ast_t *ast; int isolated_depth = 0; @@ -1791,13 +1798,13 @@ eval_make_iseq(VALUE src, VALUE fname, int line, rb_parser_set_context(parser, parent, FALSE); if (ruby_vm_keep_script_lines) rb_parser_set_script_lines(parser); - vast = rb_parser_compile_string_path(parser, fname, src, line); + ast_value = rb_parser_compile_string_path(parser, fname, src, line); - ast = rb_ruby_ast_data_get(vast); + ast = rb_ruby_ast_data_get(ast_value); if (ast->body.root) { ast->body.coverage_enabled = coverage_enabled; - iseq = rb_iseq_new_eval(vast, + iseq = rb_iseq_new_eval(ast_value, ISEQ_BODY(parent)->location.label, fname, Qnil, line, parent, isolated_depth); diff --git a/vm_insnhelper.c b/vm_insnhelper.c index a1893b1ba2..1e1fcea120 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -5932,6 +5932,22 @@ rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t num, const VALUE *p return vm_opt_newarray_hash(ec, num, ptr); } +VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len, bool freeze); +VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer); + +VALUE +rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr, VALUE fmt) +{ + if (BASIC_OP_UNREDEFINED_P(BOP_PACK, ARRAY_REDEFINED_OP_FLAG)) { + struct RArray fake_ary; + VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, num, true); + return rb_ec_pack_ary(ec, ary, fmt, Qnil); + } + else { + return rb_vm_call_with_refinements(ec, rb_ary_new4(num, ptr), idPack, 1, &fmt, RB_PASS_CALLED_KEYWORDS); + } +} + #undef id_cmp #define IMEMO_CONST_CACHE_SHAREABLE IMEMO_FL_USER0 @@ -985,7 +985,7 @@ wkmap_inspect(VALUE self) * * Keys in the map are compared by identity. * - * m = ObjectSpace::WeekMap.new + * m = ObjectSpace::WeakMap.new * key1 = "foo" * val1 = Object.new * m[key1] = val1 @@ -1041,13 +1041,13 @@ wkmap_inspect(VALUE self) * * val = nil * GC.start - * # There is no more references to `val`, yet the pair isn't + * # There are no more references to `val`, yet the pair isn't * # garbage-collected. * map["name"] #=> 2023-12-07 00:00:00 +0200 * * key = nil * GC.start - * # There is no more references to `key`, key and value are + * # There are no more references to `key`, key and value are * # garbage-collected. * map["name"] #=> nil * diff --git a/win32/Makefile.sub b/win32/Makefile.sub index 7dbb020eee..61cca10857 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -160,6 +160,9 @@ OPTFLAGS = -O2b2xg- OPTFLAGS = -O2sy- !endif !endif +!if !defined(incflags) +incflags = +!endif !if !defined(PLATFORM) PLATFORM = mswin32 !endif @@ -970,6 +973,7 @@ s,@WERRORFLAG@,$(WERRORFLAG),;t t s,@DEFS@,$(DEFS),;t t s,@CPPFLAGS@,$(CPPFLAGS),;t t s,@CXXFLAGS@,$(CXXFLAGS),;t t +s,@incflags@,$(incflags),;t t s,@FFLAGS@,$(FFLAGS),;t t s,@LDFLAGS@,$(LDFLAGS),;t t s,@LIBS@,user32.lib,;t t @@ -201,13 +201,27 @@ module RubyVM::YJIT # If a method or proc is passed in, get its iseq iseq = RubyVM::InstructionSequence.of(iseq) - if self.enabled? - # Produce the disassembly string - # Include the YARV iseq disasm in the string for additional context - iseq.disasm + "\n" + Primitive.rb_yjit_disasm_iseq(iseq) - else - iseq.disasm + if !self.enabled? + warn( + "YJIT needs to be enabled to produce disasm output, e.g.\n" + + "ruby --yjit-call-threshold=1 my_script.rb (see doc/yjit/yjit.md)" + ) + return nil end + + disasm_str = Primitive.rb_yjit_disasm_iseq(iseq) + + if !disasm_str + warn( + "YJIT disasm is only available when YJIT is built in dev mode, i.e.\n" + + "./configure --enable-yjit=dev (see doc/yjit/yjit.md)\n" + ) + return nil + end + + # Produce the disassembly string + # Include the YARV iseq disasm in the string for additional context + iseq.disasm + "\n" + disasm_str end # Produce a list of instructions compiled by YJIT for an iseq diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 953ab0ac42..a7473c1bf6 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -380,6 +380,7 @@ fn main() { // From internal/object.h .allowlist_function("rb_class_allocate_instance") + .allowlist_function("rb_obj_equal") // From gc.h and internal/gc.h .allowlist_function("rb_obj_info") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 000f9fb516..747fa1769d 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -250,6 +250,33 @@ impl JITState { } } + pub fn assume_expected_cfunc( + &mut self, + asm: &mut Assembler, + ocb: &mut OutlinedCb, + class: VALUE, + method: ID, + cfunc: *mut c_void, + ) -> bool { + let cme = unsafe { rb_callable_method_entry(class, method) }; + + if cme.is_null() { + return false; + } + + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { + return false; + } + if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc { + return false; + } + + self.assume_method_lookup_stable(asm, ocb, cme); + + true + } + pub fn assume_method_lookup_stable(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, cme: CmePtr) -> Option<()> { jit_ensure_block_entry_exit(self, asm, ocb)?; self.method_lookup_assumptions.push(cme); @@ -4208,11 +4235,50 @@ fn gen_opt_newarray_send( gen_opt_newarray_max(jit, asm, _ocb) } else if method == ID!(hash) { gen_opt_newarray_hash(jit, asm, _ocb) + } else if method == ID!(pack) { + gen_opt_newarray_pack(jit, asm, _ocb) } else { None } } +fn gen_opt_newarray_pack( + jit: &mut JITState, + asm: &mut Assembler, + _ocb: &mut OutlinedCb, +) -> Option<CodegenStatus> { + // num == 4 ( for this code ) + let num = jit.get_arg(0).as_u32(); + + // Save the PC and SP because we may call #pack + jit_prepare_non_leaf_call(jit, asm); + + extern "C" { + fn rb_vm_opt_newarray_pack(ec: EcPtr, num: u32, elts: *const VALUE, fmt: VALUE) -> VALUE; + } + + let values_opnd = asm.ctx.sp_opnd(-(num as i32)); + let values_ptr = asm.lea(values_opnd); + + let fmt_string = asm.ctx.sp_opnd(-1); + + let val_opnd = asm.ccall( + rb_vm_opt_newarray_pack as *const u8, + vec![ + EC, + (num - 1).into(), + values_ptr, + fmt_string + ], + ); + + asm.stack_pop(num.as_usize()); + let stack_ret = asm.stack_push(Type::CString); + asm.mov(stack_ret, val_opnd); + + Some(KeepCompiling) +} + fn gen_opt_newarray_hash( jit: &mut JITState, asm: &mut Assembler, @@ -6164,6 +6230,34 @@ fn jit_rb_class_superclass( true } +fn jit_rb_case_equal( + jit: &mut JITState, + asm: &mut Assembler, + ocb: &mut OutlinedCb, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + _argc: i32, + known_recv_class: Option<VALUE>, +) -> bool { + if !jit.assume_expected_cfunc( asm, ocb, known_recv_class.unwrap(), ID!(eq), rb_obj_equal as _) { + return false; + } + + asm_comment!(asm, "case_equal: {}#===", get_class_name(known_recv_class)); + + // Compare the arguments + let arg1 = asm.stack_pop(1); + let arg0 = asm.stack_pop(1); + asm.cmp(arg0, arg1); + let ret_opnd = asm.csel_e(Qtrue.into(), Qfalse.into()); + + let stack_ret = asm.stack_push(Type::UnknownImm); + asm.mov(stack_ret, ret_opnd); + + true +} + fn jit_thread_s_current( _jit: &mut JITState, asm: &mut Assembler, @@ -8254,6 +8348,13 @@ fn gen_struct_aref( } } + if c_method_tracing_currently_enabled(jit) { + // Struct accesses need fire c_call and c_return events, which we can't support + // See :attr-tracing: + gen_counter_incr(asm, Counter::send_cfunc_tracing); + return None; + } + // This is a .send call and we need to adjust the stack if flags & VM_CALL_OPT_SEND != 0 { handle_opt_send_shift_stack(asm, argc); @@ -8298,6 +8399,13 @@ fn gen_struct_aset( return None; } + if c_method_tracing_currently_enabled(jit) { + // Struct accesses need fire c_call and c_return events, which we can't support + // See :attr-tracing: + gen_counter_incr(asm, Counter::send_cfunc_tracing); + return None; + } + // This is a .send call and we need to adjust the stack if flags & VM_CALL_OPT_SEND != 0 { handle_opt_send_shift_stack(asm, argc); @@ -8564,10 +8672,8 @@ fn gen_send_general( // Handling the C method tracing events for attr_accessor // methods is easier than regular C methods as we know the // "method" we are calling into never enables those tracing - // events. Once global invalidation runs, the code for the - // attr_accessor is invalidated and we exit at the closest - // instruction boundary which is always outside of the body of - // the attr_accessor code. + // events. We are never inside the code that needs to be + // invalidated when invalidation happens. gen_counter_incr(asm, Counter::send_cfunc_tracing); return None; } @@ -8827,11 +8933,16 @@ fn gen_send_general( } } +/// Get class name from a class pointer. +fn get_class_name(class: Option<VALUE>) -> String { + class.and_then(|class| unsafe { + cstr_to_rust_string(rb_class2name(class)) + }).unwrap_or_else(|| "Unknown".to_string()) +} + /// Assemble "{class_name}#{method_name}" from a class pointer and a method ID fn get_method_name(class: Option<VALUE>, mid: u64) -> String { - let class_name = class.and_then(|class| unsafe { - cstr_to_rust_string(rb_class2name(class)) - }).unwrap_or_else(|| "Unknown".to_string()); + let class_name = get_class_name(class); let method_name = if mid != 0 { unsafe { cstr_to_rust_string(rb_id2name(mid)) } } else { @@ -10166,6 +10277,10 @@ pub fn yjit_reg_method_codegen_fns() { yjit_reg_method(rb_cString, "<<", jit_rb_str_concat); yjit_reg_method(rb_cString, "+@", jit_rb_str_uplus); + yjit_reg_method(rb_cNilClass, "===", jit_rb_case_equal); + yjit_reg_method(rb_cTrueClass, "===", jit_rb_case_equal); + yjit_reg_method(rb_cFalseClass, "===", jit_rb_case_equal); + yjit_reg_method(rb_cArray, "empty?", jit_rb_ary_empty_p); yjit_reg_method(rb_cArray, "length", jit_rb_ary_length); yjit_reg_method(rb_cArray, "size", jit_rb_ary_length); diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index d07262ad4f..53586cb4f4 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -83,7 +83,7 @@ #![allow(non_upper_case_globals)] use std::convert::From; -use std::ffi::CString; +use std::ffi::{CString, CStr}; use std::os::raw::{c_char, c_int, c_uint}; use std::panic::{catch_unwind, UnwindSafe}; @@ -208,8 +208,6 @@ pub use rb_RCLASS_ORIGIN as RCLASS_ORIGIN; /// Helper so we can get a Rust string for insn_name() pub fn insn_name(opcode: usize) -> String { - use std::ffi::CStr; - unsafe { // Look up Ruby's NULL-terminated insn name string let op_name = raw_insn_name(VALUE(opcode)); @@ -608,7 +606,6 @@ pub fn rust_str_to_sym(str: &str) -> VALUE { pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { assert!(c_char_ptr != std::ptr::null()); - use std::ffi::CStr; let c_str: &CStr = unsafe { CStr::from_ptr(c_char_ptr) }; match c_str.to_str() { @@ -620,17 +617,20 @@ pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { /// A location in Rust code for integrating with debugging facilities defined in C. /// Use the [src_loc!] macro to crate an instance. pub struct SourceLocation { - pub file: CString, + pub file: &'static CStr, pub line: c_int, } /// Make a [SourceLocation] at the current spot. macro_rules! src_loc { () => { - // NOTE(alan): `CString::new` allocates so we might want to limit this to debug builds. - $crate::cruby::SourceLocation { - file: std::ffi::CString::new(file!()).unwrap(), // ASCII source file paths - line: line!().try_into().unwrap(), // not that many lines + { + // Nul-terminated string with static lifetime, make a CStr out of it safely. + let file: &'static str = concat!(file!(), '\0'); + $crate::cruby::SourceLocation { + file: unsafe { std::ffi::CStr::from_ptr(file.as_ptr().cast()) }, + line: line!().try_into().unwrap(), + } } }; } @@ -668,17 +668,16 @@ where Err(_) => { // Theoretically we can recover from some of these panics, // but it's too late if the unwind reaches here. - use std::{process, str}; let _ = catch_unwind(|| { // IO functions can panic too. eprintln!( "YJIT panicked while holding VM lock acquired at {}:{}. Aborting...", - str::from_utf8(loc.file.as_bytes()).unwrap_or("<not utf8>"), + loc.file.to_string_lossy(), line, ); }); - process::abort(); + std::process::abort(); } }; @@ -803,8 +802,10 @@ pub(crate) mod ids { name: min content: b"min" name: max content: b"max" name: hash content: b"hash" + name: pack content: b"pack" name: respond_to_missing content: b"respond_to_missing?" name: to_ary content: b"to_ary" + name: eq content: b"==" } } diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 70578ec7e9..d53e630a4a 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -342,7 +342,8 @@ pub const BOP_AND: ruby_basic_operators = 28; pub const BOP_OR: ruby_basic_operators = 29; pub const BOP_CMP: ruby_basic_operators = 30; pub const BOP_DEFAULT: ruby_basic_operators = 31; -pub const BOP_LAST_: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 32; +pub const BOP_LAST_: ruby_basic_operators = 33; pub type ruby_basic_operators = u32; pub type rb_serial_t = ::std::os::raw::c_ulonglong; pub const imemo_env: imemo_type = 0; @@ -1024,6 +1025,7 @@ extern "C" { pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; pub fn rb_obj_info_dump(obj: VALUE); pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; + pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE; pub fn rb_ary_tmp_new_from_values( arg1: VALUE, diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index f32d51131d..6639fd677b 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -549,21 +549,23 @@ pub extern "C" fn rb_yjit_invalidate_no_singleton_class(klass: VALUE) { // We apply this optimization only to Array, Hash, and String for now. if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&klass) } { - let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; - match no_singleton_classes.get_mut(&klass) { - Some(blocks) => { - // Invalidate existing blocks and let has_singleton_class_of() - // return true when they are compiled again - for block in mem::take(blocks) { - invalidate_block_version(&block); - incr_counter!(invalidate_no_singleton_class); + with_vm_lock(src_loc!(), || { + let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; + match no_singleton_classes.get_mut(&klass) { + Some(blocks) => { + // Invalidate existing blocks and let has_singleton_class_of() + // return true when they are compiled again + for block in mem::take(blocks) { + invalidate_block_version(&block); + incr_counter!(invalidate_no_singleton_class); + } + } + None => { + // Let has_singleton_class_of() return true for this class + no_singleton_classes.insert(klass, HashSet::new()); } } - None => { - // Let has_singleton_class_of() return true for this class - no_singleton_classes.insert(klass, HashSet::new()); - } - } + }); } } @@ -576,23 +578,24 @@ pub extern "C" fn rb_yjit_invalidate_ep_is_bp(iseq: IseqPtr) { return; } - // If an EP escape for this ISEQ is detected for the first time, invalidate all blocks - // associated to the ISEQ. - let no_ep_escape_iseqs = &mut Invariants::get_instance().no_ep_escape_iseqs; - match no_ep_escape_iseqs.get_mut(&iseq) { - Some(blocks) => { - // Invalidate existing blocks and let jit.ep_is_bp() - // return true when they are compiled again - for block in mem::take(blocks) { - invalidate_block_version(&block); - incr_counter!(invalidate_no_singleton_class); + with_vm_lock(src_loc!(), || { + // If an EP escape for this ISEQ is detected for the first time, invalidate all blocks + // associated to the ISEQ. + let no_ep_escape_iseqs = &mut Invariants::get_instance().no_ep_escape_iseqs; + match no_ep_escape_iseqs.get_mut(&iseq) { + Some(blocks) => { + // Invalidate existing blocks and make jit.ep_is_bp() return false + for block in mem::take(blocks) { + invalidate_block_version(&block); + incr_counter!(invalidate_ep_escape); + } + } + None => { + // Let jit.ep_is_bp() return false for this ISEQ + no_ep_escape_iseqs.insert(iseq, HashSet::new()); } } - None => { - // Let jit.ep_is_bp() return false for this ISEQ - no_ep_escape_iseqs.insert(iseq, HashSet::new()); - } - } + }); } // Invalidate all generated code and patch C method return code to contain diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index 0a63fab8b0..6ffe28f12a 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -268,7 +268,7 @@ macro_rules! make_counters { /// The list of counters that are available without --yjit-stats. /// They are incremented only by `incr_counter!` and don't use `gen_counter_incr`. -pub const DEFAULT_COUNTERS: [Counter; 16] = [ +pub const DEFAULT_COUNTERS: [Counter; 17] = [ Counter::code_gc_count, Counter::compiled_iseq_entry, Counter::cold_iseq_entry, @@ -286,6 +286,7 @@ pub const DEFAULT_COUNTERS: [Counter; 16] = [ Counter::invalidate_constant_state_bump, Counter::invalidate_constant_ic_fill, Counter::invalidate_no_singleton_class, + Counter::invalidate_ep_escape, ]; /// Macro to increase a counter by name and count @@ -568,6 +569,7 @@ make_counters! { invalidate_constant_state_bump, invalidate_constant_ic_fill, invalidate_no_singleton_class, + invalidate_ep_escape, // Currently, it's out of the ordinary (might be impossible) for YJIT to leave gaps in // executable memory, so this should be 0. |