diff options
Diffstat (limited to 'tool')
228 files changed, 11539 insertions, 4409 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile index 138adc48de..d1fb1839c9 100644 --- a/tool/annocheck/Dockerfile +++ b/tool/annocheck/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest RUN dnf -y install annobin-annocheck WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy index 0a79f3a50a..d437f27387 100644 --- a/tool/annocheck/Dockerfile-copy +++ b/tool/annocheck/Dockerfile-copy @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest ARG IN_DIR RUN dnf -y install annobin-annocheck diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..38adf9fdb7 --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,172 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + + UPSTREAM_COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + UPSTREAM_COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + REDMINE_TICKET_PATTERN = /\[(Bug|Feature|Misc)\s*#(\d+)\]/ + REDMINE_COMMENT_PREFIX = 'This pull request references the following Redmine tickets:' + + FORK_COMMENT_PREFIX = 'It looks like this pull request was filed from a branch in ruby/ruby.' + FORK_COMMENT_BODY = <<~COMMENT + #{FORK_COMMENT_PREFIX} + + Since ruby/ruby is bi-directionally mirrored with the official git repository at git.ruby-lang.org, \ + having topic branches in ruby/ruby makes it harder to manage the mirror. + + Could you please close this pull request and re-file it from a branch in your personal fork instead? \ + You can fork https://github.com/ruby/ruby, push your branch there, and open a new pull request from it. + + Thank you for your contribution! + COMMENT + + def initialize(client) + @client = client + end + + def review(pr_number) + existing_comments = fetch_existing_comments(pr_number) + pr = @client.get("/repos/#{REPO}/pulls/#{pr_number}") + review_non_fork_branch(pr_number, pr, existing_comments) + review_upstream_repos(pr_number, existing_comments) + review_redmine_links(pr_number, pr, existing_comments) + end + + private + + def fetch_existing_comments(pr_number) + comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + comments.map { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + end + + def already_commented?(existing_comments, prefix) + existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(prefix) } + end + + def post_comment(pr_number, comment) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + # Suggest re-filing from a fork if the PR branch is in ruby/ruby itself + def review_non_fork_branch(pr_number, pr, existing_comments) + if already_commented?(existing_comments, FORK_COMMENT_PREFIX) + puts "Skipped: The PR ##{pr_number} already has a fork branch comment." + return + end + + head_repo = pr.dig(:head, :repo, :full_name) + if head_repo != REPO + puts "Skipped: The PR ##{pr_number} is already from a fork (#{head_repo})." + return + end + + author = pr.dig(:user, :login) + if author == 'dependabot[bot]' + puts "Skipped: The PR ##{pr_number} is from dependabot." + return + end + + post_comment(pr_number, FORK_COMMENT_BODY) + end + + # Suggest filing PRs to upstream repositories for files that have one + def review_upstream_repos(pr_number, existing_comments) + if already_commented?(existing_comments, UPSTREAM_COMMENT_PREFIX) + puts "Skipped: The PR ##{pr_number} already has an upstream repos comment." + return + end + + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + post_comment(pr_number, format_upstream_comment(upstream_repos)) + end + + def review_redmine_links(pr_number, pr, existing_comments) + if already_commented?(existing_comments, REDMINE_COMMENT_PREFIX) + puts "Skipped: The PR ##{pr_number} already has a Redmine links comment." + return + end + + text = "#{pr[:title]}\n#{pr[:body]}" + + tickets = text.scan(REDMINE_TICKET_PATTERN).uniq + tickets.reject! { |_, number| text.include?("https://bugs.ruby-lang.org/issues/#{number}") } + if tickets.empty? + puts "Skipped: The PR ##{pr_number} doesn't reference any Redmine tickets." + return + end + + post_comment(pr_number, format_redmine_comment(tickets)) + end + + def format_redmine_comment(tickets) + comment = +"#{REDMINE_COMMENT_PREFIX}\n\n" + tickets.each do |type, number| + comment << "* [#{type} ##{number}](https://bugs.ruby-lang.org/issues/#{number})\n" + end + comment + end + + def format_upstream_comment(upstream_repos) + comment = +'' + comment << "#{UPSTREAM_COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{UPSTREAM_COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb index 53e460f5b7..c8e4d5345c 100644 --- a/tool/bundler/dev_gems.rb +++ b/tool/bundler/dev_gems.rb @@ -3,15 +3,18 @@ source "https://rubygems.org" gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" gem "rake", "~> 13.1" -gem "rb_sys" +gem "rb_sys", ">= 0.9.128" gem "turbo_tests", "~> 2.2.3" -gem "parallel_tests", "~> 4.7" +gem "parallel_tests", "~> 4.10.1" gem "parallel", "~> 1.19" gem "rspec-core", "~> 3.12" gem "rspec-expectations", "~> 3.12" gem "rspec-mocks", "~> 3.12" +gem "rubygems-generate_index", "~> 1.1" +gem "simplecov", "~> 0.22" group :doc do gem "ronn-ng", "~> 0.10.1", platform: :ruby diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock index 362bf25690..ee91c8baff 100644 --- a/tool/bundler/dev_gems.rb.lock +++ b/tool/bundler/dev_gems.rb.lock @@ -1,60 +1,74 @@ GEM remote: https://rubygems.org/ specs: - diff-lcs (1.5.1) - kramdown (2.5.1) - rexml (>= 3.3.9) + compact_index (0.15.0) + diff-lcs (1.6.2) + docile (1.4.1) + kramdown (2.5.2) + rexml (>= 3.4.4) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - mini_portile2 (2.8.8) - mustache (1.1.1) - nokogiri (1.18.1) + mini_portile2 (2.8.9) + mustache (1.1.2) + nokogiri (1.19.3) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.18.1-aarch64-linux-gnu) + nokogiri (1.19.3-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.1-arm-linux-gnu) + nokogiri (1.19.3-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.18.1-arm64-darwin) + nokogiri (1.19.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.1-java) + nokogiri (1.19.3-java) racc (~> 1.4) - nokogiri (1.18.1-x64-mingw-ucrt) + nokogiri (1.19.3-x64-mingw-ucrt) racc (~> 1.4) - nokogiri (1.18.1-x86_64-darwin) + nokogiri (1.19.3-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.1-x86_64-linux-gnu) + nokogiri (1.19.3-x86_64-linux-gnu) racc (~> 1.4) - parallel (1.24.0) - parallel_tests (4.7.2) + parallel (1.28.0) + parallel_tests (4.10.1) parallel - power_assert (2.0.3) + power_assert (3.0.1) racc (1.8.1) racc (1.8.1-java) - rake (13.2.1) - rb_sys (0.9.91) - rexml (3.4.0) + rake (13.4.2) + rake-compiler-dock (1.12.0) + rb_sys (0.9.128) + rake-compiler-dock (= 1.12.0) + rexml (3.4.4) ronn-ng (0.10.1) kramdown (~> 2, >= 2.1) kramdown-parser-gfm (~> 1, >= 1.0.1) mustache (~> 1) nokogiri (~> 1, >= 1.14.3) - rspec (3.13.0) + rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.6) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.8) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - test-unit (3.6.2) + rspec-support (3.13.7) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.13.2) + simplecov_json_formatter (0.1.4) + test-unit (3.7.7) power_assert - turbo_tests (2.2.3) + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) parallel_tests (>= 3.3.0, < 5) rspec (>= 3.10) @@ -67,52 +81,64 @@ PLATFORMS ruby universal-java x64-mingw-ucrt + x64-mswin64-140 x86-linux x86_64-darwin x86_64-linux DEPENDENCIES parallel (~> 1.19) - parallel_tests (~> 4.7) + parallel_tests (~> 4.10.1) rake (~> 13.1) - rb_sys + rb_sys (>= 0.9.128) ronn-ng (~> 0.10.1) rspec-core (~> 3.12) rspec-expectations (~> 3.12) rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + simplecov (~> 0.22) test-unit (~> 3.0) + test-unit-ruby-core turbo_tests (~> 2.2.3) CHECKSUMS - diff-lcs (1.5.1) sha256=273223dfb40685548436d32b4733aa67351769c7dea621da7d9dd4813e63ddfe - kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e + kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 - mini_portile2 (2.8.8) sha256=8e47136cdac04ce81750bb6c09733b37895bf06962554e4b4056d78168d70a75 - mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 - nokogiri (1.18.1) sha256=df18be7e96c34736b6abfdeda80c6e845134fb9afe2fe5d4fbc1cf1f89c68475 - nokogiri (1.18.1-aarch64-linux-gnu) sha256=35837013800e34342fcbaca305f8c49231f6bd4f779bfa23fe7b4686ae82d5b8 - nokogiri (1.18.1-arm-linux-gnu) sha256=3b873fd6b0cd1ad7c77e87af701075bdfd14c9a6b2f2965c5e00ed29a5627a37 - nokogiri (1.18.1-arm64-darwin) sha256=d75193f284c899d225943a8944479faedd995a7573ddd5c8308ffbdf2ec55204 - nokogiri (1.18.1-java) sha256=e0e19b340f92d09b2b731e22d68895b2062d6555188aff370b05617516d3a781 - nokogiri (1.18.1-x64-mingw-ucrt) sha256=50d81e905a60dff706b99c980abefedaf1c3d2c434a3b49afaf1b69b80f7f5b4 - nokogiri (1.18.1-x86_64-darwin) sha256=d94e3aa6483577495fc8969d6b4b5c075840ce6b1ab09636a6d4177ad171051d - nokogiri (1.18.1-x86_64-linux-gnu) sha256=e516cf16ccde67ed4cc595a2621ca5ddd42562ecb24928914b0045a20a41620e - parallel (1.24.0) sha256=5bf38efb9b37865f8e93d7a762727f8c5fc5deb19949f4040c76481d5eee9397 - parallel_tests (4.7.2) sha256=d6b3a46d3c36f5167716bba38e571d813ae7c754e9b096b2daa41095e70a2612 - power_assert (2.0.3) sha256=cd5e13c267370427c9804ce6a57925d6030613e341cb48e02eec1f3c772d4cf8 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.2) sha256=d420243400354da78ded2d81541b381ad8d94e8e9b95022d0d71d66f8ef36c00 + nokogiri (1.19.3) sha256=78312cbac32a40c812780d9678221b79d51288eec00054c1a8d15f7ce05960e8 + nokogiri (1.19.3-aarch64-linux-gnu) sha256=46b89e5d7b9e844c2ee360794240c6ea2a4e6fa0c5892a4ed487db621224b639 + nokogiri (1.19.3-arm-linux-gnu) sha256=3919d5ffc334ad778a4a9eb88fda7dcb8b1fb58c8a52ac640c6dcd2f038e774f + nokogiri (1.19.3-arm64-darwin) sha256=71b9bd424b1b7abc18b05052a1a3cfd3627abdca62be280854cc411791357e42 + nokogiri (1.19.3-java) sha256=40ea6ebf5cf2005dae1dee26dd557d3afb41fb6de6c9764aca8cf06fdb841db1 + nokogiri (1.19.3-x64-mingw-ucrt) sha256=8bb7132cad356c879a1286eaabcb5e68326cb2490317984280fbc62f456d506a + nokogiri (1.19.3-x86_64-darwin) sha256=77f3fba57d46c53ab31e62fc6c28f705109d1bf6264356c76f132b2be5728d4d + nokogiri (1.19.3-x86_64-linux-gnu) sha256=2f5078620fe12e83669b5b17311b32532a8153d02eee7ad06948b926d6080976 + parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 - rake (13.2.1) sha256=46cb38dae65d7d74b6020a4ac9d48afed8eb8149c040eccf0523bec91907059d - rb_sys (0.9.91) sha256=8c6ad8f97fd86f80530e942f1a904c229a510ca372c6b92dc05270a84e51ecda - rexml (3.4.0) sha256=efbea1efba7fa151158e0ee1e643525834da2d8eb4cf744aa68f6480bc9804b2 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rake-compiler-dock (1.12.0) sha256=f13205c2738f3d2053afcd03491a9e4541b22a59a0bfc53fc8bc883bd8188023 + rb_sys (0.9.128) sha256=9ab81f4d6d4e1895de18762232362d1264475aa7035756b50441e442130538fd + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 - rspec (3.13.0) sha256=d490914ac1d5a5a64a0e1400c1d54ddd2a501324d703b8cfe83f458337bab993 - rspec-core (3.13.0) sha256=557792b4e88da883d580342b263d9652b6a10a12d5bda9ef967b01a48f15454c - rspec-expectations (3.13.0) sha256=621d48c62262f955421eaa418130744760802cad47e781df70dba4d9f897102e - rspec-mocks (3.13.0) sha256=735a891215758d77cdb5f4721fffc21078793959d1f0ee4a961874311d9b7f66 - rspec-support (3.13.1) sha256=48877d4f15b772b7538f3693c22225f2eda490ba65a0515c4e7cf6f2f17de70f - test-unit (3.6.2) sha256=3ce480c23990ca504a3f0d6619be2a560e21326cefd1b86d0f9433c387f26039 - turbo_tests (2.2.3) sha256=c1a8763361a019c3ff68e8a47c5e1acb32c1e7668f9d4a4e08416ca4786ea8a0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 + rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5 + simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246 + simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f BUNDLED WITH - 2.7.0.dev + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 4d0b21060a..c71b862318 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -4,9 +4,10 @@ source "https://rubygems.org" gem "rubocop", ">= 1.52.1", "< 2" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" -gem "rb_sys" +gem "rb_sys", ">= 0.9.128" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock index 8ca5f13f18..f265e7c9eb 100644 --- a/tool/bundler/rubocop_gems.rb.lock +++ b/tool/bundler/rubocop_gems.rb.lock @@ -1,56 +1,91 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.5.1) - json (2.9.1) - json (2.9.1-java) - language_server-protocol (3.17.0.3) - minitest (5.22.3) - parallel (1.26.3) - parser (3.3.6.0) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.4) + erb (6.0.4-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.18.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.7) + json (2.19.4) + json (2.19.4-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (2.1.0) + parser (3.3.11.1) ast (~> 2.4.1) racc - power_assert (2.0.3) + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) racc (1.8.1) racc (1.8.1-java) rainbow (3.1.1) - rake (13.2.1) - rake-compiler (1.2.7) + rake (13.4.2) + rake-compiler (1.3.1) rake - rb_sys (0.9.91) - regexp_parser (2.10.0) - rspec (3.13.0) + rake-compiler-dock (1.12.0) + rb_sys (0.9.128) + rake-compiler-dock (= 1.12.0) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.12.0) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.6) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.8) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubocop (1.70.0) + rspec-support (3.13.7) + rubocop (1.86.1) json (~> 2.3) - language_server-protocol (>= 3.17.0) - parallel (~> 1.10) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (>= 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.36.2, < 2.0) + rubocop-ast (>= 1.49.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.37.0) - parser (>= 3.3.1.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) ruby-progressbar (1.13.0) - test-unit (3.6.2) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (3.1.4) - unicode-emoji (~> 4.0, >= 4.0.4) - unicode-emoji (4.0.4) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS aarch64-darwin @@ -59,46 +94,67 @@ PLATFORMS ruby universal-java x64-mingw-ucrt + x64-mswin64-140 x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler - rb_sys + rb_sys (>= 0.9.128) rspec rubocop (>= 1.52.1, < 2) test-unit CHECKSUMS - ast (2.4.2) sha256=1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12 - diff-lcs (1.5.1) sha256=273223dfb40685548436d32b4733aa67351769c7dea621da7d9dd4813e63ddfe - json (2.9.1) sha256=d2bdef4644052fad91c1785d48263756fe32fcac08b96a20bb15840e96550d11 - json (2.9.1-java) sha256=88de8c79b54fee6ae1b4854bc48b8d7089f524cbacaf4596df24f86b10896ee8 - language_server-protocol (3.17.0.3) sha256=3d5c58c02f44a20d972957a9febe386d7e7468ab3900ce6bd2b563dd910c6b3f - minitest (5.22.3) sha256=ea84676290cb5e2b4f31f25751af6050aa90d3e43e4337141c3e3e839611981e - parallel (1.26.3) sha256=d86babb7a2b814be9f4b81587bf0b6ce2da7d45969fab24d8ae4bf2bb4d4c7ef - parser (3.3.6.0) sha256=25d4e67cc4f0f7cab9a2ae1f38e2005b6904d2ea13c34734511d0faad038bc3b - power_assert (2.0.3) sha256=cd5e13c267370427c9804ce6a57925d6030613e341cb48e02eec1f3c772d4cf8 + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.4) sha256=38e3803694be357fe2bfe312487c74beaf9fb4e5beb3e22498952fe1645b95d9 + erb (6.0.4-java) sha256=3014611d37917a20e14ea3ba71e06a8d581b71c073858d7796eeee45b01e8407 + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 + jar-dependencies (0.5.7) sha256=013ce5f4639414ac8cf1169cdbe763da164b81e2d2c983d11042b5ff7bfcce80 + json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac + json (2.19.4-java) sha256=f7f0fe701e2bef648497b0eb59422f5b453e5038cfbaf9cde09af20e22241efb + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 + parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a - rake (13.2.1) sha256=46cb38dae65d7d74b6020a4ac9d48afed8eb8149c040eccf0523bec91907059d - rake-compiler (1.2.7) sha256=5176f8527bbf86db4b333915335eb5fa0b4f578cb82428c3e5e47e48179f0dee - rb_sys (0.9.91) sha256=8c6ad8f97fd86f80530e942f1a904c229a510ca372c6b92dc05270a84e51ecda - regexp_parser (2.10.0) sha256=cb6f0ddde88772cd64bff1dbbf68df66d376043fe2e66a9ef77fcb1b0c548c61 - rspec (3.13.0) sha256=d490914ac1d5a5a64a0e1400c1d54ddd2a501324d703b8cfe83f458337bab993 - rspec-core (3.13.0) sha256=557792b4e88da883d580342b263d9652b6a10a12d5bda9ef967b01a48f15454c - rspec-expectations (3.13.0) sha256=621d48c62262f955421eaa418130744760802cad47e781df70dba4d9f897102e - rspec-mocks (3.13.0) sha256=735a891215758d77cdb5f4721fffc21078793959d1f0ee4a961874311d9b7f66 - rspec-support (3.13.1) sha256=48877d4f15b772b7538f3693c22225f2eda490ba65a0515c4e7cf6f2f17de70f - rubocop (1.70.0) sha256=96751f8440b36a0ac6e9a8ab596900803118d83d6b83f2037bf8b3d7a5bc440e - rubocop-ast (1.37.0) sha256=9513ac88aaf113d04b52912533ffe46475de1362d4aa41141b51b2455827c080 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.12.0) sha256=f13205c2738f3d2053afcd03491a9e4541b22a59a0bfc53fc8bc883bd8188023 + rb_sys (0.9.128) sha256=9ab81f4d6d4e1895de18762232362d1264475aa7035756b50441e442130538fd + rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 + regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 + rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c + rubocop (1.86.1) sha256=44415f3f01d01a21e01132248d2fd0867572475b566ca188a0a42133a08d4531 + rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035 ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 - test-unit (3.6.2) sha256=3ce480c23990ca504a3f0d6619be2a560e21326cefd1b86d0f9433c387f26039 - unicode-display_width (3.1.4) sha256=8caf2af1c0f2f07ec89ef9e18c7d88c2790e217c482bfc78aaa65eadd5415ac1 - unicode-emoji (4.0.4) sha256=2c2c4ef7f353e5809497126285a50b23056cc6e61b64433764a35eff6c36532a + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f BUNDLED WITH - 2.7.0.dev + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 20c1ecd827..84028a385d 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -4,9 +4,10 @@ source "https://rubygems.org" gem "standard", "~> 1.0" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" -gem "rb_sys" +gem "rb_sys", ">= 0.9.128" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock index 9539b1aa85..8ef7806bcc 100644 --- a/tool/bundler/standard_gems.rb.lock +++ b/tool/bundler/standard_gems.rb.lock @@ -1,72 +1,107 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.5.1) - json (2.7.1) - json (2.7.1-java) - language_server-protocol (3.17.0.3) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.4) + erb (6.0.4-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.18.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.7) + json (2.19.4) + json (2.19.4-java) + language_server-protocol (3.17.0.5) lint_roller (1.1.0) - minitest (5.22.3) - parallel (1.24.0) - parser (3.3.0.5) + minitest (5.27.0) + parallel (1.28.0) + parser (3.3.11.1) ast (~> 2.4.1) racc - power_assert (2.0.3) - racc (1.7.3) - racc (1.7.3-java) + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) rainbow (3.1.1) - rake (13.2.1) - rake-compiler (1.2.7) + rake (13.4.2) + rake-compiler (1.3.1) rake - rb_sys (0.9.91) - regexp_parser (2.9.0) - rexml (3.2.6) - rspec (3.13.0) + rake-compiler-dock (1.12.0) + rb_sys (0.9.128) + rake-compiler-dock (= 1.12.0) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.12.0) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.6) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.8) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubocop (1.62.1) + rspec-support (3.13.7) + rubocop (1.84.2) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) - rubocop-performance (1.20.2) - rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) ruby-progressbar (1.13.0) - standard (1.35.1) + standard (1.54.0) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) - rubocop (~> 1.62.0) + rubocop (~> 1.84.0) standard-custom (~> 1.0.0) - standard-performance (~> 1.3) + standard-performance (~> 1.8) standard-custom (1.0.2) lint_roller (~> 1.0) rubocop (~> 1.50) - standard-performance (1.3.1) + standard-performance (1.9.0) lint_roller (~> 1.1) - rubocop-performance (~> 1.20.2) - test-unit (3.6.2) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (2.5.0) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS aarch64-darwin @@ -75,51 +110,71 @@ PLATFORMS ruby universal-java x64-mingw-ucrt + x64-mswin64-140 x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler - rb_sys + rb_sys (>= 0.9.128) rspec standard (~> 1.0) test-unit CHECKSUMS - ast (2.4.2) sha256=1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12 - diff-lcs (1.5.1) sha256=273223dfb40685548436d32b4733aa67351769c7dea621da7d9dd4813e63ddfe - json (2.7.1) sha256=187ea312fb58420ff0c40f40af1862651d4295c8675267c6a1c353f1a0ac3265 - json (2.7.1-java) sha256=bfd628c0f8357058c2cf848febfa6f140f70f94ec492693a31a0a1933038a61b - language_server-protocol (3.17.0.3) sha256=3d5c58c02f44a20d972957a9febe386d7e7468ab3900ce6bd2b563dd910c6b3f + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.4) sha256=38e3803694be357fe2bfe312487c74beaf9fb4e5beb3e22498952fe1645b95d9 + erb (6.0.4-java) sha256=3014611d37917a20e14ea3ba71e06a8d581b71c073858d7796eeee45b01e8407 + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 + jar-dependencies (0.5.7) sha256=013ce5f4639414ac8cf1169cdbe763da164b81e2d2c983d11042b5ff7bfcce80 + json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac + json (2.19.4-java) sha256=f7f0fe701e2bef648497b0eb59422f5b453e5038cfbaf9cde09af20e22241efb + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 - minitest (5.22.3) sha256=ea84676290cb5e2b4f31f25751af6050aa90d3e43e4337141c3e3e839611981e - parallel (1.24.0) sha256=5bf38efb9b37865f8e93d7a762727f8c5fc5deb19949f4040c76481d5eee9397 - parser (3.3.0.5) sha256=7748313e505ca87045dc0465c776c802043f777581796eb79b1654c5d19d2687 - power_assert (2.0.3) sha256=cd5e13c267370427c9804ce6a57925d6030613e341cb48e02eec1f3c772d4cf8 - racc (1.7.3) sha256=b785ab8a30ec43bce073c51dbbe791fd27000f68d1c996c95da98bf685316905 - racc (1.7.3-java) sha256=b2ad737e788cfa083263ce7c9290644bb0f2c691908249eb4f6eb48ed2815dbf + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970 + parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a - rake (13.2.1) sha256=46cb38dae65d7d74b6020a4ac9d48afed8eb8149c040eccf0523bec91907059d - rake-compiler (1.2.7) sha256=5176f8527bbf86db4b333915335eb5fa0b4f578cb82428c3e5e47e48179f0dee - rb_sys (0.9.91) sha256=8c6ad8f97fd86f80530e942f1a904c229a510ca372c6b92dc05270a84e51ecda - regexp_parser (2.9.0) sha256=81a00ba141cec0d4b4bf58cb80cd9193e5180836d3fa6ef623f7886d3ba8bdd9 - rexml (3.2.6) sha256=e0669a2d4e9f109951cb1fde723d8acd285425d81594a2ea929304af50282816 - rspec (3.13.0) sha256=d490914ac1d5a5a64a0e1400c1d54ddd2a501324d703b8cfe83f458337bab993 - rspec-core (3.13.0) sha256=557792b4e88da883d580342b263d9652b6a10a12d5bda9ef967b01a48f15454c - rspec-expectations (3.13.0) sha256=621d48c62262f955421eaa418130744760802cad47e781df70dba4d9f897102e - rspec-mocks (3.13.0) sha256=735a891215758d77cdb5f4721fffc21078793959d1f0ee4a961874311d9b7f66 - rspec-support (3.13.1) sha256=48877d4f15b772b7538f3693c22225f2eda490ba65a0515c4e7cf6f2f17de70f - rubocop (1.62.1) sha256=aeb1ec501aef5833617b3b6a1512303806218c349c28ce5b3ea72e3782ad4a35 - rubocop-ast (1.31.2) sha256=7c206fb094553779923eca862aceece3913ce384f1bf85730208228e884578ec - rubocop-performance (1.20.2) sha256=1bb1fa8c427fac7ba3c8dd2decb9860f23cb2d6c40350bedc88538de8875c731 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.12.0) sha256=f13205c2738f3d2053afcd03491a9e4541b22a59a0bfc53fc8bc883bd8188023 + rb_sys (0.9.128) sha256=9ab81f4d6d4e1895de18762232362d1264475aa7035756b50441e442130538fd + rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 + regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 + rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c + rubocop (1.84.2) sha256=5692cea54168f3dc8cb79a6fe95c5424b7ea893c707ad7a4307b0585e88dbf5f + rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035 + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 - standard (1.35.1) sha256=69a633610864f76e84b438d44b605fda020a3fc9b31a2d50d3487edb77a572ad + standard (1.54.0) sha256=7a4b08f83d9893083c8f03bc486f0feeb6a84d48233b40829c03ef4767ea0100 standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b - standard-performance (1.3.1) sha256=0e813d7347fc116b395ae4a6bffcece3ad3114b06e537436a76da79b9194d119 - test-unit (3.6.2) sha256=3ce480c23990ca504a3f0d6619be2a560e21326cefd1b86d0f9433c387f26039 - unicode-display_width (2.5.0) sha256=7e7681dcade1add70cb9fda20dd77f300b8587c81ebbd165d14fd93144ff0ab4 + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f BUNDLED WITH - 2.7.0.dev + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 5b211391b1..71230c32b7 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,13 +2,16 @@ source "https://rubygems.org" -gem "rack", "~> 3.0" -gem "rackup", "~> 2.1" -gem "webrick", "~> 1.9" +gem "rack", "~> 3.1" gem "rack-test", "~> 2.1" -gem "compact_index", "~> 0.15.0" gem "sinatra", "~> 4.1" gem "rake", "~> 13.1" gem "builder", "~> 3.2" -gem "rb_sys" +gem "rb_sys", ">= 0.9.128" +gem "fiddle" gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock index 91a48dea85..0b9ac34162 100644 --- a/tool/bundler/test_gems.rb.lock +++ b/tool/bundler/test_gems.rb.lock @@ -1,78 +1,102 @@ GEM remote: https://rubygems.org/ specs: - base64 (0.2.0) + base64 (0.3.0) builder (3.3.0) compact_index (0.15.0) - logger (1.6.5) - mustermann (3.0.3) - ruby2_keywords (~> 0.0.1) - rack (3.1.8) - rack-protection (4.1.1) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.7) + logger (1.7.0) + mustermann (3.1.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.6) + rack-protection (4.2.1) base64 (>= 0.1.0) logger (>= 1.6.0) rack (>= 3.0.0, < 4) - rack-session (2.1.0) + rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) - rack-test (2.1.0) + rack-test (2.2.0) rack (>= 1.3) - rackup (2.1.0) - rack (>= 3) - webrick (~> 1.8) - rake (13.2.1) - rb_sys (0.9.102) - ruby2_keywords (0.0.5) + rake (13.4.2) + rake-compiler-dock (1.12.0) + rb_sys (0.9.128) + rake-compiler-dock (= 1.12.0) rubygems-generate_index (1.1.3) compact_index (~> 0.15.0) - sinatra (4.1.1) + shellwords (0.2.2) + sinatra (4.2.1) logger (>= 1.6.0) mustermann (~> 3.0) rack (>= 3.0.0, < 4) - rack-protection (= 4.1.1) + rack-protection (= 4.2.1) rack-session (>= 2.0.0, < 3) tilt (~> 2.0) - tilt (2.5.0) - webrick (1.9.0) + stringio (3.2.0) + tilt (2.7.0) PLATFORMS java ruby universal-java x64-mingw-ucrt + x64-mswin64-140 x86_64-darwin x86_64-linux DEPENDENCIES builder (~> 3.2) - compact_index (~> 0.15.0) - rack (~> 3.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) rack-test (~> 2.1) - rackup (~> 2.1) rake (~> 13.1) - rb_sys + rb_sys (>= 0.9.128) rubygems-generate_index (~> 1.1) + shellwords sinatra (~> 4.1) - webrick (~> 1.9) CHECKSUMS - base64 (0.2.0) sha256=0f25e9b21a02a0cc0cea8ef92b2041035d39350946e8789c562b2d1a3da01507 + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b - logger (1.6.5) sha256=c3cfe56d01656490ddd103d38b8993d73d86296adebc5f58cefc9ec03741e56b - mustermann (3.0.3) sha256=d1f8e9ba2ddaed47150ddf81f6a7ea046826b64c672fbc92d83bce6b70657e88 - rack (3.1.8) sha256=d3fbcbca43dc2b43c9c6d7dfbac01667ae58643c42cea10013d0da970218a1b1 - rack-protection (4.1.1) sha256=51a254a5d574a7f0ca4f0672025ce2a5ef7c8c3bd09c431349d683e825d7d16a - rack-session (2.1.0) sha256=437c3916535b58ef71c816ce4a2dee0a01c8a52ae6077dc2b6cd19085760a290 - rack-test (2.1.0) sha256=0c61fc61904049d691922ea4bb99e28004ed3f43aa5cfd495024cc345f125dfb - rackup (2.1.0) sha256=6ecb884a581990332e45ee17bdfdc14ccbee46c2f710ae1566019907869a6c4d - rake (13.2.1) sha256=46cb38dae65d7d74b6020a4ac9d48afed8eb8149c040eccf0523bec91907059d - rb_sys (0.9.102) sha256=6ed736cc0d0bc236327e233f349ba16913231051df1c886c471ed268ce0e623b - ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.7) sha256=013ce5f4639414ac8cf1169cdbe763da164b81e2d2c983d11042b5ff7bfcce80 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.1.1) sha256=4c6170c7234d5499c345562ba7c7dfe73e1754286dcc1abb053064d66a127198 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.6) sha256=5ed78e1f73b2e25679bec7d45ee2d4483cc4146eb1be0264fc4d94cb5ef212c2 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.2) sha256=595434f8c0c3473ae7d7ac56ecda6cc6dfd9d37c0b2b5255330aa1576967ffe8 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rake-compiler-dock (1.12.0) sha256=f13205c2738f3d2053afcd03491a9e4541b22a59a0bfc53fc8bc883bd8188023 + rb_sys (0.9.128) sha256=9ab81f4d6d4e1895de18762232362d1264475aa7035756b50441e442130538fd rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c - sinatra (4.1.1) sha256=4e997b859aa1b5d2e624f85d5b0fd0f0b3abc0da44daa6cbdf10f7c0da9f4d00 - tilt (2.5.0) sha256=3c871a9ffb0fd8191944d8bbd776a371ba1eeb683483cecf1b2572b292293b15 - webrick (1.9.0) sha256=9ee50c57006489960b2a07544f68de6f23dfbee30e7b424167b5c14b72ace964 + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.7.0) sha256=0d5b9ba69f6a36490c64b0eee9f6e9aad517e20dcc848800a06eb116f08c6ab3 BUNDLED WITH - 2.7.0.dev + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index b8525c2e90..8d12c5adde 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -2,16 +2,16 @@ source "https://rubygems.org" -gem "fileutils", "1.7.3" -gem "molinillo", github: "cocoapods/molinillo" -gem "net-http", "0.6.0" -gem "net-http-persistent", "4.0.4" +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" gem "net-protocol", "0.2.2" -gem "optparse", "0.6.0" -gem "pub_grub", github: "jhawthorn/pub_grub" -gem "resolv", "0.6.0" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" gem "securerandom", "0.4.1" -gem "timeout", "0.4.3" -gem "thor", "1.3.2" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" gem "tsort", "0.2.0" -gem "uri", "1.0.2" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/downloader.rb b/tool/downloader.rb index a1520eb6a9..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,41 +1,14 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. # -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader def self.find(dlname) @@ -44,34 +17,25 @@ class Downloader end end - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' - end - - def self.https - @@https - end - def self.get_option(argv, options) false end class GNU < self + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + def self.download(name, *rest, **options) - if https? - begin - super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest, **options) - rescue => e - m1, m2 = e.message.split("\n", 2) - STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest, **options) - end + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest, **options) + return end end end @@ -222,11 +186,6 @@ class Downloader if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end if verbose $stdout.print "downloading #{name} ... " $stdout.flush @@ -234,13 +193,7 @@ class Downloader mtime = nil options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(10) do - data = url.read(options) - if mtime = data.meta["last-modified"] - mtime = Time.httpdate(mtime) - end - data - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error case http_error.message when /^304 / # 304 Not Modified @@ -268,6 +221,10 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath @@ -386,8 +343,6 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} diff --git a/tool/dump_ast.c b/tool/dump_ast.c new file mode 100644 index 0000000000..58250e9b8c --- /dev/null +++ b/tool/dump_ast.c @@ -0,0 +1,77 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> + +/* + * When prism is compiled as part of CRuby, the xmalloc/xfree/etc. macros are + * redirected to ruby_xmalloc/ruby_xfree/etc. Since this is a standalone + * program that links against those same object files, we need to provide + * implementations of these functions. + */ +void *ruby_xmalloc(size_t size) { return malloc(size); } +void *ruby_xcalloc(size_t nelems, size_t elemsiz) { return calloc(nelems, elemsiz); } +void *ruby_xrealloc(void *ptr, size_t newsiz) { return realloc(ptr, newsiz); } +void ruby_xfree(void *ptr) { free(ptr); } +void ruby_xfree_sized(void *ptr, size_t _oldsize) { free(ptr); } +void *ruby_xrealloc_sized(void *ptr, size_t newsiz, size_t _oldsiz) { return realloc(ptr, newsiz); } + +#include "prism.h" + +static void +print_error(const pm_diagnostic_t *diagnostic, void *data) +{ + const pm_parser_t *parser = (const pm_parser_t *) data; + pm_location_t loc = pm_diagnostic_location(diagnostic); + const pm_line_column_t line_column = pm_line_offset_list_line_column(pm_parser_line_offsets(parser), loc.start, pm_parser_start_line(parser)); + fprintf(stderr, "%" PRIi32 ":%" PRIu32 ":%s\n", line_column.line, line_column.column, pm_diagnostic_message(diagnostic)); +} + +int +main(int argc, const char *argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s <filename>\n", argv[0]); + return EXIT_FAILURE; + } + + const char *filepath = argv[1]; + pm_source_init_result_t init_result; + pm_source_t *source = pm_source_mapped_new(filepath, 0, &init_result); + + if (init_result != PM_SOURCE_INIT_SUCCESS) + { + fprintf(stderr, "unable to map file: %s\n", filepath); + return EXIT_FAILURE; + } + + pm_options_t *options = pm_options_new(); + pm_options_line_set(options, 1); + pm_options_filepath_set(options, filepath); + + pm_arena_t *arena = pm_arena_new(); + pm_parser_t *parser = pm_parser_new(arena, pm_source_source(source), pm_source_length(source), options); + + pm_node_t *node = pm_parse(parser); + int exit_status; + + if (pm_parser_errors_size(parser) > 0) + { + fprintf(stderr, "error parsing %s\n", filepath); + pm_parser_errors_each(parser, print_error, parser); + exit_status = EXIT_FAILURE; + } + else { + pm_buffer_t *json = pm_buffer_new(); + pm_dump_json(json, parser, node); + printf("%.*s\n", (int) pm_buffer_length(json), pm_buffer_value(json)); + pm_buffer_free(json); + exit_status = EXIT_SUCCESS; + } + + pm_parser_free(parser); + pm_arena_free(arena); + pm_source_free(source); + pm_options_free(options); + + return exit_status; +} diff --git a/tool/dump_ast.mkmf.rb b/tool/dump_ast.mkmf.rb new file mode 100755 index 0000000000..eec6b72f79 --- /dev/null +++ b/tool/dump_ast.mkmf.rb @@ -0,0 +1,37 @@ +#!ruby -s +require 'mkmf' +require 'pathname' +require 'fileutils' + +workdir, src, *objs = ARGV +src = Pathname(src) +tooldir = src.parent.relative_path_from(workdir) +srcdir = tooldir.parent +target = src.basename.sub_ext('') +dirs = objs.map {|obj| File.dirname(obj)}.uniq - %w[.] +link = MakeMakefile::TRY_LINK.sub(MakeMakefile::CONFTEST+$EXEEXT, '$(@)') +prismdir= "$(srcdir)/#{dirs.first}" +$VPATH = ["$(srcdir)", "$(srcdir)/#{tooldir.basename}", prismdir, tooldir] +$INCFLAGS << " -I#{prismdir}" +$CPPFLAGS = $CFLAGS = $INCFLAGS + +include FileUtils::Verbose +mkpath(workdir) +Dir.chdir(workdir) { + mkpath(dirs) + File.write('Makefile', [MakeMakefile.configuration(srcdir.to_s), <<~MAKEFILE].join("")) + target = #{target}#{$EXEEXT} + objs = #{objs.join(' ')} + + $(target): $(objs) + \t#{link} $(objs) + + objs: $(objs) + .c.#{$OBJEXT}: + \t#{MakeMakefile::COMPILE_C} + + clean: + \t$(RM) $(target) $(objs) Makefile + \t$(RMDIRS) #{dirs.join(' ')} + MAKEFILE +} diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 9d49f427bb..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -12,6 +12,9 @@ # You can get source file for gperf. After this, simply make ruby. # Or directly run: # tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. while arg = ARGV.shift case arg @@ -143,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -161,14 +165,24 @@ def parse_scripts(data, categories) names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -486,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index f50bda360a..4d2af06a85 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,4 +1,4 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' require_relative 'lib/colorize' @@ -21,27 +21,21 @@ BEGIN { n, v, u, r = $F next unless n -next if n =~ /^#/ next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} -if File.directory?(n) - puts "updating #{color.notice(n)} ..." - system("git", "fetch", "--all", chdir: n) or abort -else +unless File.exist?("#{n}/.git") puts "retrieving #{color.notice(n)} ..." - system(*%W"git clone #{u} #{n}") or abort + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort end -if r - puts "fetching #{color.notice(r)} ..." - system("git", "fetch", "origin", r, chdir: n) or abort -end +c = (r ? [r] : ["v#{v}", v]).find do |c| + puts "fetching #{n} #{color.notice(c)} ..." + system("git", "fetch", "origin", r || "refs/tags/#{c}:refs/tags/#{c}", chdir: n) +end or abort -c = r || "v#{v}" checkout = %w"git -c advice.detachedHead=false checkout" -print %[checking out #{color.notice(c)} (v=#{color.info(v)}] -print %[, r=#{color.info(r)}] if r -puts ") ..." +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." unless system(*checkout, c, "--", chdir: n) abort if r or !system(*checkout, v, "--", chdir: n) end diff --git a/tool/format-release b/tool/format-release index 72fc173000..d02154df1f 100755 --- a/tool/format-release +++ b/tool/format-release @@ -9,6 +9,7 @@ end require "open-uri" require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, @@ -30,10 +31,9 @@ class Tarball def gz?; @url.end_with?('.gz'); end def zip?; @url.end_with?('.zip'); end - def bz2?; @url.end_with?('.bz2'); end def xz?; @url.end_with?('.xz'); end - def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -51,29 +51,14 @@ eom # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 - def self.parse(wwwdir, version, rubydir) + def self.parse(wwwdir, version, rubydir, source_ref_or_sha = nil) unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" - info = YAML.load(URI(uri).read) + info = YAML.unsafe_load(URI(uri).read) if info.size != 1 raise "unexpected info.yml '#{uri}'" end @@ -88,10 +73,14 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" - stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` + tag = source_ref_or_sha || RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = IO.popen(["git", "-C", rubydir, "diff", "-l0", "--shortstat", "#{prev_tag}..#{tag}"], &:read) + unless $?.success? + raise "failed to diff #{prev_tag}..#{tag}" + end files_changed, insertions, deletions = stat.scan(/\d+/) end @@ -184,7 +173,7 @@ eom if /\.0(?:-\w+)?\z/ =~ ver # preview, rc, or first release entry <<= <<eom - tag: v#{ver.tr('.-', '_')} + tag: #{RubyVersion.tag(ver)} stats: files_changed: #{files_changed} insertions: #{insertions} @@ -196,34 +185,25 @@ eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz size: gz: #{ary.find{|x|x.gz? }.size} zip: #{ary.find{|x|x.zip?}.size} - bz2: #{ary.find{|x|x.bz2?}&.size} xz: #{ary.find{|x|x.xz? }.size} sha1: gz: #{ary.find{|x|x.gz? }.sha1} zip: #{ary.find{|x|x.zip?}.sha1} - bz2: #{ary.find{|x|x.bz2?}&.sha1} xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}&.sha256} xz: #{ary.find{|x|x.xz? }.sha256} sha512: gz: #{ary.find{|x|x.gz? }.sha512} zip: #{ary.find{|x|x.zip?}.sha512} - bz2: #{ary.find{|x|x.bz2?}&.sha512} xz: #{ary.find{|x|x.xz? }.sha512} eom - if ver.start_with?("3.") - entry = entry.gsub(/ bz2: .*\n/, "") - end - if data.include?("\n- version: #{ver}\n") # update existing entry data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") @@ -258,11 +238,12 @@ def main wwwdir = ARGV.shift version = ARGV.shift rubydir = ARGV.shift + source_ref_or_sha = ARGV.shift unless rubydir - STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir>" + STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir> [source-ref-or-sha]" exit end - Tarball.parse(wwwdir, version, rubydir) + Tarball.parse(wwwdir, version, rubydir, source_ref_or_sha) end main diff --git a/tool/ifchange b/tool/ifchange index 9e4a89533c..2a5f3db522 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -1,7 +1,8 @@ #!/bin/sh # usage: ifchange target temporary -# Used in generating revision.h via Makefiles. +# Used in generating various files such as rbconfig.rb, revision.h, +# etc. via Makefiles. help() { cat <<HELP @@ -20,6 +21,7 @@ timestamp= keepsuffix= srcavail=f color=auto +[ "x${NO_COLOR-}" = x ] || color=never until [ $# -eq 0 ]; do case "$1" in --) diff --git a/tool/leaked-globals b/tool/leaked-globals index 6118cd56e8..73da769318 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -96,7 +96,7 @@ Pipe.new(NM + ARGV).each do |line| next when /\Aruby_static_id_/ next unless so - when /\A(?:RUBY_|ruby_|rb_)/ + when /\A(?:RUBY_|ruby_|rb_|rbimpl_)/ next unless so and /_(threadptr|ec)_/ =~ n when *SYMBOLS_IN_EMPTYLIB next diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb index fd429dab37..ac5b9be792 100644 --- a/tool/lib/_tmpdir.rb +++ b/tool/lib/_tmpdir.rb @@ -4,11 +4,11 @@ template = "rubytest." # Assume the directory by these environment variables are safe. base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| next unless tmp and tmp.size <= 50 and File.directory?(tmp) - # On macOS, the default TMPDIR is very long, inspite of UNIX socket - # path length is limited. + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. # # Also Rubygems creates its own temporary directory per tests, and - # some tests copy the full path of gemhome there. In that caes, the + # some tests copy the full path of gemhome there. In that case, the # path contains both temporary names twice, and can exceed path name # limit very easily. tmp @@ -28,66 +28,71 @@ END { Dir.rmdir(tmpdir) rescue Errno::ENOENT rescue Errno::ENOTEMPTY - require_relative "colorize" - colorize = Colorize.new - ls = Struct.new(:colorize) do - def mode_inspect(m, s) - [ - (m & 0o4 == 0 ? ?- : ?r), - (m & 0o2 == 0 ? ?- : ?w), - (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), - ] - end - def decorate_path(path, st) - case - when st.directory? - color = "bold;blue" - type = "/" - when st.symlink? - color = "bold;cyan" - # type = "@" - when st.executable? - color = "bold;green" - type = "*" - when path.end_with?(".gem") - color = "green" + unless $no_report_tmpdir ||= nil + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] end - colorize.decorate(path, color) + (type || "") - end - def list_tree(parent, indent = "", &block) - children = Dir.children(parent).map do |child| - [child, path = File.join(parent, child), File.lstat(path)] + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") end - nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size - size_width = children.map {|child, path, st| st.size}.max.to_s.size + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size - children.each do |child, path, st| - m = st.mode - m = [ - (st.file? ? ?- : st.ftype[0]), - mode_inspect(m >> 6, (?s unless m & 04000 == 0)), - mode_inspect(m >> 3, (?s unless m & 02000 == 0)), - mode_inspect(m, (?t unless m & 01000 == 0)), - ].join("") - warn sprintf("%s* %s %*d %*d %s % s%s", - indent, m, nlink_width, st.nlink, size_width, st.size, - st.mtime.to_s, decorate_path(child, st), - (" -> " + decorate_path(File.readlink(path), File.stat(path)) if - st.symlink?)) - if st.directory? - list_tree(File.join(parent, child), indent + " ", &block) + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block end - yield path, st if block end - end - end.new(colorize) - warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" - Dir.chdir(tmpdir) do - ls.list_tree(".") do |path, st| - if st.directory? - Dir.rmdir(path) - else - File.unlink(path) + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + rescue Errno::EACCES + # On Windows, a killed process may still hold file locks briefly. + # Ignore and let FileUtils.rm_rf handle it below. end end end diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index 45e41ac648..ad103825bc 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -16,11 +16,21 @@ module BundledGem "psych" # rdoc ] + def self.command(gem, cmd) + if stub = Gem::Specification.latest_spec_for(gem) + spec = stub.spec + File.join(spec.gem_dir, spec.bindir, cmd) + end + end + module_function def unpack(file, *rest) pkg = Gem::Package.new(file) - prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end puts "Unpacked #{file}" rescue Gem::Package::FormatError, Errno::ENOENT puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." @@ -120,4 +130,46 @@ module BundledGem command = "#{git} checkout --detach #{rev}" system(command, chdir: gemdir) or raise "failed: #{command}" end + + class GemspecLoader + module NoPipe + refine IO.singleton_class do + def popen(...) ""; end + end + end + using NoPipe + + def `(command) ""; end + + def load_gemspec(file) + code = File.read(file, encoding: "utf-8:-") + eval(code, binding, file) + rescue + nil + end + end + + def load_gemspec(g) + spec = GemspecLoader.new.load_gemspec(g) + spec.files.clear + spec.extensions.clear + src = spec.to_ruby + src.sub!(/^$$/) { + %[# default: #{g} #{File.mtime(g).strftime(%[%s.%N])}\n] + } + return spec.full_name+'.gemspec', src + end + + def update_default_gemspecs(basedirs, out, quiet: true) + basedirs.each do |basedir| + Dir.glob(basedir+'/**/*.gemspec') do |g| + name, src = BundledGem.load_gemspec(g) + unless src + puts "Ignoring #{g}" unless quiet + next + end + out.write(src, name: name, newer: File.mtime(g), quiet: quiet) + end + end + end end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 0904312119..89da90e075 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -1,57 +1,78 @@ # frozen-string-literal: true +# Decorate TTY output using ANSI Select Graphic Rendition control +# sequences. class Colorize # call-seq: # Colorize.new(colorize = nil) # Colorize.new(color: color, colors_file: colors_file) - def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) - @colors = @reset = nil - @color = opts && opts[:color] || color + # + # Creates and load color settings. + def initialize(_color = nil, color: _color, colors_file: nil) + @colors = nil + @color = color if color or (color == nil && coloring?) - if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) + if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", err: IO::NULL, &:read)} rescue nil) @beg = "\e[" - colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} - if opts and colors_file = opts[:colors_file] + colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(COLORS_PATTERN)] : {} + if colors_file begin - File.read(colors_file).scan(/(\w+)=([^:\n]*)/) do |n, c| + File.read(colors_file).scan(COLORS_PATTERN) do |n, c| colors[n] ||= c end rescue Errno::ENOENT end end @colors = colors - @reset = "#{@beg}m" end end self end + COLORS_PATTERN = /(\w+)=([^:\n]*)/ + private_constant :COLORS_PATTERN + DEFAULTS = { # color names "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", - "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bold"=>"1", "faint"=>"2", "underline"=>"4", "reverse"=>"7", "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + "bg_black"=>"40", "bg_red"=>"41", "bg_green"=>"42", "bg_yellow"=>"43", + "bg_blue"=>"44", "bg_magenta"=>"45", "bg_cyan"=>"46", "bg_white"=>"47", + "bg_bright_black"=>"100", "bg_bright_red"=>"101", + "bg_bright_green"=>"102", "bg_bright_yellow"=>"103", + "bg_bright_blue"=>"104", "bg_bright_magenta"=>"105", + "bg_bright_cyan"=>"106", "bg_bright_white"=>"107", # abstract decorations "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", - } - - def coloring? - STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) - end + }.freeze + private_constant :DEFAULTS # colorize.decorate(str, name = color_name) def decorate(str, name = @color) if coloring? and color = resolve_color(name) - "#{@beg}#{color}m#{str}#{@reset}" + "#{@beg}#{color}m#{str}#{reset_color(color)}" else str end end + DEFAULTS.each_key do |name| + define_method(name) {|str| + decorate(str, name) + } + end + + private + + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end + def resolve_color(color = @color, seen = {}, colors = nil) return unless @colors color.to_s.gsub(/\b[a-z][\w ]+/) do |n| @@ -69,10 +90,23 @@ class Colorize end end - DEFAULTS.each_key do |name| - define_method(name) {|str| - decorate(str, name) - } + def reset_color(colors) + resets = [] + colors.scan(/\G;*\K(?:[34]8;(?:5;\d+|2(?:;\d+){3})|\d+)/) do |c| + case c + when '1', '2' + resets << '22' + when '4' + resets << '24' + when '7' + resets << '27' + when /\A[39]\d(?:;|\z)/ + resets << '39' + when /\A(?:4|10)\d(?:;|\z)/ + resets << '49' + end + end + "#{@beg}#{resets.reverse.join(';')}m" end end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 66822b4e3b..5ca318a598 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -75,9 +75,18 @@ module Test require_relative 'envutil' require 'pp' begin - require '-test-/asan' + require '-test-/sanitizers' rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end end + module_function :sanitizers nil.pretty_inspect @@ -97,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -159,7 +171,7 @@ module Test pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # ASAN has the same problem - its shadow memory greatly increases memory usage # (plus asan has better ways to detect memory leaks than this assertion) - pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled? + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -291,9 +303,34 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } if defined?(Test::Unit::Runner) Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) @@ -327,7 +364,16 @@ eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -338,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -358,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -369,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -379,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -489,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -670,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -684,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end @@ -831,6 +882,9 @@ eom rescue # Constants may be defined but not implemented, e.g., mingw. else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end PERFORMANCE_CLOCK = clk end end @@ -857,10 +911,11 @@ eom first = seq.first *arg = pre.call(first) - times = (0..(rehearsal || (2 * first))).map do + raw_times = (0..(rehearsal || (2 * first))).map do measure[arg, "rehearsal"].nonzero? end - times.compact! + times = raw_times.compact + raise "all measurements are zero: #{raw_times.inspect}" if times.empty? tmin, tmax = times.minmax # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 7b8aa99a39..b4c7d1d035 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -63,6 +63,14 @@ module EnvUtil end end + if RUBY_ENGINE == "truffleruby" + # Tests relying on timeout have high variance on TruffleRuby due to the highly-optimizing JIT, deoptimization, profiling interpreter, different GC, etc. + # Setting a default timeout scale helps avoid transient failures for tests relying on timeouts. + # We choose 10 because it is the same number used in CRuby CI on macOS: + # https://github.com/ruby/ruby/blob/9d46b0c735877f152a0b4b16b8153c6f395dee28/.github/workflows/macos.yml#L133 + self.timeout_scale = 10 + end + def apply_timeout_scale(t) if scale = EnvUtil.timeout_scale t * scale @@ -79,6 +87,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -94,17 +168,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -166,8 +235,8 @@ module EnvUtil args = [args] if args.kind_of?(String) # use the same parser as current ruby - if args.none? { |arg| arg.start_with?("--parser=") } - current_parser = RUBY_DESCRIPTION =~ /prism/i ? "prism" : "parse.y" + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) args = ["--parser=#{current_parser}"] + args end pid = spawn(child_env, *precommand, rubybin, *args, opt) @@ -217,6 +286,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -233,6 +308,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -279,7 +369,8 @@ module EnvUtil end module_function :without_gc - def with_default_external(enc) + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -287,7 +378,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb index 70a2469db2..1893e07657 100644 --- a/tool/lib/gem_env.rb +++ b/tool/lib/gem_env.rb @@ -1,2 +1 @@ -ENV['GEM_HOME'] = gem_home = File.expand_path('.bundle') -ENV['GEM_PATH'] = [gem_home, File.expand_path('../../../.bundle', __FILE__)].uniq.join(File::PATH_SEPARATOR) +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 69aeb2c254..33a546699f 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -77,6 +77,7 @@ class LeakChecker end (h[fd] ||= []) << [io, autoclose, inspect] } + inspect = {} fd_leaked.select! {|fd| str = ''.dup pos = nil @@ -98,6 +99,7 @@ class LeakChecker s = io.stat rescue Errno::EBADF # something un-stat-able + live2.delete(fd) next else next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev) @@ -106,15 +108,41 @@ class LeakChecker io&.close end end - puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" - puts " The IO was created at #{pos}" if pos + inspect[fd] = [str, pos] true } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + begin + open_list = IO.popen(%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], &:readlines) + rescue + @@try_lsof = false + else + @@try_lsof |= $?.success? + end + if header = open_list&.shift + columns = header.split + fd_index, node_index = columns.index('FD'), columns.index('NODE') + open_list.reject! do |of| + of = of.chomp.split(' ', node_index + 2) + if of[node_index] == 'TCP' and of.last.end_with?('(CLOSE_WAIT)') + fd = of[fd_index].to_i + inspect.delete(fd) + h.delete(fd) + live2.delete(fd) + true + else + false + end + end + puts(header, open_list) unless open_list.empty? + end end end + inspect.each {|fd, (str, pos)| + puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" + puts " The IO was created at #{pos}" if pos + } h.each {|fd, list| next if list.length <= 1 if 1 < list.count {|io, autoclose, inspect| autoclose } @@ -156,7 +184,7 @@ class LeakChecker [prev_count, []] else tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| - t.instance_variables.empty? || t.closed? + t.instance_variables.empty? || (t.closed? rescue true) } [count, tempfiles] end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 60632523a8..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/output.rb b/tool/lib/output.rb index 8cb426ae4a..8590e0ffe2 100644 --- a/tool/lib/output.rb +++ b/tool/lib/output.rb @@ -31,8 +31,8 @@ class Output @vpath.def_options(opt) end - def write(data, overwrite: @overwrite, create_only: @create_only) - unless @path + def write(data, overwrite: @overwrite, create_only: @create_only, name: nil, newer: nil, quiet: false) + unless (name = name ? (@path ? File.join(@path, name) : name) : @path) $stdout.print data return true end @@ -41,20 +41,21 @@ class Output updated = color.fail("updated") outpath = nil - if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f| + if (@ifchange or overwrite or create_only or newer) and (@vpath.open(name, "rb") {|f| outpath = f.path + next true if newer and f.mtime > newer if @ifchange or create_only original = f.read (@ifchange and original == data) or (create_only and !original.empty?) end } rescue false) - puts "#{outpath} #{unchanged}" + puts "#{outpath} #{unchanged}" unless quiet written = false else unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil) - File.binwrite(outpath = @path, data) + File.binwrite(outpath = name, data) end - puts "#{outpath} #{updated}" + puts "#{outpath} #{updated}" unless quiet written = true end if timestamp = @timestamp diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 30f30df62e..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -19,6 +19,7 @@ require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit @@ -249,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -260,27 +263,8 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags - begin - if fifo = $3 - fifo.gsub!(/\\(?=.)/, '') - r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) - w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) - else - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - end - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 256 # number of tokens to acquire first - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super @@ -298,7 +282,7 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing @@ -370,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -415,6 +403,7 @@ module Test end def kill + EnvUtil::Debugger.search&.dump(@pid) signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV Process.kill(signal, @pid) warn "worker #{to_s} does not respond; #{signal} is sent" @@ -597,7 +586,7 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) @@ -1292,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1617,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1856,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index 19581fc3ab..0908666166 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -128,8 +128,16 @@ module Test def assert_in_delta exp, act, delta = 0.001, msg = nil n = (exp - act).abs + loadavg = begin + if File.readable?("/proc/loadavg") + " (/proc/loadavg=#{File.read("/proc/loadavg").strip})" + end + rescue StandardError + nil + end + loadavg ||= "" msg = message(msg) { - "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" + "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}#{loadavg}" } assert delta >= n, msg end diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 51cdb0fdc3..d6374f9de0 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -51,23 +51,9 @@ module DebugPOpen end using DebugPOpen module DebugSystem - def system(*args) + def system(*args, exception: true, **opts) VCS.dump(args, "args: ") if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + super(*args, exception: exception, **opts) end end @@ -183,19 +169,7 @@ class VCS ) last or raise VCS::NotFoundError, "last revision not found" changed or raise VCS::NotFoundError, "changed revision not found" - if modified - /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or - raise "unknown time format - #{modified}" - match = $~[1..6].map { |x| x.to_i } - off = $7 ? "#{$7}:#{$8}" : "+00:00" - match << off - begin - modified = Time.new(*match) - rescue ArgumentError - modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 - end - modified = modified.getlocal(@zone) - end + modified &&= parse_iso_date(modified) return last, changed, modified, *rest end @@ -204,9 +178,9 @@ class VCS modified end - def relative_to(path) + def relative_to(path, srcdir = @srcdir) if path - srcdir = File.realpath(@srcdir) + srcdir = File.realpath(srcdir || @srcdir) path = File.realdirpath(path) list1 = srcdir.split(%r{/}) list2 = path.split(%r{/}) @@ -224,6 +198,20 @@ class VCS end end + def parse_iso_date(date) + /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ date or + raise "unknown time format - #{date}" + match = $~[1..6].map { |x| x.to_i } + off = $7 ? "#{$7}:#{$8}" : "+00:00" + match << off + begin + date = Time.new(*match) + rescue ArgumentError + date = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 + end + date.getlocal(@zone) + end + def after_export(dir) FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) @@ -271,155 +259,6 @@ class VCS code end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") - end - - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS.dump(args, "commit: ") - return true - end - system(*args) - end - end - class GIT < self register(".git") do |path, dir| SAFE_DIRECTORIES ||= @@ -525,6 +364,11 @@ class VCS [last, changed, modified, branch, title] end + def author_date(path, srcdir = @srcdir) + log = cmd_read_at(srcdir, [[COMMAND, 'log', '-n1', '--pretty=%at', path]]) + Time.at(log.to_i, in: @zone) + end + def self.revision_name(rev) short_revision(rev) end @@ -533,15 +377,6 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) @@ -616,60 +451,50 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + to ||= url.to_str + unless from&.match?(/./) or (from = branch_beginning(to))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) else warn "Could not fetch notes/commits tree", uplevel: 1 end - to ||= url.to_str if from arg = ["#{from}^..#{to}"] else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -678,9 +503,10 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV cmd = %W[#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] @@ -692,17 +518,32 @@ class VCS cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end next if /^Author: *dependabot\[bot\]/ =~ h h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') @@ -739,7 +580,7 @@ class VCS next end end - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -761,35 +602,9 @@ class VCS s = s.join('') end - if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s - issue = "#{$1}pull/" - s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} - end - s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -826,46 +641,6 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun? - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true - end - end - class Null < self def get_revisions(path, srcdir = nil) @modified ||= Time.now - 10 diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md index a535332ec3..693b46f018 100644 --- a/tool/lrama/NEWS.md +++ b/tool/lrama/NEWS.md @@ -1,8 +1,383 @@ # NEWS for Lrama +## Lrama 0.8.0 (2026-03-01) + +### Support parser generation without %union directive (Bison compatibility) + +When writing simple parsers or prototypes, defining `%union` can be cumbersome and unnecessary. +Lrama now supports generating parsers without the `%union` directive, defaulting `YYSTYPE` to `int`, the same behavior as Bison. + +When `%union` is not defined: + +- `YYSTYPE` defaults to `int` +- Semantic value references are generated without union member access +- The generated parser behaves identically to Bison-generated parsers + +https://github.com/ruby/lrama/pull/765 + +### Allow named references on parameterized rule calls inside %rule + +Named references (e.g. `[opt]`) can now be used with parameterized rule calls inside a `%rule` body. +Previously, writing `f_opt_arg(value)[opt]` inside a `%rule` resulted in a parse error. + +```yacc +%rule example(X): f_opt_arg(X)[opt] { $$ = $opt; } + ; +``` + +https://github.com/ruby/lrama/pull/778 + +### Fix nested parameterized rule calls in subsequent argument positions + +Nested parameterized rule calls (e.g. `f_opt(number)`) can now appear in any argument position, not only the first one. +Previously, using a nested call as the second or later argument caused a parse error. + +```yacc +%% +program: args_list(f_opt(number), opt_tail(string), number) + ; +``` + +https://github.com/ruby/lrama/pull/779 + +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + ## Lrama 0.7.0 (2025-01-21) -## [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper Support the generation of the IELR(1) parser described in this paper. https://www.sciencedirect.com/science/article/pii/S0167642309001191 @@ -15,12 +390,12 @@ If you use IELR(1) parser, you can write the following directive in your grammar But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. -## Support `-t` option as same as `--debug` option +### Support `-t` option as same as `--debug` option Support to `-t` option as same as `--debug` option. These options align with Bison behavior. So same as `--debug` option. -## Trace only explicit rules +### Trace only explicit rules Support to trace only explicit rules. If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. @@ -97,9 +472,9 @@ nterm.y:6:7: symbol EOI redeclared as a nonterminal ## Lrama 0.6.10 (2024-09-11) -### Aliased Named References for actions of RHS in parameterizing rules +### Aliased Named References for actions of RHS in Parameterizing rules -Allow to use aliased named references for actions of RHS in parameterizing rules. +Allow to use aliased named references for actions of RHS in Parameterizing rules. ```yacc %rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } @@ -109,9 +484,9 @@ Allow to use aliased named references for actions of RHS in parameterizing rules https://github.com/ruby/lrama/pull/410 -### Named References for actions of RHS in parameterizing rules caller side +### Named References for actions of RHS in Parameterizing rules caller side -Allow to use named references for actions of RHS in parameterizing rules caller side. +Allow to use named references for actions of RHS in Parameterizing rules caller side. ```yacc opt_nl: '\n'?[nl] <str> { $$ = $nl; } @@ -120,9 +495,9 @@ opt_nl: '\n'?[nl] <str> { $$ = $nl; } https://github.com/ruby/lrama/pull/414 -### Widen the definable position of parameterizing rules +### Widen the definable position of Parameterizing rules -Allow to define parameterizing rules in the middle of the grammar. +Allow to define Parameterizing rules in the middle of the grammar. ```yacc %rule defined_option(X): /* empty */ @@ -186,15 +561,15 @@ Change to `%locations` directive not set by default. https://github.com/ruby/lrama/pull/446 -### Diagnostics report for parameterizing rules redefine +### Diagnostics report for parameterized rules redefine -Support to warning redefined parameterizing rules. -Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterizing rules. +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. ```console $ exe/lrama -W sample/calc.y -parameterizing rule redefined: redefined_method(X) -parameterizing rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) ``` https://github.com/ruby/lrama/pull/448 @@ -208,9 +583,9 @@ https://github.com/ruby/lrama/pull/457 ## Lrama 0.6.9 (2024-05-02) -### Callee side tag specification of parameterizing rules +### Callee side tag specification of Parameterizing rules -Allow to specify tag on callee side of parameterizing rules. +Allow to specify tag on callee side of Parameterizing rules. ```yacc %union { @@ -221,9 +596,9 @@ Allow to specify tag on callee side of parameterizing rules. ; ``` -### Named References for actions of RHS in parameterizing rules +### Named References for actions of RHS in Parameterizing rules -Allow to use named references for actions of RHS in parameterizing rules. +Allow to use named references for actions of RHS in Parameterizing rules. ```yacc %rule option(number): /* empty */ @@ -233,9 +608,9 @@ Allow to use named references for actions of RHS in parameterizing rules. ## Lrama 0.6.8 (2024-04-29) -### Nested parameterizing rules with tag +### Nested Parameterizing rules with tag -Allow to nested parameterizing rules with tag. +Allow to nested Parameterizing rules with tag. ```yacc %union { @@ -257,9 +632,9 @@ Allow to nested parameterizing rules with tag. ## Lrama 0.6.7 (2024-04-28) -### RHS of user defined parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. -User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined parameterizing rules. +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. ``` %rule with_word_seps(X): /* empty */ @@ -319,7 +694,7 @@ expr : number { $$ = $1; } ### Typed Midrule Actions -User can specify the type of mid rule action by tag (`<bar>`) instead of specifying it with in an action. +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. ```yacc primary: k_case expr_value terms? @@ -394,7 +769,7 @@ https://github.com/ruby/lrama/pull/382 User can set codes for freeing semantic value resources by using `%destructor`. In general, these resources are freed by actions or after parsing. -However if syntax error happens in parsing, these codes may not be executed. +However, if syntax error happens in parsing, these codes may not be executed. Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. ```yacc @@ -432,7 +807,7 @@ Lrama introduces two features to support another semantic value stack by parser 1. Callback entry points User can emulate semantic value stack by these callbacks. -Lrama provides these five callbacks. Registered functions are called when each event happen. For example %after-shift function is called when shift happens on original semantic value stack. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. * `%after-shift` function_name * `%before-reduce` function_name @@ -460,15 +835,15 @@ https://github.com/ruby/lrama/pull/367 ### %no-stdlib directive If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for -parameterizing rules, stdlib.y. +parameterized rules, stdlib.y. https://github.com/ruby/lrama/pull/344 ## Lrama 0.6.1 (2024-01-13) -### Nested parameterizing rules +### Nested Parameterizing rules -Allow to pass an instantiated rule to other parameterizing rules. +Allow to pass an instantiated rule to other Parameterizing rules. ```yacc %rule constant(X) : X @@ -485,7 +860,7 @@ program : option(constant(number)) // Nested rule %% ``` -Allow to use nested parameterizing rules when define parameterizing rules. +Allow to use nested Parameterizing rules when define Parameterizing rules. ```yacc %rule option(x) : /* empty */ @@ -510,9 +885,9 @@ https://github.com/ruby/lrama/pull/337 ## Lrama 0.6.0 (2023-12-25) -### User defined parameterizing rules +### User defined Parameterizing rules -Allow to define parameterizing rule by `%rule` directive. +Allow to define Parameterizing rule by `%rule` directive. ```yacc %rule pair(X, Y): X Y { $$ = $1 + $2; } @@ -532,7 +907,7 @@ https://github.com/ruby/lrama/pull/285 ## Lrama 0.5.11 (2023-12-02) -### Type specification of parameterizing rules +### Type specification of Parameterizing rules Allow to specify type of rules by specifying tag, `<i>` in below example. Tag is post-modification style. @@ -556,13 +931,13 @@ https://github.com/ruby/lrama/pull/272 ### Parameterizing rules (option, nonempty_list, list) -Support function call style parameterizing rules for `option`, `nonempty_list` and `list`. +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. https://github.com/ruby/lrama/pull/197 ### Parameterizing rules (separated_list) -Support `separated_list` and `separated_nonempty_list` parameterizing rules. +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. ```text program: separated_list(',', number) @@ -618,7 +993,7 @@ https://github.com/ruby/lrama/pull/181 ### Racc parser -Replace Lrama's parser from hand written parser to LR parser generated by Racc. +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). https://github.com/ruby/lrama/pull/62 diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama index 1aece5d141..710ac0cb96 100755 --- a/tool/lrama/exe/lrama +++ b/tool/lrama/exe/lrama @@ -4,4 +4,4 @@ $LOAD_PATH << File.join(__dir__, "../lib") require "lrama" -Lrama::Command.new.run(ARGV.dup) +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb index fe2e05807c..56ba0044d4 100644 --- a/tool/lrama/lib/lrama.rb +++ b/tool/lrama/lib/lrama.rb @@ -4,19 +4,19 @@ require_relative "lrama/bitmap" require_relative "lrama/command" require_relative "lrama/context" require_relative "lrama/counterexamples" -require_relative "lrama/diagnostics" +require_relative "lrama/diagram" require_relative "lrama/digraph" +require_relative "lrama/erb" require_relative "lrama/grammar" -require_relative "lrama/grammar_validator" require_relative "lrama/lexer" require_relative "lrama/logger" require_relative "lrama/option_parser" require_relative "lrama/options" require_relative "lrama/output" require_relative "lrama/parser" -require_relative "lrama/report" +require_relative "lrama/reporter" require_relative "lrama/state" require_relative "lrama/states" -require_relative "lrama/states_reporter" -require_relative "lrama/trace_reporter" +require_relative "lrama/tracer" require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb index 098c6e0b77..88b255b012 100644 --- a/tool/lrama/lib/lrama/bitmap.rb +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -3,7 +3,10 @@ module Lrama module Bitmap - # @rbs (Array[Integer] ary) -> Integer + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap def self.from_array(ary) bit = 0 @@ -14,21 +17,31 @@ module Lrama bit end - # @rbs (Integer int) -> Array[Integer] + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] def self.to_array(int) a = [] #: Array[Integer] i = 0 - while int > 0 do - if int & 1 == 1 + len = int.bit_length + while i < len do + if int[i] == 1 a << i end i += 1 - int >>= 1 end a end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end end end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb index 3ff39d578d..17aad1a1c1 100644 --- a/tool/lrama/lib/lrama/command.rb +++ b/tool/lrama/lib/lrama/command.rb @@ -5,64 +5,116 @@ module Lrama LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') - def run(argv) - begin - options = OptionParser.new.parse(argv) - rescue => e - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message - end - - Report::Duration.enable if options.trace_opts[:time] + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end - text = options.y.read - options.y.close if options.y != STDIN - begin - grammar = Lrama::Parser.new(text, options.grammar_file, options.debug, options.define).parse - unless grammar.no_stdlib - stdlib_grammar = Lrama::Parser.new(File.read(STDLIB_FILE_PATH), STDLIB_FILE_PATH, options.debug).parse - grammar.insert_before_parameterizing_rules(stdlib_grammar.parameterizing_rules) + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow end - grammar.prepare - grammar.validate! - rescue => e - raise e if options.debug - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message end - states = Lrama::States.new(grammar, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure])) + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) states.compute states.compute_ielr if grammar.ielr_defined? - context = Lrama::Context.new(states) + [states, Lrama::Context.new(states)] + end - if options.report_file - reporter = Lrama::StatesReporter.new(states) - File.open(options.report_file, "w+") do |f| - reporter.report(f, **options.report_opts) - end + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) end + end - reporter = Lrama::TraceReporter.new(grammar) - reporter.report(**options.trace_opts) + def render_diagram(grammar) + return unless @options.diagram - File.open(options.outfile, "w+") do |f| + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| Lrama::Output.new( out: f, - output_file_path: options.outfile, - template_name: options.skeleton, - grammar_file_path: options.grammar_file, - header_file_path: options.header_file, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, context: context, grammar: grammar, - error_recovery: options.error_recovery, + error_recovery: @options.error_recovery, ).render end - - logger = Lrama::Logger.new - exit false unless Lrama::GrammarValidator.new(grammar, states, logger).valid? - Lrama::Diagnostics.new(grammar, states, logger).run(options.diagnostic) end end end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb index 9f406f8de0..eb068c1b9e 100644 --- a/tool/lrama/lib/lrama/context.rb +++ b/tool/lrama/lib/lrama/context.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -require_relative "report/duration" +require_relative "tracer/duration" module Lrama # This is passed to a template class Context - include Report::Duration + include Tracer::Duration ErrorActionNumber = -Float::INFINITY BaseMin = -Float::INFINITY @@ -231,8 +231,8 @@ module Lrama end # Shift is selected when S/R conflict exists. - state.selected_term_transitions.each do |shift, next_state| - actions[shift.next_sym.number] = next_state.id + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id end state.resolved_conflicts.select do |conflict| @@ -292,18 +292,18 @@ module Lrama # of a default nterm transition destination. @yydefgoto = Array.new(@states.nterms.count, 0) # Mapping from nterm to next_states - nterm_to_next_states = {} + nterm_to_to_states = {} @states.states.each do |state| - state.nterm_transitions.each do |shift, next_state| - key = shift.next_sym - nterm_to_next_states[key] ||= [] - nterm_to_next_states[key] << [state, next_state] # [from_state, to_state] + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] end end @states.nterms.each do |nterm| - if (states = nterm_to_next_states[nterm]) + if (states = nterm_to_to_states[nterm]) default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first default_goto = default_state.id not_default_gotos = [] @@ -417,27 +417,25 @@ module Lrama res = lowzero - froms_and_tos.first[0] + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` while true do - ok = true + advanced = false - froms_and_tos.each do |from, to| - loc = res + from - - if @table[loc] - # If the cell of table is set, can not use the cell. - ok = false - break - end + while used_res[res] + res += 1 + advanced = true end - if ok && used_res[res] - ok = false + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end end - if ok + unless advanced + # no advance means that the current `res` satisfies the condition break - else - res += 1 end end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb index ee2b5d5959..60d830d048 100644 --- a/tool/lrama/lib/lrama/counterexamples.rb +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -1,35 +1,64 @@ +# rbs_inline: enabled # frozen_string_literal: true require "set" +require "timeout" require_relative "counterexamples/derivation" require_relative "counterexamples/example" +require_relative "counterexamples/node" require_relative "counterexamples/path" -require_relative "counterexamples/production_path" -require_relative "counterexamples/start_path" require_relative "counterexamples/state_item" -require_relative "counterexamples/transition_path" require_relative "counterexamples/triple" module Lrama # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf # 4. Constructing Nonunifying Counterexamples class Counterexamples - attr_reader :transitions, :productions - + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void def initialize(states) @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items setup_transitions setup_productions end + # @rbs () -> "#<Counterexamples>" def to_s "#<Counterexamples>" end alias :inspect :to_s + # @rbs (State conflict_state) -> Array[Example] def compute(conflict_state) conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + case conflict.type when :shift_reduce # @type var conflict: State::ShiftReduceConflict @@ -38,22 +67,50 @@ module Lrama # @type var conflict: State::ReduceReduceConflict reduce_reduce_examples(conflict_state, conflict) end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil end.compact end private + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void def setup_transitions - # Hash [StateItem, Symbol] => StateItem @transitions = {} - # Hash [StateItem, Symbol] => Set(StateItem) @reverse_transitions = {} @states.states.each do |src_state| trans = {} #: Hash[Grammar::Symbol, State] - src_state.transitions.each do |shift, next_state| - trans[shift.next_sym] = next_state + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state end src_state.items.each do |src_item| @@ -63,8 +120,8 @@ module Lrama dest_state.kernels.each do |dest_item| next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) - src_state_item = StateItem.new(src_state, src_item) - dest_state_item = StateItem.new(dest_state, dest_item) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) @transitions[[src_state_item, sym]] = dest_state_item @@ -77,21 +134,20 @@ module Lrama end end + # @rbs () -> void def setup_productions - # Hash [StateItem] => Set(Item) @productions = {} - # Hash [State, Symbol] => Set(Item). Symbol is nterm @reverse_productions = {} @states.states.each do |state| - # LHS => Set(Item) - h = {} #: Hash[Grammar::Symbol, Set[States::Item]] + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] state.closure.each do |item| sym = item.lhs h[sym] ||= Set.new - h[sym] << item + h[sym] << get_state_item(state, item) end state.items.each do |item| @@ -99,101 +155,118 @@ module Lrama next if item.next_sym.term? sym = item.next_sym - state_item = StateItem.new(state, item) - # @type var key: [State, Grammar::Symbol] - key = [state, sym] - + state_item = get_state_item(state, item) @productions[state_item] = h[sym] + # @type var key: [State, Grammar::Symbol] + key = [state, sym] @reverse_productions[key] ||= Set.new - @reverse_productions[key] << item + @reverse_productions[key] << state_item end end end + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example def shift_reduce_example(conflict_state, conflict) conflict_symbol = conflict.symbols.first - # @type var shift_conflict_item: ::Lrama::States::Item + # @type var shift_conflict_item: ::Lrama::State::Item shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } - path2 = shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) - path1 = find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end Example.new(path1, path2, conflict, conflict_symbol, self) end + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example def reduce_reduce_examples(conflict_state, conflict) conflict_symbol = conflict.symbols.first - path1 = shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) - path2 = shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end Example.new(path1, path2, conflict, conflict_symbol, self) end - def find_shift_conflict_shortest_path(reduce_path, conflict_state, conflict_item) - state_items = find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - build_paths_from_state_items(state_items) - end + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 - def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - target_state_item = StateItem.new(conflict_state, conflict_item) + target_state_item = get_state_item(conflict_state, conflict_item) result = [target_state_item] - reversed_reduce_path = reduce_path.to_a.reverse + reversed_state_items = reduce_state_items.to_a.reverse # Index for state_item i = 0 - while (path = reversed_reduce_path[i]) + while (state_item = reversed_state_items[i]) # Index for prev_state_item j = i + 1 _j = j - while (prev_path = reversed_reduce_path[j]) - if prev_path.production? + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production j += 1 else break end end - state_item = path.to - prev_state_item = prev_path&.to - if target_state_item == state_item || target_state_item.item.start_item? result.concat( - reversed_reduce_path[_j..-1] #: Array[StartPath|TransitionPath|ProductionPath] - .map(&:to)) + reversed_state_items[_j..-1] #: Array[StateItem] + ) break end - if target_state_item.item.beginning_of_rule? - queue = [] #: Array[Array[StateItem]] - queue << [target_state_item] + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) # Find reverse production while (sis = queue.shift) - si = sis.last + @iterate_count += 1 + si = sis.elem # Reach to start state if si.item.start_item? - sis.shift - result.concat(sis) + a = Node.to_a(sis).reverse + a.shift + result.concat(a) target_state_item = si break end - if si.item.beginning_of_rule? + if si.type == :production # @type var key: [State, Grammar::Symbol] key = [si.state, si.item.lhs] - @reverse_productions[key].each do |item| - state_item = StateItem.new(si.state, item) - queue << (sis + [state_item]) + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) end else # @type var key: [StateItem, Grammar::Symbol] key = [si, si.item.previous_sym] @reverse_transitions[key].each do |prev_target_state_item| next if prev_target_state_item.state != prev_state_item&.state - sis.shift - result.concat(sis) + a = Node.to_a(sis).reverse + a.shift + result.concat(a) result << prev_target_state_item target_state_item = prev_target_state_item i = j @@ -216,68 +289,106 @@ module Lrama end end + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + result.reverse end - def build_paths_from_state_items(state_items) - state_items.zip([nil] + state_items).map do |si, prev_si| - case - when prev_si.nil? - StartPath.new(si) - when si.item.beginning_of_rule? - ProductionPath.new(prev_si, si) - else - TransitionPath.new(prev_si, si) + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end end end + + result end + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? def shortest_path(conflict_state, conflict_reduce_item, conflict_term) - # queue: is an array of [Triple, [Path]] - queue = [] #: Array[[Triple, Array[StartPath|TransitionPath|ProductionPath]]] + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] visited = {} #: Hash[Triple, true] start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) - start = Triple.new(start_state, start_state.kernels.first, Set.new([@states.eof_symbol])) + queue << [start, Path.new(start.state_item, nil)] - queue << [start, [StartPath.new(start.state_item)]] + while (triple, path = queue.shift) + @iterate_count += 1 - while true - triple, paths = queue.shift + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] - next if visited[triple] - visited[triple] = true + while (path = path.parent) + state_items << path.state_item + end - # Found - if triple.state == conflict_state && triple.item == conflict_reduce_item && triple.l.include?(conflict_term) - return paths + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse end # transition - triple.state.transitions.each do |shift, next_state| - next unless triple.item.next_sym && triple.item.next_sym == shift.next_sym - next_state.kernels.each do |kernel| - next if kernel.rule != triple.item.rule - t = Triple.new(next_state, kernel, triple.l) - queue << [t, paths + [TransitionPath.new(triple.state_item, t.state_item)]] + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] end end # production step - triple.state.closure.each do |item| - next unless triple.item.next_sym && triple.item.next_sym == item.lhs + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + l = follow_l(triple.item, triple.l) - t = Triple.new(triple.state, item, l) - queue << [t, paths + [ProductionPath.new(triple.state_item, t.state_item)]] + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end end - - break if queue.empty? end return nil end + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap def follow_l(item, current_l) # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal @@ -287,11 +398,28 @@ module Lrama when item.number_of_rest_symbols == 1 current_l when item.next_next_sym.term? - Set.new([item.next_next_sym]) + item.next_next_sym.number_bitmap when !item.next_next_sym.nullable - item.next_next_sym.first_set + item.next_next_sym.first_set_bitmap else - item.next_next_sym.first_set + follow_l(item.new_by_next_position, current_l) + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" end end end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb index 368d7f1032..a2b74767a9 100644 --- a/tool/lrama/lib/lrama/counterexamples/derivation.rb +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -1,34 +1,44 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Derivation - attr_reader :item, :left, :right - attr_writer :right + # @rbs! + # @item: State::Item + # @left: Derivation? - def initialize(item, left, right = nil) + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) @item = item @left = left - @right = right end + # @rbs () -> ::String def to_s "#<Derivation(#{item.display_name})>" end alias :inspect :to_s + # @rbs () -> Array[String] def render_strings_for_report result = [] #: Array[String] _render_for_report(self, 0, result, 0) result.map(&:rstrip) end + # @rbs () -> String def render_for_report render_strings_for_report.join("\n") end private + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer def _render_for_report(derivation, offset, strings, index) item = derivation.item if strings[index] diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb index bb08428fcd..c007f45af4 100644 --- a/tool/lrama/lib/lrama/counterexamples/example.rb +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -1,12 +1,31 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Example - attr_reader :path1, :path2, :conflict, :conflict_symbol + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol # path1 is shift conflict when S/R conflict # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void def initialize(path1, path2, conflict, conflict_symbol, counterexamples) @path1 = path1 @path2 = path2 @@ -15,69 +34,75 @@ module Lrama @counterexamples = counterexamples end + # @rbs () -> (:shift_reduce | :reduce_reduce) def type @conflict.type end + # @rbs () -> State::Item def path1_item - @path1.last.to.item + @path1.last.item end + # @rbs () -> State::Item def path2_item - @path2.last.to.item + @path2.last.item end + # @rbs () -> Derivation def derivations1 @derivations1 ||= _derivations(path1) end + # @rbs () -> Derivation def derivations2 @derivations2 ||= _derivations(path2) end private - def _derivations(paths) + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) derivation = nil #: Derivation current = :production - last_path = paths.last #: Path - lookahead_sym = last_path.to.item.end_of_rule? ? @conflict_symbol : nil + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil - paths.reverse_each do |path| - item = path.to.item + state_items.reverse_each do |si| + item = si.item case current when :production - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition derivation = Derivation.new(item, derivation) current = :transition - when ProductionPath + when :production derivation = Derivation.new(item, derivation) current = :production else - raise "Unexpected. #{path}" + raise "Unexpected. #{si}" end if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) - state_item = @counterexamples.transitions[[path.to, item.next_sym]] - derivation2 = find_derivation_for_symbol(state_item, lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) derivation.right = derivation2 # steep:ignore lookahead_sym = nil end when :transition - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition # ignore current = :transition - when ProductionPath + when :production # ignore current = :production end @@ -91,6 +116,7 @@ module Lrama derivation end + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? def find_derivation_for_symbol(state_item, sym) queue = [] #: Array[Array[StateItem]] queue << [state_item] @@ -110,9 +136,8 @@ module Lrama end if next_sym.nterm? && next_sym.first_set.include?(sym) - @counterexamples.productions[si].each do |next_item| - next if next_item.empty_rule? - next_si = StateItem.new(si.state, next_item) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? next if sis.include?(next_si) queue << (sis + [next_si]) end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb index 0a5823dd21..6b1325f73b 100644 --- a/tool/lrama/lib/lrama/counterexamples/path.rb +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -1,29 +1,27 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Path - def initialize(from_state_item, to_state_item) - @from_state_item = from_state_item - @to_state_item = to_state_item - end + # @rbs! + # @state_item: StateItem + # @parent: Path? - def from - @from_state_item - end + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? - def to - @to_state_item + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent end + # @rbs () -> ::String def to_s - "#<Path(#{type})>" + "#<Path>" end alias :inspect :to_s - - def type - raise NotImplementedError - end end end end diff --git a/tool/lrama/lib/lrama/counterexamples/production_path.rb b/tool/lrama/lib/lrama/counterexamples/production_path.rb deleted file mode 100644 index 0a230c7fce..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/production_path.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class ProductionPath < Path - def type - :production - end - - def transition? - false - end - - def production? - true - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/start_path.rb b/tool/lrama/lib/lrama/counterexamples/start_path.rb deleted file mode 100644 index c0351c8248..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/start_path.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class StartPath < Path - def initialize(to_state_item) - super nil, to_state_item - end - - def type - :start - end - - def transition? - false - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb index c919818324..8c2481d793 100644 --- a/tool/lrama/lib/lrama/counterexamples/state_item.rb +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -1,8 +1,31 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples - class StateItem < Struct.new(:state, :item) + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end end end end diff --git a/tool/lrama/lib/lrama/counterexamples/transition_path.rb b/tool/lrama/lib/lrama/counterexamples/transition_path.rb deleted file mode 100644 index 47bfbc4f98..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/transition_path.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class TransitionPath < Path - def type - :transition - end - - def transition? - true - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb index 64014ee223..98fe051f53 100644 --- a/tool/lrama/lib/lrama/counterexamples/triple.rb +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -1,21 +1,39 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples - # s: state - # itm: item within s - # l: precise lookahead set - class Triple < Struct.new(:s, :itm, :l) - alias :state :s - alias :item :itm - alias :precise_lookahead_set :l + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem def state_item - StateItem.new(state, item) + @state_item end + # @rbs () -> ::String def inspect - "#{state.inspect}. #{item.display_name}. #{l.map(&:id).map(&:s_value)}" + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" end alias :to_s :inspect end diff --git a/tool/lrama/lib/lrama/diagnostics.rb b/tool/lrama/lib/lrama/diagnostics.rb deleted file mode 100644 index e9da398c89..0000000000 --- a/tool/lrama/lib/lrama/diagnostics.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Diagnostics - def initialize(grammar, states, logger) - @grammar = grammar - @states = states - @logger = logger - end - - def run(diagnostic) - if diagnostic - diagnose_conflict - diagnose_parameterizing_redefined - end - end - - private - - def diagnose_conflict - if @states.sr_conflicts_count != 0 - @logger.warn("shift/reduce conflicts: #{@states.sr_conflicts_count} found") - end - - if @states.rr_conflicts_count != 0 - @logger.warn("reduce/reduce conflicts: #{@states.rr_conflicts_count} found") - end - end - - def diagnose_parameterizing_redefined - @grammar.parameterizing_rule_resolver.redefined_rules.each do |rule| - @logger.warn("parameterizing rule redefined: #{rule}") - end - end - end -end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb index 2161f30474..52865f52dd 100644 --- a/tool/lrama/lib/lrama/digraph.rb +++ b/tool/lrama/lib/lrama/digraph.rb @@ -2,13 +2,34 @@ # frozen_string_literal: true module Lrama - # Algorithm Digraph of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) # - # @rbs generic X < Object -- Type of a member of `sets` - # @rbs generic Y < _Or -- Type of sets assigned to a member of `sets` + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) class Digraph - # TODO: rbs-inline 0.10.0 doesn't support instance variables. + # TODO: rbs-inline 0.11.0 doesn't support instance variables. # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 # # @rbs! # interface _Or @@ -21,9 +42,9 @@ module Lrama # @h: Hash[X, (Integer|Float)?] # @result: Hash[X, Y] - # @rbs sets: Array[X] - # @rbs relation: Hash[X, Array[X]] - # @rbs base_function: Hash[X, Y] + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes # @rbs return: void def initialize(sets, relation, base_function) diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb index 214ca1a3f2..95a80bb01c 100644 --- a/tool/lrama/lib/lrama/grammar.rb +++ b/tool/lrama/lib/lrama/grammar.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" @@ -7,7 +8,8 @@ require_relative "grammar/code" require_relative "grammar/counter" require_relative "grammar/destructor" require_relative "grammar/error_token" -require_relative "grammar/parameterizing_rule" +require_relative "grammar/inline" +require_relative "grammar/parameterized" require_relative "grammar/percent_code" require_relative "grammar/precedence" require_relative "grammar/printer" @@ -23,19 +25,89 @@ require_relative "lexer" module Lrama # Grammar is the result of parsing an input grammar file class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + extend Forwardable - attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux, :parameterizing_rule_resolver - attr_accessor :union, :expect, :printers, :error_tokens, :lex_param, :parse_param, :initial_action, - :after_shift, :before_reduce, :after_reduce, :after_shift_error_token, :after_pop_stack, - :symbols_resolver, :types, :rules, :rule_builders, :sym_to_rules, :no_stdlib, :locations, :define + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! - def initialize(rule_counter, define = {}) + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) @rule_counter = rule_counter # Code defined by "%code" @@ -48,7 +120,7 @@ module Lrama @rule_builders = [] @rules = [] @sym_to_rules = {} - @parameterizing_rule_resolver = ParameterizingRule::Resolver.new + @parameterized_resolver = Parameterized::Resolver.new @empty_symbol = nil @eof_symbol = nil @error_symbol = nil @@ -56,93 +128,131 @@ module Lrama @accept_symbol = nil @aux = Auxiliary.new @no_stdlib = false - @locations = false - @define = define.map {|d| d.split('=') }.to_h + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil append_special_symbols end + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder def create_rule_builder(rule_counter, midrule_action_counter) - RuleBuilder.new(rule_counter, midrule_action_counter, @parameterizing_rule_resolver) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) end + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] def add_percent_code(id:, code:) @percent_codes << PercentCode.new(id.s_value, code.s_value) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] def add_destructor(ident_or_tags:, token_code:, lineno:) @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] def add_printer(ident_or_tags:, token_code:, lineno:) @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] def add_error_token(ident_or_tags:, token_code:, lineno:) @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] def add_type(id:, tag:) @types << Type.new(id: id, tag: tag) end - def add_nonassoc(sym, precedence) - set_precedence(sym, Precedence.new(type: :nonassoc, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) end - def add_left(sym, precedence) - set_precedence(sym, Precedence.new(type: :left, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) end - def add_right(sym, precedence) - set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) end - def add_precedence(sym, precedence) - set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end end + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) def set_precedence(sym, precedence) - raise "" if sym.nterm? + @precedences << precedence sym.precedence = precedence end + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union def set_union(code, lineno) @union = Union.new(code: code, lineno: lineno) end + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] def add_rule_builder(builder) @rule_builders << builder end - def add_parameterizing_rule(rule) - @parameterizing_rule_resolver.add_parameterizing_rule(rule) + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) end - def parameterizing_rules - @parameterizing_rule_resolver.rules + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules end - def insert_before_parameterizing_rules(rules) - @parameterizing_rule_resolver.rules = rules + @parameterizing_rule_resolver.rules + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules end + # @rbs (Integer prologue_first_lineno) -> Integer def prologue_first_lineno=(prologue_first_lineno) @aux.prologue_first_lineno = prologue_first_lineno end + # @rbs (String prologue) -> String def prologue=(prologue) @aux.prologue = prologue end + # @rbs (Integer epilogue_first_lineno) -> Integer def epilogue_first_lineno=(epilogue_first_lineno) @aux.epilogue_first_lineno = epilogue_first_lineno end + # @rbs (String epilogue) -> String def epilogue=(epilogue) @aux.epilogue = epilogue end + # @rbs () -> void def prepare resolve_inline_rules normalize_rules @@ -151,6 +261,7 @@ module Lrama fill_default_precedence fill_symbols fill_sym_to_rules + sort_precedence compute_nullable compute_first_set set_locations @@ -159,25 +270,51 @@ module Lrama # TODO: More validation methods # # * Validation for no_declared_type_reference + # + # @rbs () -> void def validate! @symbols_resolver.validate! + validate_no_precedence_for_nterm! validate_rule_lhs_is_nterm! + validate_duplicated_precedence! end + # @rbs (Grammar::Symbol sym) -> Array[Rule] def find_rules_by_symbol!(sym) find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") end + # @rbs (Grammar::Symbol sym) -> Array[Rule]? def find_rules_by_symbol(sym) @sym_to_rules[sym.number] end + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool def ielr_defined? @define.key?('lr.type') && @define['lr.type'] == 'ielr' end private + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] def compute_nullable @rules.each do |rule| case @@ -227,6 +364,7 @@ module Lrama end end + # @rbs () -> Array[Grammar::Symbol] def compute_first_set terms.each do |term| term.first_set = Set.new([term]).freeze @@ -262,12 +400,14 @@ module Lrama end end + # @rbs () -> Array[RuleBuilder] def setup_rules @rule_builders.each do |builder| builder.setup_rules end end + # @rbs () -> Grammar::Symbol def append_special_symbols # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) @@ -298,11 +438,12 @@ module Lrama @accept_symbol = term end + # @rbs () -> void def resolve_inline_rules while @rule_builders.any?(&:has_inline_rules?) do @rule_builders = @rule_builders.flat_map do |builder| if builder.has_inline_rules? - builder.resolve_inline_rules + Inline::Resolver.new(builder).resolve else builder end @@ -310,14 +451,10 @@ module Lrama end end + # @rbs () -> void def normalize_rules - # Add $accept rule to the top of rules - rule_builder = @rule_builders.first # : RuleBuilder - lineno = rule_builder ? rule_builder.line : 0 - @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [rule_builder.lhs, @eof_symbol.id], token_code: nil, lineno: lineno) - + add_accept_rule setup_rules - @rule_builders.each do |builder| builder.rules.each do |rule| add_nterm(id: rule._lhs, tag: rule.lhs_tag) @@ -325,23 +462,42 @@ module Lrama end end - @rules.sort_by!(&:id) + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end end # Collect symbols from rules + # + # @rbs () -> void def collect_symbols @rules.flat_map(&:_rhs).each do |s| case s when Lrama::Lexer::Token::Char add_term(id: s) - when Lrama::Lexer::Token + when Lrama::Lexer::Token::Base # skip else raise "Unknown class: #{s}" end end + + terms.freeze end + # @rbs () -> void def set_lhs_and_rhs @rules.each do |rule| rule.lhs = token_to_symbol(rule._lhs) if rule._lhs @@ -355,6 +511,8 @@ module Lrama # Rule inherits precedence from the last term in RHS. # # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void def fill_default_precedence @rules.each do |rule| # Explicitly specified precedence has the highest priority @@ -369,6 +527,7 @@ module Lrama end end + # @rbs () -> Array[Grammar::Symbol] def fill_symbols fill_symbol_number fill_nterm_type(@types) @@ -378,6 +537,7 @@ module Lrama sort_by_number! end + # @rbs () -> Array[Rule] def fill_sym_to_rules @rules.each do |rule| key = rule.lhs.number @@ -386,13 +546,48 @@ module Lrama end end + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void def validate_rule_lhs_is_nterm! errors = [] #: Array[String] rules.each do |rule| next if rule.lhs.nterm? - errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is term. It should be nterm." + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end end return if errors.empty? @@ -400,6 +595,7 @@ module Lrama raise errors.join("\n") end + # @rbs () -> void def set_locations @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb index 2bacee6f1a..76cfb74d4d 100644 --- a/tool/lrama/lib/lrama/grammar/auxiliary.rb +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -1,9 +1,14 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar # Grammar file information not used by States but by Output - class Auxiliary < Struct.new(:prologue_first_lineno, :prologue, :epilogue_first_lineno, :epilogue, keyword_init: true) + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? end end end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb index 2efb918a0b..94d00a410e 100644 --- a/tool/lrama/lib/lrama/grammar/binding.rb +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -4,51 +4,64 @@ module Lrama class Grammar class Binding - # @rbs @actual_args: Array[Lexer::Token] - # @rbs @param_to_arg: Hash[String, Lexer::Token] + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] - # @rbs (Array[Lexer::Token] params, Array[Lexer::Token] actual_args) -> void + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void def initialize(params, actual_args) @actual_args = actual_args - @param_to_arg = map_params_to_args(params, @actual_args) + @param_to_arg = build_param_to_arg(params, @actual_args) end - # @rbs (Lexer::Token sym) -> Lexer::Token + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base def resolve_symbol(sym) - if sym.is_a?(Lexer::Token::InstantiateRule) - Lrama::Lexer::Token::InstantiateRule.new( - s_value: sym.s_value, location: sym.location, args: resolved_args(sym), lhs_tag: sym.lhs_tag - ) - else - param_to_arg(sym) - end + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) end # @rbs (Lexer::Token::InstantiateRule token) -> String def concatenated_args_str(token) - "#{token.rule_name}_#{token_to_args_s_values(token).join('_')}" + "#{token.rule_name}_#{format_args(token)}" end private - # @rbs (Array[Lexer::Token] params, Array[Lexer::Token] actual_args) -> Hash[String, Lexer::Token] - def map_params_to_args(params, actual_args) - params.zip(actual_args).map do |param, arg| - [param.s_value, arg] - end.to_h + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + alias_name: sym.alias_name, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) end - # @rbs (Lexer::Token::InstantiateRule sym) -> Array[Lexer::Token] - def resolved_args(sym) - sym.args.map { |arg| resolve_symbol(arg) } + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } end - # @rbs (Lexer::Token sym) -> Lexer::Token - def param_to_arg(sym) - if (arg = @param_to_arg[sym.s_value].dup) + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) arg.alias_name = sym.alias_name + arg + else + sym end - arg || sym + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') end # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb index b6c1cc49e7..f1b860eeba 100644 --- a/tool/lrama/lib/lrama/grammar/code.rb +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" @@ -10,17 +11,28 @@ require_relative "code/rule_action" module Lrama class Grammar class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + extend Forwardable def_delegators "token_code", :s_value, :line, :column, :references - attr_reader :type, :token_code + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void def initialize(type:, token_code:) @type = type @token_code = token_code end + # @rbs (Code other) -> bool def ==(other) self.class == other.class && self.type == other.type && @@ -28,6 +40,8 @@ module Lrama end # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String def translated_code t_code = s_value.dup @@ -45,6 +59,7 @@ module Lrama private + # @rbs (Lrama::Grammar::Reference ref) -> bot def reference_to_c(ref) raise NotImplementedError.new("#reference_to_c is not implemented") end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb index 794017257c..d71b62e513 100644 --- a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -1,9 +1,18 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void def initialize(type:, token_code:, tag:) super(type: type, token_code: token_code) @tag = tag @@ -17,6 +26,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb index 02f2badc9e..cb36041524 100644 --- a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -12,6 +13,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb index ab12f32e29..1d39919979 100644 --- a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -12,6 +13,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> bot def reference_to_c(ref) case when ref.type == :dollar # $$, $n diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb index c0b8d24306..c6e25d5235 100644 --- a/tool/lrama/lib/lrama/grammar/code/printer_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -1,9 +1,18 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void def initialize(type:, token_code:, tag:) super(type: type, token_code: token_code) @tag = tag @@ -17,6 +26,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb index 363ecdf25d..24729a1ee0 100644 --- a/tool/lrama/lib/lrama/grammar/code/rule_action.rb +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -1,12 +1,23 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class RuleAction < Code - def initialize(type:, token_code:, rule:) + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + # @grammar: Grammar + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule, grammar: Grammar) -> void + def initialize(type:, token_code:, rule:, grammar:) super(type: type, token_code: token_code) @rule = rule + @grammar = grammar end private @@ -38,13 +49,21 @@ module Lrama # "Position in grammar" $1 # "Index for yyvsp" 0 # "$:n" $:1 + # + # @rbs (Reference ref) -> String def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ tag = ref.ex_tag || lhs.tag - raise_tag_not_found_error(ref) unless tag - # @type var tag: Lexer::Token::Tag - "(yyval.#{tag.member})" + if tag + # @type var tag: Lexer::Token::Tag + "(yyval.#{tag.member})" + elsif union_not_defined? + # When %union is not defined, YYSTYPE defaults to int + "(yyval)" + else + raise_tag_not_found_error(ref) + end when ref.type == :at && ref.name == "$" # @$ "(yyloc)" when ref.type == :index && ref.name == "$" # $:$ @@ -52,9 +71,15 @@ module Lrama when ref.type == :dollar # $n i = -position_in_rhs + ref.index tag = ref.ex_tag || rhs[ref.index - 1].tag - raise_tag_not_found_error(ref) unless tag - # @type var tag: Lexer::Token::Tag - "(yyvsp[#{i}].#{tag.member})" + if tag + # @type var tag: Lexer::Token::Tag + "(yyvsp[#{i}].#{tag.member})" + elsif union_not_defined? + # When %union is not defined, YYSTYPE defaults to int + "(yyvsp[#{i}])" + else + raise_tag_not_found_error(ref) + end when ref.type == :at # @n i = -position_in_rhs + ref.index "(yylsp[#{i}])" @@ -66,6 +91,7 @@ module Lrama end end + # @rbs () -> Integer def position_in_rhs # If rule is not derived rule, User Code is only action at # the end of rule RHS. In such case, the action is located on @@ -74,15 +100,25 @@ module Lrama end # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] def rhs (@rule.original_rule || @rule).rhs end # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol def lhs @rule.lhs end + # @rbs () -> bool + def union_not_defined? + @grammar.union.nil? + end + + # @rbs (Reference ref) -> bot def raise_tag_not_found_error(ref) raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb index dc91b87b71..ced934309d 100644 --- a/tool/lrama/lib/lrama/grammar/counter.rb +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -1,12 +1,22 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void def initialize(number) @number = number end + # @rbs () -> Integer def increment n = @number @number += 1 diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb index a2b6fde0ed..0ce8611e77 100644 --- a/tool/lrama/lib/lrama/grammar/destructor.rb +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Destructor < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb index 50eaafeebc..9d9ed54ae2 100644 --- a/tool/lrama/lib/lrama/grammar/error_token.rb +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ErrorToken < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb index 06f2f1cef7..558f308190 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -1,40 +1,49 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ParameterizingRule + class Parameterized class Resolver - attr_accessor :rules, :created_lhs_list + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + # @rbs () -> void def initialize @rules = [] @created_lhs_list = [] end - def add_parameterizing_rule(rule) + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) @rules << rule end + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? def find_rule(token) select_rules(@rules, token).last end + # @rbs (Lexer::Token::Base token) -> Rule? def find_inline(token) - @rules.reverse.find { |rule| rule.name == token.s_value && rule.is_inline } + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } end + # @rbs (String lhs_s_value) -> Lexer::Token::Base? def created_lhs(lhs_s_value) @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } end + # @rbs () -> Array[Rule] def redefined_rules @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } end private + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] def select_rules(rules, token) - rules = select_not_inline_rules(rules) + rules = reject_inline_rules(rules) rules = select_rules_by_name(rules, token.rule_name) rules = rules.select { |rule| rule.required_parameters_count == token.args_count } if rules.empty? @@ -44,14 +53,16 @@ module Lrama end end - def select_not_inline_rules(rules) - rules.select { |rule| !rule.is_inline } + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) end + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] def select_rules_by_name(rules, rule_name) rules = rules.select { |rule| rule.name == rule_name } if rules.empty? - raise "Parameterizing rule does not exist. `#{rule_name}`" + raise "Parameterized rule does not exist. `#{rule_name}`" else rules end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb index f60781c053..663de49100 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -1,17 +1,22 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ParameterizingRule + class Parameterized class Rhs - attr_accessor :symbols, :user_code, :precedence_sym + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + # @rbs () -> void def initialize @symbols = [] @user_code = nil @precedence_sym = nil end + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? def resolve_user_code(bindings) return unless user_code diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb deleted file mode 100644 index ddc1a467ce..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -require_relative 'parameterizing_rule/resolver' -require_relative 'parameterizing_rule/rhs' -require_relative 'parameterizing_rule/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb deleted file mode 100644 index cc200d2fb6..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Grammar - class ParameterizingRule - class Rule - attr_reader :name, :parameters, :rhs_list, :required_parameters_count, :tag, :is_inline - - def initialize(name, parameters, rhs_list, tag: nil, is_inline: false) - @name = name - @parameters = parameters - @rhs_list = rhs_list - @tag = tag - @is_inline = is_inline - @required_parameters_count = parameters.count - end - - def to_s - "#{@name}(#{@parameters.map(&:s_value).join(', ')})" - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb index 416a2d2753..9afb903056 100644 --- a/tool/lrama/lib/lrama/grammar/percent_code.rb +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -1,10 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class PercentCode - attr_reader :name, :code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void def initialize(name, code) @name = name @code = code diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb index 13cf960c32..b4c6403372 100644 --- a/tool/lrama/lib/lrama/grammar/precedence.rb +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -1,13 +1,55 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Precedence < Struct.new(:type, :precedence, keyword_init: true) + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer def <=>(other) self.precedence <=> other.precedence end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end end end end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb index b78459e819..490fe701db 100644 --- a/tool/lrama/lib/lrama/grammar/printer.rb +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -1,8 +1,17 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb index b044516bdb..7e3badfecc 100644 --- a/tool/lrama/lib/lrama/grammar/reference.rb +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -8,6 +9,18 @@ module Lrama # index: Integer # ex_tag: "$<tag>1" (Optional) class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) def value name || number end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb index 445752ae0d..b023b0e454 100644 --- a/tool/lrama/lib/lrama/grammar/rule.rb +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -1,11 +1,38 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar # _rhs holds original RHS element. Use rhs to refer to Symbol. class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) - attr_accessor :original_rule + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool def ==(other) self.class == other.class && self.lhs == other.lhs && @@ -18,12 +45,14 @@ module Lrama self.lineno == other.lineno end + # @rbs () -> String def display_name l = lhs.id.s_value r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") "#{l} -> #{r}" end + # @rbs () -> String def display_name_without_action l = lhs.id.s_value r = empty_rule? ? "ε" : rhs.map do |r| @@ -33,7 +62,18 @@ module Lrama "#{l} -> #{r}" end + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + # Used by #user_actions + # + # @rbs () -> String def as_comment l = lhs.id.s_value r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") @@ -41,35 +81,55 @@ module Lrama "#{l}: #{r}" end + # @rbs () -> String def with_actions "#{display_name} {#{token_code&.s_value}}" end # opt_nl: ε <-- empty_rule # | '\n' <-- not empty_rule + # + # @rbs () -> bool def empty_rule? rhs.empty? end + # @rbs () -> Precedence? def precedence precedence_sym&.precedence end + # @rbs () -> bool def initial_rule? id == 0 end - def translated_code + # @rbs (Grammar grammar) -> String? + def translated_code(grammar) return nil unless token_code - Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self, grammar: grammar).translated_code end + # @rbs () -> bool def contains_at_reference? return false unless token_code token_code.references.any? {|r| r.type == :at } end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end end end end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb index 481a3780f4..34fdca6c86 100644 --- a/tool/lrama/lib/lrama/grammar/rule_builder.rb +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -1,15 +1,38 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class RuleBuilder - attr_accessor :lhs, :line - attr_reader :lhs_tag, :rhs, :user_code, :precedence_sym - - def initialize(rule_counter, midrule_action_counter, parameterizing_rule_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) @rule_counter = rule_counter @midrule_action_counter = midrule_action_counter - @parameterizing_rule_resolver = parameterizing_rule_resolver + @parameterized_resolver = parameterized_resolver @position_in_original_rule_rhs = position_in_original_rule_rhs @skip_preprocess_references = skip_preprocess_references @@ -20,12 +43,13 @@ module Lrama @precedence_sym = nil @line = nil @rules = [] - @rule_builders_for_parameterizing_rules = [] + @rule_builders_for_parameterized = [] @rule_builders_for_derived_rules = [] - @parameterizing_rules = [] + @parameterized_rules = [] @midrule_action_rules = [] end + # @rbs (Lexer::Token::Base rhs) -> void def add_rhs(rhs) @line ||= rhs.line @@ -34,6 +58,7 @@ module Lrama @rhs << rhs end + # @rbs (Lexer::Token::UserCode? user_code) -> void def user_code=(user_code) @line ||= user_code&.line @@ -42,72 +67,59 @@ module Lrama @user_code = user_code end + # @rbs (Grammar::Symbol? precedence_sym) -> void def precedence_sym=(precedence_sym) flush_user_code @precedence_sym = precedence_sym end + # @rbs () -> void def complete_input freeze_rhs end + # @rbs () -> void def setup_rules preprocess_references unless @skip_preprocess_references process_rhs + resolve_inline_rules build_rules end + # @rbs () -> Array[Grammar::Rule] def rules - @parameterizing_rules + @midrule_action_rules + @rules + @parameterized_rules + @midrule_action_rules + @rules end + # @rbs () -> bool def has_inline_rules? - rhs.any? { |token| @parameterizing_rule_resolver.find_inline(token) } - end - - def resolve_inline_rules - resolved_builders = [] #: Array[RuleBuilder] - rhs.each_with_index do |token, i| - if (inline_rule = @parameterizing_rule_resolver.find_inline(token)) - inline_rule.rhs_list.each do |inline_rhs| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: lhs_tag) - if token.is_a?(Lexer::Token::InstantiateRule) - resolve_inline_rhs(rule_builder, inline_rhs, i, Binding.new(inline_rule.parameters, token.args)) - else - resolve_inline_rhs(rule_builder, inline_rhs, i) - end - rule_builder.lhs = lhs - rule_builder.line = line - rule_builder.precedence_sym = precedence_sym - rule_builder.user_code = replace_inline_user_code(inline_rhs, i) - resolved_builders << rule_builder - end - break - end - end - resolved_builders + rhs.any? { |token| @parameterized_resolver.find_inline(token) } end private + # @rbs () -> void def freeze_rhs @rhs.freeze end + # @rbs () -> void def preprocess_references numberize_references end + # @rbs () -> void def build_rules - tokens = @replaced_rhs + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } rule = Rule.new( id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line ) @rules = [rule] - @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder| + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| rule_builder.rules end.flatten @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| @@ -120,31 +132,33 @@ module Lrama # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void def process_rhs return if @replaced_rhs - @replaced_rhs = [] + replaced_rhs = [] #: Array[Lexer::Token::Base] rhs.each_with_index do |token, i| case token when Lrama::Lexer::Token::Char - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::Ident - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::InstantiateRule - parameterizing_rule = @parameterizing_rule_resolver.find_rule(token) - raise "Unexpected token. #{token}" unless parameterizing_rule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule - bindings = Binding.new(parameterizing_rule.parameters, token.args) + bindings = Binding.new(parameterized_rule.parameters, token.args) lhs_s_value = bindings.concatenated_args_str(token) - if (created_lhs = @parameterizing_rule_resolver.created_lhs(lhs_s_value)) - @replaced_rhs << created_lhs + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs else lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) - @replaced_rhs << lhs_token - @parameterizing_rule_resolver.created_lhs_list << lhs_token - parameterizing_rule.rhs_list.each do |r| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: token.lhs_tag || parameterizing_rule.tag) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) rule_builder.lhs = lhs_token r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } rule_builder.line = line @@ -152,51 +166,48 @@ module Lrama rule_builder.user_code = r.resolve_user_code(bindings) rule_builder.complete_input rule_builder.setup_rules - @rule_builders_for_parameterizing_rules << rule_builder + @rule_builders_for_parameterized << rule_builder end end when Lrama::Lexer::Token::UserCode prefix = token.referred ? "@" : "$@" tag = token.tag || lhs_tag new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) - @replaced_rhs << new_token + replaced_rhs << new_token - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) rule_builder.lhs = new_token rule_builder.user_code = token rule_builder.complete_input rule_builder.setup_rules @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop else raise "Unexpected token. #{token}" end end - end - def resolve_inline_rhs(rule_builder, inline_rhs, index, bindings = nil) - rhs.each_with_index do |token, i| - if index == i - inline_rhs.symbols.each { |sym| rule_builder.add_rhs(bindings.nil? ? sym : bindings.resolve_symbol(sym)) } - else - rule_builder.add_rhs(token) - end - end + @replaced_rhs = replaced_rhs end - def replace_inline_user_code(inline_rhs, index) - return user_code if inline_rhs.user_code.nil? - return user_code if user_code.nil? - - code = user_code.s_value.gsub(/\$#{index + 1}/, inline_rhs.user_code.s_value) - user_code.references.each do |ref| - next if ref.index.nil? || ref.index <= index # nil is a case for `$$` - code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (inline_rhs.symbols.count-1)}") - code = code.gsub(/@#{ref.index}/, "@#{ref.index + (inline_rhs.symbols.count-1)}") + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end + end end - Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) end + # @rbs () -> void def numberize_references # Bison n'th component is 1-origin (rhs + [user_code]).compact.each.with_index(1) do |token, i| @@ -209,7 +220,10 @@ module Lrama if ref_name == '$' ref.name = '$' else - candidates = ([lhs] + rhs).each_with_index.select {|token, _i| token.referred_by?(ref_name) } + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end if candidates.size >= 2 token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") @@ -244,6 +258,7 @@ module Lrama end end + # @rbs () -> void def flush_user_code if (c = @user_code) @rhs << c diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y index d6e89c908c..dd397c9e08 100644 --- a/tool/lrama/lib/lrama/grammar/stdlib.y +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -3,26 +3,43 @@ stdlib.y This is lrama's standard library. It provides a number of - parameterizing rule definitions, such as options and lists, + parameterized rule definitions, such as options and lists, that should be useful in a number of situations. **********************************************************************/ +%% + // ------------------------------------------------------------------- // Options /* - * program: option(number) + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) * * => * - * program: option_number - * option_number: %empty - * option_number: number + * program: %empty + * program: X */ -%rule option(X): /* empty */ - | X - ; +%rule %inline ioption(X) + : /* empty */ + | X + ; // ------------------------------------------------------------------- // Sequences @@ -35,8 +52,9 @@ * program: preceded_opening_X * preceded_opening_X: opening X */ -%rule preceded(opening, X): opening X { $$ = $2; } - ; +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; /* * program: terminated(X, closing) @@ -46,8 +64,9 @@ * program: terminated_X_closing * terminated_X_closing: X closing */ -%rule terminated(X, closing): X closing { $$ = $1; } - ; +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; /* * program: delimited(opening, X, closing) @@ -57,66 +76,67 @@ * program: delimited_opening_X_closing * delimited_opening_X_closing: opening X closing */ -%rule delimited(opening, X, closing): opening X closing { $$ = $2; } - ; +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; // ------------------------------------------------------------------- // Lists /* - * program: list(number) + * program: list(X) * * => * - * program: list_number - * list_number: %empty - * list_number: list_number number + * program: list_X + * list_X: %empty + * list_X: list_X X */ -%rule list(X): /* empty */ - | list(X) X - ; +%rule list(X) + : /* empty */ + | list(X) X + ; /* - * program: nonempty_list(number) + * program: nonempty_list(X) * * => * - * program: nonempty_list_number - * nonempty_list_number: number - * nonempty_list_number: nonempty_list_number number + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X */ -%rule nonempty_list(X): X - | nonempty_list(X) X - ; +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; /* - * program: separated_nonempty_list(comma, number) + * program: separated_nonempty_list(separator, X) * * => * - * program: separated_nonempty_list_comma_number - * separated_nonempty_list_comma_number: number - * separated_nonempty_list_comma_number: separated_nonempty_list_comma_number comma number + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X */ -%rule separated_nonempty_list(separator, X): X - | separated_nonempty_list(separator, X) separator X - ; +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; /* - * program: separated_list(comma, number) + * program: separated_list(separator, X) * * => * - * program: separated_list_comma_number - * separated_list_comma_number: option_separated_nonempty_list_comma_number - * option_separated_nonempty_list_comma_number: %empty - * option_separated_nonempty_list_comma_number: separated_nonempty_list_comma_number - * separated_nonempty_list_comma_number: number - * separated_nonempty_list_comma_number: comma separated_nonempty_list_comma_number number + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X */ -%rule separated_list(separator, X): option(separated_nonempty_list(separator, X)) - ; - -%% - -%union{}; +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb index f9dffcad6c..07aee0c0a2 100644 --- a/tool/lrama/lib/lrama/grammar/symbol.rb +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -1,19 +1,35 @@ +# rbs_inline: enabled # frozen_string_literal: true # Symbol is both of nterm and term # `number` is both for nterm and term # `token_id` is tokentype for term, internal sequence number for nterm # -# TODO: Add validation for ASCII code range for Token::Char module Lrama class Grammar class Symbol - attr_accessor :id, :alias_name, :tag, :number, :token_id, :nullable, :precedence, - :printer, :destructor, :error_token, :first_set, :first_set_bitmap - attr_reader :term - attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) @id = id @alias_name = alias_name @@ -27,77 +43,105 @@ module Lrama @destructor = destructor end + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool def term? term end + # @rbs () -> bool def nterm? !term end + # @rbs () -> bool def eof_symbol? !!@eof_symbol end + # @rbs () -> bool def error_symbol? !!@error_symbol end + # @rbs () -> bool def undef_symbol? !!@undef_symbol end + # @rbs () -> bool def accept_symbol? !!@accept_symbol end + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String def display_name - alias_name || id.s_value + alias_name || name end # name for yysymbol_kind_t # # See: b4_symbol_kind_base # @type var name: String + # @rbs () -> String def enum_name case when accept_symbol? - name = "YYACCEPT" + res = "YYACCEPT" when eof_symbol? - name = "YYEOF" + res = "YYEOF" when term? && id.is_a?(Lrama::Lexer::Token::Char) - name = number.to_s + display_name + res = number.to_s + display_name when term? && id.is_a?(Lrama::Lexer::Token::Ident) - name = id.s_value - when nterm? && (id.s_value.include?("$") || id.s_value.include?("@")) - name = number.to_s + id.s_value + res = name + when midrule? + res = number.to_s + name when nterm? - name = id.s_value + res = name else raise "Unexpected #{self}" end - "YYSYMBOL_" + name.gsub(/\W+/, "_") + "YYSYMBOL_" + res.gsub(/\W+/, "_") end # comment for yysymbol_kind_t + # + # @rbs () -> String? def comment case when accept_symbol? # YYSYMBOL_YYACCEPT - id.s_value + name when eof_symbol? # YYEOF alias_name when (term? && 0 < token_id && token_id < 128) # YYSYMBOL_3_backslash_, YYSYMBOL_14_ - alias_name || id.s_value - when id.s_value.include?("$") || id.s_value.include?("@") + display_name + when midrule? # YYSYMBOL_21_1 - id.s_value + name else # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 - alias_name || id.s_value + display_name end end end diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb index 52f4ff90bd..085a835d28 100644 --- a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -1,24 +1,54 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Symbols class Resolver - attr_reader :terms, :nterms - + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void def initialize @terms = [] @nterms = [] end + # @rbs () -> Array[Grammar::Symbol] def symbols @symbols ||= (@terms + @nterms) end + # @rbs () -> Array[Grammar::Symbol] def sort_by_number! symbols.sort_by!(&:number) end + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) if token_id && (sym = find_symbol_by_token_id(token_id)) if replace @@ -43,6 +73,7 @@ module Lrama term end + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol def add_nterm(id:, alias_name: nil, tag: nil) if (sym = find_symbol_by_id(id)) return sym @@ -57,32 +88,39 @@ module Lrama nterm end + # @rbs (::String s_value) -> Grammar::Symbol? def find_term_by_s_value(s_value) terms.find { |s| s.id.s_value == s_value } end + # @rbs (::String s_value) -> Grammar::Symbol? def find_symbol_by_s_value(s_value) symbols.find { |s| s.id.s_value == s_value } end + # @rbs (::String s_value) -> Grammar::Symbol def find_symbol_by_s_value!(s_value) find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") end + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? def find_symbol_by_id(id) symbols.find do |s| s.id == id || s.alias_name == id.s_value end end + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol def find_symbol_by_id!(id) find_symbol_by_id(id) || (raise "Symbol not found. #{id}") end + # @rbs (Integer token_id) -> Grammar::Symbol? def find_symbol_by_token_id(token_id) symbols.find {|s| s.token_id == token_id } end + # @rbs (Integer number) -> Grammar::Symbol def find_symbol_by_number!(number) sym = symbols[number] @@ -92,6 +130,7 @@ module Lrama sym end + # @rbs () -> void def fill_symbol_number # YYEMPTY = -2 # YYEOF = 0 @@ -102,6 +141,7 @@ module Lrama fill_nterms_number end + # @rbs (Array[Grammar::Type] types) -> void def fill_nterm_type(types) types.each do |type| nterm = find_nterm_by_id!(type.id) @@ -109,6 +149,7 @@ module Lrama end end + # @rbs (Array[Grammar::Printer] printers) -> void def fill_printer(printers) symbols.each do |sym| printers.each do |printer| @@ -126,6 +167,7 @@ module Lrama end end + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) def fill_destructor(destructors) symbols.each do |sym| destructors.each do |destructor| @@ -143,6 +185,7 @@ module Lrama end end + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void def fill_error_token(error_tokens) symbols.each do |sym| error_tokens.each do |token| @@ -160,28 +203,33 @@ module Lrama end end + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol def token_to_symbol(token) case token - when Lrama::Lexer::Token + when Lrama::Lexer::Token::Base find_symbol_by_id!(token) else raise "Unknown class: #{token}" end end + # @rbs () -> void def validate! validate_number_uniqueness! validate_alias_name_uniqueness! + validate_symbols! end private + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol def find_nterm_by_id!(id) @nterms.find do |s| s.id == id end || (raise "Symbol not found. #{id}") end + # @rbs () -> void def fill_terms_number # Character literal in grammar file has # token id corresponding to ASCII code by default, @@ -245,6 +293,7 @@ module Lrama end end + # @rbs () -> void def fill_nterms_number token_id = 0 @@ -266,6 +315,7 @@ module Lrama end end + # @rbs () -> Hash[Integer, bool] def used_numbers return @used_numbers if defined?(@used_numbers) @@ -276,6 +326,7 @@ module Lrama @used_numbers end + # @rbs () -> void def validate_number_uniqueness! invalid = symbols.group_by(&:number).select do |number, syms| syms.count > 1 @@ -286,6 +337,7 @@ module Lrama raise "Symbol number is duplicated. #{invalid}" end + # @rbs () -> void def validate_alias_name_uniqueness! invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| syms.count > 1 @@ -295,6 +347,15 @@ module Lrama raise "Symbol alias name is duplicated. #{invalid}" end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end end end end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb index 65537288b3..c631769447 100644 --- a/tool/lrama/lib/lrama/grammar/type.rb +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -1,15 +1,27 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Type - attr_reader :id, :tag + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void def initialize(id:, tag:) @id = id @tag = tag end + # @rbs (Grammar::Type other) -> bool def ==(other) self.class == other.class && self.id == other.id && diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb index 5f1bee0069..774cc66fc6 100644 --- a/tool/lrama/lib/lrama/grammar/union.rb +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -1,8 +1,19 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Union < Struct.new(:code, :lineno, keyword_init: true) + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String def braces_less_code # Braces is already removed by lexer code.s_value diff --git a/tool/lrama/lib/lrama/grammar_validator.rb b/tool/lrama/lib/lrama/grammar_validator.rb deleted file mode 100644 index 7790499589..0000000000 --- a/tool/lrama/lib/lrama/grammar_validator.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class GrammarValidator - def initialize(grammar, states, logger) - @grammar = grammar - @states = states - @logger = logger - end - - def valid? - conflicts_within_threshold? - end - - private - - def conflicts_within_threshold? - return true unless @grammar.expect - - [sr_conflicts_within_threshold(@grammar.expect), rr_conflicts_within_threshold(0)].all? - end - - def sr_conflicts_within_threshold(expected) - return true if expected == @states.sr_conflicts_count - - @logger.error("shift/reduce conflicts: #{@states.sr_conflicts_count} found, #{expected} expected") - false - end - - def rr_conflicts_within_threshold(expected) - return true if expected == @states.rr_conflicts_count - - @logger.error("reduce/reduce conflicts: #{@states.rr_conflicts_count} found, #{expected} expected") - false - end - end -end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb index c50af82ae4..ce98b505a7 100644 --- a/tool/lrama/lib/lrama/lexer.rb +++ b/tool/lrama/lib/lrama/lexer.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "strscan" @@ -8,10 +9,26 @@ require_relative "lexer/token" module Lrama class Lexer - attr_reader :head_line, :head_column, :line - attr_accessor :status, :end_symbol - - SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] PERCENT_TOKENS = %w( %union %token @@ -42,8 +59,11 @@ module Lrama %no-stdlib %inline %locations - ).freeze + %categories + %start + ).freeze #: Array[String] + # @rbs (GrammarFile grammar_file) -> void def initialize(grammar_file) @grammar_file = grammar_file @scanner = StringScanner.new(grammar_file.text) @@ -53,6 +73,7 @@ module Lrama @end_symbol = nil end + # @rbs () -> token? def next_token case @status when :initial @@ -62,10 +83,12 @@ module Lrama end end + # @rbs () -> Integer def column @scanner.pos - @head end + # @rbs () -> Location def location Location.new( grammar_file: @grammar_file, @@ -74,13 +97,14 @@ module Lrama ) end + # @rbs () -> lexer_token? def lex_token until @scanner.eos? do case when @scanner.scan(/\n/) newline when @scanner.scan(/\s+/) - # noop + @scanner.matched.count("\n").times { newline } when @scanner.scan(/\/\*/) lex_comment when @scanner.scan(/\/\/.*(?<newline>\n)?/) @@ -96,11 +120,11 @@ module Lrama when @scanner.eos? return when @scanner.scan(/#{SYMBOLS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/[\?\+\*]/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/<\w+>/) return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/'.'/) @@ -108,9 +132,9 @@ module Lrama when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/".*?"/) - return [:STRING, %Q(#{@scanner.matched})] + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] when @scanner.scan(/\d+/) - return [:INTEGER, Integer(@scanner.matched)] + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) type = @@ -121,51 +145,53 @@ module Lrama end return [type, token] else - raise ParseError, "Unexpected token: #{@scanner.peek(10).chomp}." + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant end end + # @rbs () -> c_token def lex_c_code nested = 0 - code = '' + code = +'' reset_first_position until @scanner.eos? do case when @scanner.scan(/{/) - code += @scanner.matched + code << @scanner.matched nested += 1 when @scanner.scan(/}/) if nested == 0 && @end_symbol == '}' @scanner.unscan return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] else - code += @scanner.matched + code << @scanner.matched nested -= 1 end when @scanner.check(/#{@end_symbol}/) return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] when @scanner.scan(/\n/) - code += @scanner.matched + code << @scanner.matched newline when @scanner.scan(/".*?"/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) @line += @scanner.matched.count("\n") when @scanner.scan(/'.*?'/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) when @scanner.scan(/[^\"'\{\}\n]+/) - code += @scanner.matched - when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) - code += @scanner.matched + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched else - code += @scanner.getch + code << @scanner.getch end end - raise ParseError, "Unexpected code: #{code}." + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant end private + # @rbs () -> void def lex_comment until @scanner.eos? do case @@ -178,11 +204,13 @@ module Lrama end end + # @rbs () -> void def reset_first_position @head_line = line @head_column = column end + # @rbs () -> void def newline @line += 1 @head = @scanner.pos diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb index defdbf8a0b..4465576d53 100644 --- a/tool/lrama/lib/lrama/lexer/location.rb +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -69,15 +69,15 @@ module Lrama def generate_error_message(error_message) <<~ERROR.chomp #{path}:#{first_line}:#{first_column}: #{error_message} - #{line_with_carets} + #{error_with_carets} ERROR end # @rbs () -> String - def line_with_carets + def error_with_carets <<~TEXT - #{text} - #{carets} + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} TEXT end @@ -89,13 +89,30 @@ module Lrama end # @rbs () -> String - def blanks - (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ') + def carets_line + leading_whitespace + highlight_marker end # @rbs () -> String - def carets - blanks + '^' * (last_column - first_column) + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length end # @rbs () -> String diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb index 63da8be4a4..37f77aa069 100644 --- a/tool/lrama/lib/lrama/lexer/token.rb +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -1,70 +1,20 @@ # rbs_inline: enabled # frozen_string_literal: true +require_relative 'token/base' require_relative 'token/char' +require_relative 'token/empty' require_relative 'token/ident' require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' require_relative 'token/tag' +require_relative 'token/token' require_relative 'token/user_code' module Lrama class Lexer - class Token - attr_reader :s_value #: String - attr_reader :location #: Location - attr_accessor :alias_name #: String - attr_accessor :referred #: bool - - # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void - def initialize(s_value:, alias_name: nil, location: nil) - s_value.freeze - @s_value = s_value - @alias_name = alias_name - @location = location - end - - # @rbs () -> String - def to_s - "value: `#{s_value}`, location: #{location}" - end - - # @rbs (String string) -> bool - def referred_by?(string) - [self.s_value, self.alias_name].compact.include?(string) - end - - # @rbs (Token other) -> bool - def ==(other) - self.class == other.class && self.s_value == other.s_value - end - - # @rbs () -> Integer - def first_line - location.first_line - end - alias :line :first_line - - # @rbs () -> Integer - def first_column - location.first_column - end - alias :column :first_column - - # @rbs () -> Integer - def last_line - location.last_line - end - - # @rbs () -> Integer - def last_column - location.last_column - end - - # @rbs (Lrama::Grammar::Reference ref, String message) -> bot - def invalid_ref(ref, message) - location = self.location.partial_location(ref.first_column, ref.last_column) - raise location.generate_error_message(message) - end + module Token end end end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb index fcab7a588f..f4ef7c9fbc 100644 --- a/tool/lrama/lib/lrama/lexer/token/char.rb +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -3,8 +3,21 @@ module Lrama class Lexer - class Token - class Char < Token + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end end end end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb index 8b1328a040..4880be9073 100644 --- a/tool/lrama/lib/lrama/lexer/token/ident.rb +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -3,8 +3,8 @@ module Lrama class Lexer - class Token - class Ident < Token + module Token + class Ident < Base end end end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb index 37d412aa83..7051ba75a4 100644 --- a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -3,12 +3,12 @@ module Lrama class Lexer - class Token - class InstantiateRule < Token - attr_reader :args #: Array[Lexer::Token] + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] attr_reader :lhs_tag #: Lexer::Token::Tag? - # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token], ?lhs_tag: Lexer::Token::Tag?) -> void + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) super s_value: s_value, alias_name: alias_name, location: location @args = args diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb index b346ef7c5c..68c6268219 100644 --- a/tool/lrama/lib/lrama/lexer/token/tag.rb +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -3,8 +3,8 @@ module Lrama class Lexer - class Token - class Tag < Token + module Token + class Tag < Base # @rbs () -> String def member # Omit "<>" diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb index 4ef40e6dc8..166f04954a 100644 --- a/tool/lrama/lib/lrama/lexer/token/user_code.rb +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -5,8 +5,8 @@ require "strscan" module Lrama class Lexer - class Token - class UserCode < Token + module Token + class UserCode < Base attr_accessor :tag #: Lexer::Token::Tag # @rbs () -> Array[Lrama::Grammar::Reference] @@ -38,43 +38,69 @@ module Lrama # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? def scan_reference(scanner) start = scanner.pos - case - # $ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$ - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1 - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[2]), index: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) - - # @ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/@\$/) # @$ - return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) - when scanner.scan(/@(\d+)/) # @1 - return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[1]), index: Integer(scanner[1]), first_column: start, last_column: scanner.pos) - when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) - when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @[expr.right], @[expr-right] (named reference with brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) - # $: references - when scanner.scan(/\$:\$/) # $:$ - return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:(\d+)/) # $:1 - return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[1]), first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:([a-zA-Z_][a-zA-Z0-9_]*)/) # $:foo, $:expr (named reference without brackets) - return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $:[expr.right], $:[expr-right] (named reference with brackets) - return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end end end end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb index 88bb920960..291eea5296 100644 --- a/tool/lrama/lib/lrama/logger.rb +++ b/tool/lrama/lib/lrama/logger.rb @@ -8,14 +8,24 @@ module Lrama @out = out end + # @rbs () -> void + def line_break + @out << "\n" + end + # @rbs (String message) -> void - def warn(message) + def trace(message) @out << message << "\n" end # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void def error(message) - @out << message << "\n" + @out << 'error: ' << message << "\n" end end end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb index 23988a5fbb..5a15d59c7b 100644 --- a/tool/lrama/lib/lrama/option_parser.rb +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require 'optparse' @@ -5,17 +6,32 @@ require 'optparse' module Lrama # Handle option parsing for the command line interface. class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void def initialize @options = Options.new @trace = [] @report = [] + @profile = [] end + # @rbs (Array[String]) -> Lrama::Options def parse(argv) parse_by_option_parser(argv) @options.trace_opts = validate_trace(@trace) @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) @options.grammar_file = argv.shift unless @options.grammar_file @@ -46,6 +62,7 @@ module Lrama private + # @rbs (Array[String]) -> void def parse_by_option_parser(argv) ::OptionParser.new do |o| o.banner = <<~BANNER @@ -60,7 +77,14 @@ module Lrama o.separator 'Tuning the Parser:' o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } - o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") {|v| @options.define = v } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end o.separator '' o.separator 'Output:' o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } @@ -91,10 +115,19 @@ module Lrama o.on_tail ' time display generation time' o.on_tail ' all include all the above traces' o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } o.separator '' o.separator 'Diagnostics:' - o.on('-W', '--warnings', 'report the warnings') {|v| @options.diagnostic = true } + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } o.separator '' o.separator 'Error Recovery:' o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } @@ -107,9 +140,10 @@ module Lrama end end - ALIASED_REPORTS = { cex: :counterexamples }.freeze - VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_report(report) h = { grammar: true } return h if report.empty? @@ -131,6 +165,7 @@ module Lrama return h end + # @rbs (String) -> Symbol def aliased_report_option(opt) (ALIASED_REPORTS[opt.to_sym] || opt).to_sym end @@ -139,15 +174,16 @@ module Lrama locations scan parse automaton bitsets closure grammar rules only-explicit-rules actions resource sets muscles tools m4-early m4 skeleton time ielr cex - ].freeze + ].freeze #: Array[String] NOT_SUPPORTED_TRACES = %w[ locations scan parse bitsets grammar resource sets muscles tools m4-early m4 skeleton ielr cex - ].freeze - SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_trace(trace) - h = {} + h = {} #: Hash[Symbol, bool] return h if trace.empty? || trace == ['none'] all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] if trace == ['all'] @@ -159,7 +195,25 @@ module Lrama if SUPPORTED_TRACES.include?(t) h[t.gsub(/-/, '_').to_sym] = true else - raise "Invalid trace option \"#{t}\"." + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." end end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb index 08f75a770f..87aec62448 100644 --- a/tool/lrama/lib/lrama/options.rb +++ b/tool/lrama/lib/lrama/options.rb @@ -1,28 +1,46 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama # Command line options. class Options - attr_accessor :skeleton, :header, :header_file, - :report_file, :outfile, - :error_recovery, :grammar_file, - :trace_opts, :report_opts, - :diagnostic, :y, :debug, :define + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + # @rbs () -> void def initialize @skeleton = "bison/yacc.c" + @locations = false @define = {} @header = false @header_file = nil @report_file = nil @outfile = "y.tab.c" @error_recovery = false - @grammar_file = nil + @grammar_file = '' @trace_opts = nil @report_opts = nil - @diagnostic = false + @warnings = false @y = STDIN @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil end end end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb index 3c7316ac6d..24cf725c77 100644 --- a/tool/lrama/lib/lrama/output.rb +++ b/tool/lrama/lib/lrama/output.rb @@ -1,13 +1,12 @@ # frozen_string_literal: true -require "erb" require "forwardable" -require_relative "report/duration" +require_relative "tracer/duration" module Lrama class Output extend Forwardable - include Report::Duration + include Tracer::Duration attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header @@ -43,7 +42,7 @@ module Lrama end def render_partial(file) - render_template(partial_file(file)) + ERB.render(partial_file(file), context: @context, output: self) end def render @@ -247,7 +246,7 @@ module Lrama <<-STR case #{rule.id + 1}: /* #{rule.as_comment} */ #line #{code.line} "#{@grammar_file_path}" -#{spaces}{#{rule.translated_code}} +#{spaces}{#{rule.translated_code(@grammar)}} #line [@oline@] [@ofile@] break; @@ -405,16 +404,10 @@ module Lrama private def eval_template(file, path) - tmp = render_template(file) + tmp = ERB.render(file, context: @context, output: self) replace_special_variables(tmp, path) end - def render_template(file) - erb = self.class.erb(File.read(file)) - erb.filename = file - erb.result_with_hash(context: @context, output: self) - end - def template_file File.join(template_dir, @template_name) end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index 177e784e5c..04632cbae0 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # # DO NOT MODIFY!!!! # This file is automatically generated by Racc 1.8.1 @@ -654,22 +655,25 @@ end module Lrama class Parser < Racc::Parser -module_eval(<<'...end parser.y/module_eval...', 'parser.y', 428) +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 505) -include Lrama::Report::Duration +include Lrama::Tracer::Duration -def initialize(text, path, debug = false, define = {}) +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) - @yydebug = debug + @yydebug = debug || define.key?('parse.trace') @rule_counter = Lrama::Grammar::Counter.new(0) @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations @define = define end def parse - report_duration(:parse) do + message = "parse '#{File.basename(@path)}'" + report_duration(message) do @lexer = Lrama::Lexer.new(@grammar_file) - @grammar = Lrama::Grammar.new(@rule_counter, @define) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) @precedence_number = 0 reset_precs do_parse @@ -682,7 +686,14 @@ def next_token end def on_error(error_token_id, error_value, value_stack) - if error_value.is_a?(Lrama::Lexer::Token) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base location = error_value.location value = "'#{error_value.s_value}'" else @@ -696,7 +707,7 @@ def on_error(error_token_id, error_value, value_stack) end def on_action_error(error_message, error_value) - if error_value.is_a?(Lrama::Lexer::Token) + if error_value.is_a?(Lrama::Lexer::Token::Base) location = error_value.location else location = @lexer.location @@ -708,10 +719,15 @@ end private def reset_precs - @prec_seen = false + @opening_prec_seen = false + @trailing_prec_seen = false @code_after_prec = false end +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + def begin_c_declaration(end_symbol) @lexer.status = :c_declaration @lexer.end_symbol = end_symbol @@ -729,306 +745,325 @@ end ##### State transition tables begin ### racc_action_table = [ - 89, 49, 90, 167, 49, 101, 173, 49, 101, 167, - 49, 101, 173, 6, 101, 80, 49, 49, 48, 48, - 41, 76, 76, 49, 49, 48, 48, 42, 76, 76, - 49, 49, 48, 48, 101, 96, 113, 49, 87, 48, - 150, 101, 96, 151, 45, 171, 169, 170, 151, 176, - 170, 91, 169, 170, 81, 176, 170, 20, 24, 25, - 26, 27, 28, 29, 30, 31, 87, 32, 33, 34, - 35, 36, 37, 38, 39, 49, 4, 48, 5, 101, - 96, 181, 182, 183, 128, 20, 24, 25, 26, 27, - 28, 29, 30, 31, 46, 32, 33, 34, 35, 36, - 37, 38, 39, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 53, 20, 24, 25, 26, 27, 28, 29, - 30, 31, 53, 32, 33, 34, 35, 36, 37, 38, - 39, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 44, 20, 24, 25, 26, 27, 28, 29, 30, 31, - 53, 32, 33, 34, 35, 36, 37, 38, 39, 49, - 4, 48, 5, 101, 96, 49, 49, 48, 48, 101, - 101, 49, 49, 48, 48, 101, 101, 49, 49, 48, - 197, 101, 101, 49, 49, 197, 48, 101, 101, 49, - 49, 197, 48, 101, 181, 182, 183, 128, 204, 210, - 217, 205, 205, 205, 49, 49, 48, 48, 49, 49, - 48, 48, 49, 49, 48, 48, 181, 182, 183, 116, - 117, 56, 53, 53, 53, 53, 53, 62, 63, 64, - 65, 66, 68, 68, 68, 82, 53, 53, 104, 108, - 108, 115, 122, 123, 125, 128, 129, 133, 139, 140, - 141, 142, 144, 145, 101, 154, 139, 157, 154, 161, - 162, 68, 164, 165, 172, 177, 154, 184, 128, 188, - 154, 190, 128, 154, 199, 154, 128, 68, 165, 206, - 165, 68, 68, 215, 128, 68 ] + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 225, 110, 110, 53, + 53, 209, 209, 110, 110, 193, 194, 195, 137, 216, + 222, 232, 217, 217, 217, 235, 57, 53, 217, 52, + 53, 53, 52, 52, 193, 194, 195, 57, 57, 57, + 66, 67, 68, 69, 70, 72, 72, 72, 86, 89, + 47, 57, 57, 113, 117, 117, 79, 123, 124, 131, + 47, 133, 137, 139, 143, 149, 150, 151, 152, 133, + 155, 156, 157, 110, 166, 149, 169, 172, 173, 72, + 175, 176, 183, 189, 166, 196, 137, 200, 202, 137, + 166, 211, 166, 137, 72, 176, 218, 176, 72, 137, + 228, 137, 72, 231, 72 ] racc_action_check = [ - 47, 153, 47, 153, 159, 153, 159, 178, 159, 178, - 189, 178, 189, 1, 189, 39, 35, 36, 35, 36, - 5, 35, 36, 37, 38, 37, 38, 6, 37, 38, - 59, 74, 59, 74, 59, 59, 74, 60, 45, 60, - 138, 60, 60, 138, 9, 156, 153, 153, 156, 159, - 159, 47, 178, 178, 39, 189, 189, 45, 45, 45, - 45, 45, 45, 45, 45, 45, 83, 45, 45, 45, - 45, 45, 45, 45, 45, 61, 0, 61, 0, 61, - 61, 166, 166, 166, 166, 83, 83, 83, 83, 83, - 83, 83, 83, 83, 11, 83, 83, 83, 83, 83, - 83, 83, 83, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 13, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 14, 3, 3, 3, 3, 3, 3, 3, - 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 15, 8, 8, 8, 8, 8, 8, 8, 8, 97, - 2, 97, 2, 97, 97, 71, 108, 71, 108, 71, - 108, 109, 169, 109, 169, 109, 169, 176, 184, 176, - 184, 176, 184, 190, 205, 190, 205, 190, 205, 206, - 12, 206, 12, 206, 174, 174, 174, 174, 196, 201, - 214, 196, 201, 214, 69, 76, 69, 76, 104, 105, - 104, 105, 111, 113, 111, 113, 198, 198, 198, 81, - 81, 16, 17, 20, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 40, 51, 56, 67, 70, - 72, 80, 84, 85, 86, 87, 93, 107, 115, 116, - 117, 118, 127, 128, 134, 140, 141, 143, 144, 145, - 146, 150, 151, 152, 158, 163, 165, 167, 168, 171, - 172, 173, 175, 177, 187, 188, 192, 193, 195, 197, - 200, 202, 204, 209, 210, 216 ] + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 231, 218, 231, 218, 231, 186, 186, 186, 186, 208, + 213, 227, 208, 213, 227, 234, 24, 113, 234, 113, + 114, 123, 114, 123, 210, 210, 210, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 40, 42, + 47, 55, 60, 71, 74, 76, 80, 81, 87, 91, + 92, 93, 94, 102, 116, 124, 125, 126, 127, 133, + 136, 137, 138, 144, 150, 151, 153, 156, 158, 162, + 163, 164, 170, 174, 176, 178, 179, 182, 184, 187, + 189, 199, 200, 204, 205, 207, 209, 212, 214, 216, + 221, 222, 224, 225, 229 ] racc_action_pointer = [ - 66, 13, 150, 90, nil, 13, 27, nil, 118, 35, - nil, 88, 187, 63, 73, 101, 216, 173, nil, nil, - 174, nil, nil, nil, 175, 176, 177, 222, 223, 224, - 225, 226, 224, 225, 226, 13, 14, 20, 21, 10, - 233, nil, nil, nil, nil, 34, nil, -5, nil, nil, - nil, 187, nil, nil, nil, nil, 188, nil, nil, 27, - 34, 72, nil, nil, nil, nil, nil, 230, nil, 201, - 231, 162, 232, nil, 28, nil, 202, nil, nil, nil, - 200, 215, nil, 62, 233, 221, 222, 191, nil, nil, - nil, nil, nil, 244, nil, nil, nil, 156, nil, nil, - nil, nil, nil, nil, 205, 206, nil, 241, 163, 168, - nil, 209, nil, 210, nil, 243, 206, 209, 240, nil, - nil, nil, nil, nil, nil, nil, nil, 209, 248, nil, - nil, nil, nil, nil, 247, nil, nil, nil, -2, nil, - 208, 251, nil, 255, 211, 204, 210, nil, nil, nil, - 253, 257, 217, -2, nil, nil, 3, nil, 218, 1, - nil, nil, nil, 222, nil, 219, 30, 226, 214, 169, - nil, 226, 223, 230, 143, 218, 174, 226, 4, nil, - nil, nil, nil, nil, 175, nil, nil, 272, 228, 7, - 180, nil, 222, 269, nil, 232, 156, 238, 165, nil, - 234, 157, 273, nil, 274, 181, 186, nil, nil, 233, - 230, nil, nil, nil, 158, nil, 277, nil, nil ] + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 210, 221, 222, 223, 235, 236, + 237, 238, 239, 237, 238, 239, 24, 25, 31, 32, + 243, -1, 247, nil, nil, nil, 43, 237, nil, nil, + nil, -5, nil, nil, nil, 235, nil, nil, nil, nil, + 236, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 245, nil, 171, 246, 2, 247, nil, 177, 183, + 248, 249, nil, nil, nil, nil, nil, 214, 45, nil, + 63, 250, 247, 248, 207, nil, nil, -4, nil, nil, + nil, nil, 261, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 224, 227, nil, 258, 38, 188, nil, + nil, nil, nil, 228, 260, 220, 223, 257, nil, nil, + nil, nil, nil, 256, nil, nil, 224, 266, 255, nil, + nil, nil, nil, nil, 266, nil, nil, nil, -24, nil, + 224, 270, nil, 274, nil, nil, 221, nil, 261, nil, + nil, nil, 271, 275, 232, 3, nil, nil, 3, nil, + 233, 9, nil, nil, 237, nil, 234, 3, 241, 231, + 189, nil, 241, nil, 244, nil, 163, 234, 194, 240, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 289, + 242, 15, 200, nil, 238, 286, nil, 246, 174, 252, + 182, nil, 248, 175, 290, nil, 244, 201, 206, nil, + nil, 283, 246, nil, 294, 259, nil, 176, nil, 296, + nil, 207, nil, nil, 180, nil ] racc_action_default = [ - -1, -128, -1, -3, -10, -128, -128, -2, -3, -128, - -16, -128, -128, -128, -128, -128, -128, -128, -24, -25, - -128, -32, -33, -34, -128, -128, -128, -128, -128, -128, - -128, -128, -50, -50, -50, -128, -128, -128, -128, -128, - -128, -13, 219, -4, -26, -128, -17, -123, -93, -94, - -122, -14, -19, -85, -20, -21, -128, -23, -31, -128, - -128, -128, -38, -39, -40, -41, -42, -43, -51, -128, - -44, -128, -45, -46, -88, -90, -128, -47, -48, -49, - -128, -128, -11, -5, -7, -95, -128, -68, -18, -124, - -125, -126, -15, -128, -22, -27, -28, -29, -35, -83, - -84, -127, -36, -37, -128, -52, -54, -56, -128, -79, - -81, -88, -89, -128, -91, -128, -128, -128, -128, -6, - -8, -9, -120, -96, -97, -98, -69, -128, -128, -86, - -30, -55, -53, -57, -76, -82, -80, -92, -128, -62, - -66, -128, -12, -128, -66, -128, -128, -58, -77, -78, - -50, -128, -60, -64, -67, -70, -128, -121, -99, -100, - -102, -119, -87, -128, -63, -66, -68, -93, -68, -128, - -116, -128, -66, -93, -68, -68, -128, -66, -65, -71, - -72, -108, -109, -110, -128, -74, -75, -128, -66, -101, - -128, -103, -68, -50, -107, -59, -128, -93, -111, -117, - -61, -128, -50, -106, -50, -128, -128, -112, -113, -128, - -68, -104, -73, -114, -128, -118, -50, -115, -105 ] + -1, -137, -1, -3, -10, -137, -137, -2, -3, -137, + -14, -14, -137, -137, -137, -137, -137, -137, -137, -28, + -29, -34, -35, -36, -137, -137, -137, -137, -137, -137, + -137, -137, -137, -54, -54, -54, -137, -137, -137, -137, + -137, -137, -137, -13, 236, -4, -137, -14, -16, -17, + -20, -132, -100, -101, -131, -18, -23, -89, -24, -25, + -137, -27, -37, -137, -137, -137, -41, -42, -43, -44, + -45, -46, -55, -137, -47, -137, -48, -49, -92, -137, + -95, -97, -98, -50, -51, -52, -53, -137, -137, -11, + -5, -7, -14, -137, -72, -15, -21, -132, -133, -134, + -135, -19, -137, -26, -30, -31, -32, -38, -87, -88, + -136, -39, -40, -137, -56, -58, -60, -137, -83, -85, + -93, -94, -96, -137, -137, -137, -137, -137, -6, -8, + -9, -129, -104, -102, -105, -73, -137, -137, -137, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -137, -66, + -70, -137, -12, -137, -103, -109, -137, -22, -137, -62, + -81, -82, -54, -137, -64, -68, -71, -74, -137, -130, + -106, -107, -128, -91, -137, -67, -70, -72, -100, -72, + -137, -125, -137, -109, -100, -110, -72, -72, -137, -70, + -69, -75, -76, -116, -117, -118, -137, -78, -79, -137, + -70, -108, -137, -111, -72, -54, -115, -63, -137, -100, + -119, -126, -65, -137, -54, -114, -72, -137, -137, -120, + -121, -137, -72, -112, -54, -100, -122, -137, -127, -54, + -77, -137, -124, -113, -137, -123 ] racc_goto_table = [ - 69, 109, 50, 152, 57, 127, 84, 58, 112, 160, - 114, 59, 60, 61, 86, 52, 54, 55, 98, 102, - 103, 159, 106, 110, 175, 74, 74, 74, 74, 138, - 9, 1, 3, 180, 7, 43, 120, 160, 109, 109, - 195, 192, 121, 94, 119, 112, 40, 137, 118, 189, - 47, 200, 86, 92, 175, 156, 130, 131, 132, 107, - 135, 136, 88, 196, 111, 207, 111, 70, 72, 201, - 73, 77, 78, 79, 67, 147, 134, 178, 148, 149, - 93, 146, 124, 166, 179, 214, 185, 158, 208, 174, - 187, 209, 191, 193, 107, 107, 143, nil, nil, 186, - nil, 111, nil, 111, nil, nil, 194, nil, 166, nil, - 202, nil, nil, nil, 198, nil, nil, nil, 163, 174, - 198, nil, nil, nil, nil, nil, nil, nil, 216, nil, - nil, nil, nil, nil, nil, 213, 198, nil, nil, nil, + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 208, 107, 111, 112, 119, 134, 213, + 56, 58, 59, 171, 61, 1, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 227, 129, 192, 148, 74, + 76, 95, 187, 118, 118, 207, 204, 3, 234, 7, + 130, 201, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 9, 116, 42, 168, 103, 45, 78, 78, + 219, 127, 51, 71, 141, 142, 77, 83, 84, 85, + 159, 144, 190, 160, 161, 191, 132, 197, 102, 158, + 122, 177, 170, 220, 203, 205, 199, 186, 221, 153, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, 224, nil, nil, 186, 210, 174, + 229, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 226, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 210, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 203, nil, nil, nil, nil, nil, nil, nil, nil, - 211, nil, 212, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, 218 ] + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, nil, nil, nil, nil, nil, nil, nil, + nil, 230, nil, nil, nil, nil, 233 ] racc_goto_check = [ - 27, 20, 29, 33, 15, 40, 8, 15, 46, 39, - 46, 15, 15, 15, 12, 16, 16, 16, 22, 22, - 22, 50, 28, 43, 38, 29, 29, 29, 29, 32, - 7, 1, 6, 36, 6, 7, 5, 39, 20, 20, - 33, 36, 9, 15, 8, 46, 10, 46, 11, 50, - 13, 33, 12, 16, 38, 32, 22, 28, 28, 29, - 43, 43, 14, 37, 29, 36, 29, 24, 24, 37, - 25, 25, 25, 25, 23, 30, 31, 34, 41, 42, - 44, 45, 48, 20, 40, 37, 40, 49, 51, 20, - 52, 53, 40, 40, 29, 29, 54, nil, nil, 20, - nil, 29, nil, 29, nil, nil, 20, nil, 20, nil, - 40, nil, nil, nil, 20, nil, nil, nil, 27, 20, - 20, nil, nil, nil, nil, nil, nil, nil, 40, nil, - nil, nil, nil, nil, nil, 20, 20, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 39, 24, 24, 24, 45, 52, 39, + 18, 18, 18, 54, 17, 1, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 39, 5, 38, 34, 26, + 26, 14, 40, 22, 22, 35, 38, 6, 39, 6, + 9, 54, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 7, 31, 10, 34, 17, 7, 31, 31, + 38, 11, 15, 25, 30, 30, 27, 27, 27, 27, + 32, 33, 36, 43, 44, 42, 14, 42, 46, 47, + 50, 22, 53, 55, 42, 42, 56, 22, 57, 58, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, 42, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 22, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 27, nil, nil, nil, nil, nil, nil, nil, nil, - 27, nil, 27, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, 27 ] + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, nil, nil, nil, 29 ] racc_goto_pointer = [ - nil, 31, nil, nil, nil, -48, 32, 27, -39, -42, - 42, -34, -31, 38, 15, -13, 2, nil, nil, nil, - -70, nil, -41, 42, 34, 35, nil, -32, -47, -10, - -59, -31, -86, -137, -88, nil, -133, -121, -135, -135, - -82, -56, -55, -48, 27, -48, -66, nil, -3, -57, - -123, -110, -80, -108, -26 ] + nil, 25, nil, nil, nil, -55, 47, 59, -38, -41, + 60, -18, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 40, 5, 40, nil, -33, + -39, -10, -64, -35, -86, -144, -94, nil, -140, -183, + -159, nil, -92, -61, -60, -58, 31, -50, -69, nil, + 10, nil, -75, -63, -132, -117, -85, -113, -32 ] racc_goto_default = [ - nil, nil, 2, 8, 83, nil, nil, nil, nil, nil, - nil, nil, 10, nil, nil, 51, nil, 21, 22, 23, - 95, 97, nil, nil, nil, nil, 105, 71, nil, 99, - nil, nil, nil, nil, 153, 126, nil, nil, 168, 155, - nil, 100, nil, nil, nil, nil, 75, 85, nil, nil, - nil, nil, nil, nil, nil ] + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] racc_reduce_table = [ 0, 0, :racc_error, - 0, 63, :_reduce_1, - 2, 63, :_reduce_2, - 0, 64, :_reduce_3, - 2, 64, :_reduce_4, - 1, 65, :_reduce_5, - 2, 65, :_reduce_6, - 0, 66, :_reduce_none, - 1, 66, :_reduce_none, - 5, 58, :_reduce_none, - 0, 67, :_reduce_10, - 0, 68, :_reduce_11, - 5, 59, :_reduce_12, - 2, 59, :_reduce_none, - 1, 73, :_reduce_14, - 2, 73, :_reduce_15, - 1, 60, :_reduce_none, - 2, 60, :_reduce_17, - 3, 60, :_reduce_18, - 2, 60, :_reduce_none, - 2, 60, :_reduce_20, - 2, 60, :_reduce_21, - 3, 60, :_reduce_22, - 2, 60, :_reduce_23, - 1, 60, :_reduce_24, - 1, 60, :_reduce_25, - 2, 60, :_reduce_none, - 1, 78, :_reduce_27, - 1, 78, :_reduce_28, - 1, 79, :_reduce_29, - 2, 79, :_reduce_30, - 2, 69, :_reduce_31, - 1, 69, :_reduce_none, - 1, 69, :_reduce_none, - 1, 69, :_reduce_none, - 3, 69, :_reduce_35, - 3, 69, :_reduce_36, - 3, 69, :_reduce_37, - 2, 69, :_reduce_38, - 2, 69, :_reduce_39, - 2, 69, :_reduce_40, - 2, 69, :_reduce_41, - 2, 69, :_reduce_42, - 2, 74, :_reduce_none, - 2, 74, :_reduce_44, - 2, 74, :_reduce_45, - 2, 74, :_reduce_46, - 2, 74, :_reduce_47, - 2, 74, :_reduce_48, - 2, 74, :_reduce_49, - 0, 84, :_reduce_none, - 1, 84, :_reduce_none, - 1, 85, :_reduce_52, - 2, 85, :_reduce_53, - 2, 80, :_reduce_54, - 3, 80, :_reduce_55, - 0, 88, :_reduce_none, - 1, 88, :_reduce_none, - 3, 83, :_reduce_58, - 8, 75, :_reduce_59, - 5, 76, :_reduce_60, - 8, 76, :_reduce_61, - 1, 89, :_reduce_62, - 3, 89, :_reduce_63, - 1, 90, :_reduce_64, - 3, 90, :_reduce_65, - 0, 96, :_reduce_none, - 1, 96, :_reduce_none, - 0, 97, :_reduce_none, - 1, 97, :_reduce_none, - 1, 91, :_reduce_70, - 3, 91, :_reduce_71, - 3, 91, :_reduce_72, - 6, 91, :_reduce_73, - 3, 91, :_reduce_74, - 3, 91, :_reduce_75, - 0, 99, :_reduce_none, - 1, 99, :_reduce_none, - 1, 87, :_reduce_78, - 1, 100, :_reduce_79, - 2, 100, :_reduce_80, - 2, 81, :_reduce_81, - 3, 81, :_reduce_82, - 1, 77, :_reduce_none, - 1, 77, :_reduce_none, - 0, 101, :_reduce_85, - 0, 102, :_reduce_86, - 5, 72, :_reduce_87, - 1, 103, :_reduce_88, - 2, 103, :_reduce_89, - 1, 82, :_reduce_90, - 2, 82, :_reduce_91, - 3, 82, :_reduce_92, - 1, 86, :_reduce_93, - 1, 86, :_reduce_94, - 0, 105, :_reduce_none, - 1, 105, :_reduce_none, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, 2, 61, :_reduce_none, 2, 61, :_reduce_none, - 4, 104, :_reduce_99, - 1, 106, :_reduce_100, - 3, 106, :_reduce_101, - 1, 107, :_reduce_102, - 3, 107, :_reduce_103, - 5, 107, :_reduce_104, - 7, 107, :_reduce_105, - 4, 107, :_reduce_106, - 3, 107, :_reduce_107, - 1, 93, :_reduce_108, - 1, 93, :_reduce_109, - 1, 93, :_reduce_110, - 0, 108, :_reduce_none, - 1, 108, :_reduce_none, - 2, 94, :_reduce_113, - 3, 94, :_reduce_114, - 4, 94, :_reduce_115, - 0, 109, :_reduce_116, - 0, 110, :_reduce_117, - 5, 95, :_reduce_118, - 3, 92, :_reduce_119, - 0, 111, :_reduce_120, - 3, 62, :_reduce_121, - 1, 70, :_reduce_none, - 0, 71, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, 1, 71, :_reduce_none, 1, 71, :_reduce_none, 1, 71, :_reduce_none, - 1, 98, :_reduce_127 ] - -racc_reduce_n = 128 - -racc_shift_n = 219 + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 7, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 6, 97, :_reduce_123, + 4, 97, :_reduce_124, + 0, 114, :_reduce_125, + 0, 115, :_reduce_126, + 5, 98, :_reduce_127, + 3, 95, :_reduce_128, + 0, 116, :_reduce_129, + 3, 63, :_reduce_130, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_136 ] + +racc_reduce_n = 137 + +racc_shift_n = 236 racc_token_table = { false => 0, @@ -1044,52 +1079,53 @@ racc_token_table = { "%{" => 10, "%}" => 11, "%require" => 12, - "%expect" => 13, - "%define" => 14, - "%param" => 15, - "%lex-param" => 16, - "%parse-param" => 17, - "%code" => 18, - "%initial-action" => 19, - "%no-stdlib" => 20, - "%locations" => 21, - ";" => 22, - "%union" => 23, - "%destructor" => 24, - "%printer" => 25, - "%error-token" => 26, - "%after-shift" => 27, - "%before-reduce" => 28, - "%after-reduce" => 29, - "%after-shift-error-token" => 30, - "%after-pop-stack" => 31, - "-temp-group" => 32, - "%token" => 33, - "%type" => 34, - "%nterm" => 35, - "%left" => 36, - "%right" => 37, - "%precedence" => 38, - "%nonassoc" => 39, - "%rule" => 40, - "(" => 41, - ")" => 42, - ":" => 43, - "%inline" => 44, - "," => 45, - "|" => 46, - "%empty" => 47, - "%prec" => 48, - "{" => 49, - "}" => 50, - "?" => 51, - "+" => 52, - "*" => 53, - "[" => 54, - "]" => 55, - "{...}" => 56 } - -racc_nt_base = 57 + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 racc_use_result_var = true @@ -1124,8 +1160,11 @@ Racc_token_to_s_table = [ "\"%{\"", "\"%}\"", "\"%require\"", + "\";\"", "\"%expect\"", "\"%define\"", + "\"{\"", + "\"}\"", "\"%param\"", "\"%lex-param\"", "\"%parse-param\"", @@ -1133,7 +1172,6 @@ Racc_token_to_s_table = [ "\"%initial-action\"", "\"%no-stdlib\"", "\"%locations\"", - "\";\"", "\"%union\"", "\"%destructor\"", "\"%printer\"", @@ -1151,6 +1189,7 @@ Racc_token_to_s_table = [ "\"%right\"", "\"%precedence\"", "\"%nonassoc\"", + "\"%start\"", "\"%rule\"", "\"(\"", "\")\"", @@ -1160,8 +1199,6 @@ Racc_token_to_s_table = [ "\"|\"", "\"%empty\"", "\"%prec\"", - "\"{\"", - "\"}\"", "\"?\"", "\"+\"", "\"*\"", @@ -1180,7 +1217,9 @@ Racc_token_to_s_table = [ "\"-option@epilogue_declaration\"", "@1", "@2", + "parser_option", "grammar_declaration", + "\"-many@;\"", "variable", "value", "param", @@ -1204,9 +1243,9 @@ Racc_token_to_s_table = [ "rule_rhs_list", "rule_rhs", "named_ref", - "parameterizing_suffix", - "parameterizing_args", - "midrule_action", + "parameterized_suffix", + "parameterized_args", + "action", "\"-option@%empty\"", "\"-option@named_ref\"", "string_as_id", @@ -1215,11 +1254,13 @@ Racc_token_to_s_table = [ "@3", "@4", "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", "rules", - "\"-option@;\"", + "\"-many1@;\"", "rhs_list", "rhs", - "\"-option@parameterizing_suffix\"", + "\"-option@parameterized_suffix\"", "@5", "@6", "@7" ] @@ -1279,10 +1320,9 @@ module_eval(<<'.,.,', 'parser.y', 11) # reduce 9 omitted -module_eval(<<'.,.,', 'parser.y', 12) +module_eval(<<'.,.,', 'parser.y', 13) def _reduce_10(val, _values, result) - begin_c_declaration("%}") - @grammar.prologue_first_lineno = @lexer.line + begin_c_declaration("%}") result end @@ -1290,7 +1330,7 @@ module_eval(<<'.,.,', 'parser.y', 12) module_eval(<<'.,.,', 'parser.y', 17) def _reduce_11(val, _values, result) - end_c_declaration + end_c_declaration result end @@ -1298,22 +1338,29 @@ module_eval(<<'.,.,', 'parser.y', 17) module_eval(<<'.,.,', 'parser.y', 21) def _reduce_12(val, _values, result) - @grammar.prologue = val[2].s_value + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value result end .,., -# reduce 13 omitted +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true + + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 54) +module_eval(<<'.,.,', 'parser.y', 34) def _reduce_14(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 54) +module_eval(<<'.,.,', 'parser.y', 34) def _reduce_15(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result @@ -1322,150 +1369,140 @@ module_eval(<<'.,.,', 'parser.y', 54) # reduce 16 omitted -module_eval(<<'.,.,', 'parser.y', 26) - def _reduce_17(val, _values, result) - @grammar.expect = val[1] +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 27) - def _reduce_18(val, _values, result) - @grammar.define[val[1].s_value] = val[2]&.s_value +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -# reduce 19 omitted - -module_eval(<<'.,.,', 'parser.y', 31) +module_eval(<<'.,.,', 'parser.y', 36) def _reduce_20(val, _values, result) - val[1].each {|token| - @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value - } + @grammar.expect = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 37) +module_eval(<<'.,.,', 'parser.y', 40) def _reduce_21(val, _values, result) - val[1].each {|token| - @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value - } + @grammar.define[val[1].s_value] = val[2]&.s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 43) +module_eval(<<'.,.,', 'parser.y', 44) def _reduce_22(val, _values, result) - @grammar.add_percent_code(id: val[1], code: val[2]) + @grammar.define[val[1].s_value] = val[3]&.s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 47) - def _reduce_23(val, _values, result) - @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) - - result - end -.,., +# reduce 23 omitted module_eval(<<'.,.,', 'parser.y', 49) def _reduce_24(val, _values, result) - @grammar.no_stdlib = true + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + result end .,., -module_eval(<<'.,.,', 'parser.y', 50) +module_eval(<<'.,.,', 'parser.y', 55) def _reduce_25(val, _values, result) - @grammar.locations = true + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + result end .,., -# reduce 26 omitted +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) -module_eval(<<'.,.,', 'parser.y', 109) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) def _reduce_27(val, _values, result) - result = val + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 69) def _reduce_28(val, _values, result) - result = val + @grammar.no_stdlib = true + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 73) def _reduce_29(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + @grammar.locations = true + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_30(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 55) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_31(val, _values, result) - @grammar.set_union( - Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), - val[1].line - ) - + result = val result end .,., -# reduce 32 omitted - -# reduce 33 omitted - -# reduce 34 omitted - -module_eval(<<'.,.,', 'parser.y', 65) - def _reduce_35(val, _values, result) - @grammar.add_destructor( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) - +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 73) - def _reduce_36(val, _values, result) - @grammar.add_printer( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) - +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 81) +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) def _reduce_37(val, _values, result) - @grammar.add_error_token( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) result end @@ -1473,665 +1510,769 @@ module_eval(<<'.,.,', 'parser.y', 81) module_eval(<<'.,.,', 'parser.y', 89) def _reduce_38(val, _values, result) - @grammar.after_shift = val[1] + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 93) +module_eval(<<'.,.,', 'parser.y', 97) def _reduce_39(val, _values, result) - @grammar.before_reduce = val[1] + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 97) +module_eval(<<'.,.,', 'parser.y', 105) def _reduce_40(val, _values, result) - @grammar.after_reduce = val[1] + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 101) +module_eval(<<'.,.,', 'parser.y', 113) def _reduce_41(val, _values, result) - @grammar.after_shift_error_token = val[1] + @grammar.after_shift = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 105) +module_eval(<<'.,.,', 'parser.y', 117) def _reduce_42(val, _values, result) - @grammar.after_pop_stack = val[1] + @grammar.before_reduce = val[1] result end .,., -# reduce 43 omitted - -module_eval(<<'.,.,', 'parser.y', 111) - def _reduce_44(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - @grammar.add_type(id: id, tag: hash[:tag]) - } - } +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 119) - def _reduce_45(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - if @grammar.find_term_by_s_value(id.s_value) - on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) - else - @grammar.add_type(id: id, tag: hash[:tag]) - end - } - } +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 131) - def _reduce_46(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_left(sym, @precedence_number) - } - } - @precedence_number += 1 +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 141) +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) def _reduce_47(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_right(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } result end .,., -module_eval(<<'.,.,', 'parser.y', 151) +module_eval(<<'.,.,', 'parser.y', 144) def _reduce_48(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_precedence(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } result end .,., -module_eval(<<'.,.,', 'parser.y', 161) +module_eval(<<'.,.,', 'parser.y', 156) def _reduce_49(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_nonassoc(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 result end .,., -# reduce 50 omitted +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 -# reduce 51 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 184) +module_eval(<<'.,.,', 'parser.y', 186) def _reduce_52(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + result end .,., -module_eval(<<'.,.,', 'parser.y', 184) +module_eval(<<'.,.,', 'parser.y', 196) def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 172) - def _reduce_54(val, _values, result) - val[1].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[0], replace: true) - } +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } result end .,., -module_eval(<<'.,.,', 'parser.y', 178) - def _reduce_55(val, _values, result) - val[2].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[1], replace: true) - } +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } result end .,., -# reduce 56 omitted +# reduce 60 omitted -# reduce 57 omitted +# reduce 61 omitted -module_eval(<<'.,.,', 'parser.y', 183) - def _reduce_58(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 187) - def _reduce_59(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 193) - def _reduce_60(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[2].s_value, [], val[4], is_inline: true) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 198) - def _reduce_61(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 202) - def _reduce_62(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) result = [val[0]] result end .,., -module_eval(<<'.,.,', 'parser.y', 203) - def _reduce_63(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 207) - def _reduce_64(val, _values, result) - builder = val[0] - result = [builder] +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 212) - def _reduce_65(val, _values, result) - builder = val[2] - result = val[0].append(builder) +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) result end .,., -# reduce 66 omitted +# reduce 70 omitted -# reduce 67 omitted +# reduce 71 omitted -# reduce 68 omitted +# reduce 72 omitted -# reduce 69 omitted +# reduce 73 omitted -module_eval(<<'.,.,', 'parser.y', 218) - def _reduce_70(val, _values, result) - reset_precs - result = Grammar::ParameterizingRule::Rhs.new +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new result end .,., -module_eval(<<'.,.,', 'parser.y', 223) - def _reduce_71(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.symbols << token - result = builder +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 231) - def _reduce_72(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) - result = builder +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 237) - def _reduce_73(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) - result = builder +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 243) - def _reduce_74(val, _values, result) - user_code = val[1] - user_code.alias_name = val[2] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 251) - def _reduce_75(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder result end .,., -# reduce 76 omitted +# reduce 80 omitted -# reduce 77 omitted +# reduce 81 omitted -module_eval(<<'.,.,', 'parser.y', 258) - def _reduce_78(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) result = val[0].s_value if val[0] result end .,., -module_eval(<<'.,.,', 'parser.y', 271) - def _reduce_79(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 271) - def _reduce_80(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 262) - def _reduce_81(val, _values, result) - result = if val[0] - [{tag: val[0], tokens: val[1]}] - else - [{tag: nil, tokens: val[1]}] - end +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end result end .,., -module_eval(<<'.,.,', 'parser.y', 268) - def _reduce_82(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) result = val[0].append({tag: val[1], tokens: val[2]}) result end .,., -# reduce 83 omitted +# reduce 87 omitted -# reduce 84 omitted +# reduce 88 omitted -module_eval(<<'.,.,', 'parser.y', 274) - def _reduce_85(val, _values, result) - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") result end .,., -module_eval(<<'.,.,', 'parser.y', 278) - def _reduce_86(val, _values, result) - end_c_declaration +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration result end .,., -module_eval(<<'.,.,', 'parser.y', 282) - def _reduce_87(val, _values, result) - result = val[2] +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_88(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_89(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 285) - def _reduce_90(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) result = [{tag: nil, tokens: val[0]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 286) - def _reduce_91(val, _values, result) - result = [{tag: val[0], tokens: val[1]}] +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } result end .,., -module_eval(<<'.,.,', 'parser.y', 287) - def _reduce_92(val, _values, result) - result = val[0].append({tag: val[1], tokens: val[2]}) +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 289) - def _reduce_93(val, _values, result) - on_action_error("ident after %prec", val[0]) if @prec_seen +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_94(val, _values, result) - on_action_error("char after %prec", val[0]) if @prec_seen +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -# reduce 95 omitted +# reduce 104 omitted -# reduce 96 omitted +# reduce 105 omitted -# reduce 97 omitted +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end -# reduce 98 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 298) - def _reduce_99(val, _values, result) - lhs = val[0] - lhs.alias_name = val[1] - val[3].each do |builder| - builder.lhs = lhs - builder.complete_input - @grammar.add_rule_builder(builder) - end +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 309) - def _reduce_100(val, _values, result) - builder = val[0] - if !builder.line - builder.line = @lexer.line - 1 - end - result = [builder] +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) result end .,., -module_eval(<<'.,.,', 'parser.y', 317) - def _reduce_101(val, _values, result) - builder = val[2] - if !builder.line - builder.line = @lexer.line - 1 - end - result = val[0].append(builder) +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) result end .,., -module_eval(<<'.,.,', 'parser.y', 326) - def _reduce_102(val, _values, result) - reset_precs - result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 331) - def _reduce_103(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.add_rhs(token) - result = builder +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 339) - def _reduce_104(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 347) - def _reduce_105(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 355) - def _reduce_106(val, _values, result) - user_code = val[1] - user_code.alias_name = val[2] - user_code.tag = val[3] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 364) - def _reduce_107(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 371) - def _reduce_108(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) result = "option" result end .,., -module_eval(<<'.,.,', 'parser.y', 372) - def _reduce_109(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) result = "nonempty_list" result end .,., -module_eval(<<'.,.,', 'parser.y', 373) - def _reduce_110(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) result = "list" result end .,., -# reduce 111 omitted +# reduce 119 omitted -# reduce 112 omitted +# reduce 120 omitted -module_eval(<<'.,.,', 'parser.y', 377) - def _reduce_113(val, _values, result) - result = if val[1] - [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] - else - [val[0]] - end +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end result end .,., -module_eval(<<'.,.,', 'parser.y', 383) - def _reduce_114(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 384) - def _reduce_115(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) + result = val[0].append(Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2].s_value, location: @lexer.location, args: val[4])) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 459) + def _reduce_124(val, _values, result) result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] result end .,., -module_eval(<<'.,.,', 'parser.y', 388) - def _reduce_116(val, _values, result) - if @prec_seen - on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec - @code_after_prec = true - end - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 464) + def _reduce_125(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") result end .,., -module_eval(<<'.,.,', 'parser.y', 396) - def _reduce_117(val, _values, result) - end_c_declaration +module_eval(<<'.,.,', 'parser.y', 472) + def _reduce_126(val, _values, result) + end_c_declaration result end .,., -module_eval(<<'.,.,', 'parser.y', 400) - def _reduce_118(val, _values, result) - result = val[2] +module_eval(<<'.,.,', 'parser.y', 476) + def _reduce_127(val, _values, result) + result = val[2] result end .,., -module_eval(<<'.,.,', 'parser.y', 403) - def _reduce_119(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 479) + def _reduce_128(val, _values, result) result = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 407) - def _reduce_120(val, _values, result) - begin_c_declaration('\Z') - @grammar.epilogue_first_lineno = @lexer.line + 1 +module_eval(<<'.,.,', 'parser.y', 484) + def _reduce_129(val, _values, result) + begin_c_declaration('\Z') result end .,., -module_eval(<<'.,.,', 'parser.y', 412) - def _reduce_121(val, _values, result) - end_c_declaration - @grammar.epilogue = val[2].s_value +module_eval(<<'.,.,', 'parser.y', 488) + def _reduce_130(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value result end .,., -# reduce 122 omitted +# reduce 131 omitted -# reduce 123 omitted +# reduce 132 omitted -# reduce 124 omitted +# reduce 133 omitted -# reduce 125 omitted +# reduce 134 omitted -# reduce 126 omitted +# reduce 135 omitted -module_eval(<<'.,.,', 'parser.y', 423) - def _reduce_127(val, _values, result) - result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) +module_eval(<<'.,.,', 'parser.y', 500) + def _reduce_136(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) result end .,., diff --git a/tool/lrama/lib/lrama/report.rb b/tool/lrama/lib/lrama/report.rb deleted file mode 100644 index 890e5f1e8c..0000000000 --- a/tool/lrama/lib/lrama/report.rb +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true - -require_relative 'report/duration' -require_relative 'report/profile' diff --git a/tool/lrama/lib/lrama/report/duration.rb b/tool/lrama/lib/lrama/report/duration.rb deleted file mode 100644 index fe09a0d028..0000000000 --- a/tool/lrama/lib/lrama/report/duration.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Report - module Duration - def self.enable - @_report_duration_enabled = true - end - - def self.enabled? - !!@_report_duration_enabled - end - - def report_duration(method_name) - time1 = Time.now.to_f - result = yield - time2 = Time.now.to_f - - if Duration.enabled? - puts sprintf("%s %10.5f s", method_name, time2 - time1) - end - - return result - end - end - end -end diff --git a/tool/lrama/lib/lrama/report/profile.rb b/tool/lrama/lib/lrama/report/profile.rb deleted file mode 100644 index 10488cf913..0000000000 --- a/tool/lrama/lib/lrama/report/profile.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Report - module Profile - # See "Profiling Lrama" in README.md for how to use. - def self.report_profile - require "stackprof" - - StackProf.run(mode: :cpu, raw: true, out: 'tmp/stackprof-cpu-myapp.dump') do - yield - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb index 3008786ced..50912e094e 100644 --- a/tool/lrama/lib/lrama/state.rb +++ b/tool/lrama/lib/lrama/state.rb @@ -1,17 +1,62 @@ +# rbs_inline: enabled # frozen_string_literal: true -require_relative "state/reduce" +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" require_relative "state/reduce_reduce_conflict" require_relative "state/resolved_conflict" -require_relative "state/shift" require_relative "state/shift_reduce_conflict" module Lrama class State - attr_reader :id, :accessing_symbol, :kernels, :conflicts, :resolved_conflicts, - :default_reduction_rule, :closure, :items - attr_accessor :shifts, :reduces, :ielr_isocores, :lalr_isocore - + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void def initialize(id, accessing_symbol, kernels) @id = id @accessing_symbol = accessing_symbol @@ -28,48 +73,72 @@ module Lrama @ielr_isocores = [self] @internal_dependencies = {} @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id end + # @rbs (Array[Item] closure) -> void def closure=(closure) @closure = closure @items = @kernels + @closure end + # @rbs () -> Array[Action::Reduce] def non_default_reduces reduces.reject do |reduce| reduce.rule == @default_reduction_rule end end - def compute_shifts_reduces - _shifts = {} + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} reduces = [] items.each do |item| # TODO: Consider what should be pushed if item.end_of_rule? - reduces << Reduce.new(item) + reduces << Action::Reduce.new(item) else key = item.next_sym - _shifts[key] ||= [] - _shifts[key] << item.new_by_next_position + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] end end # It seems Bison 3.8.2 iterates transitions order by symbol number - shifts = _shifts.sort_by do |next_sym, new_items| + transitions = _transitions.sort_by do |next_sym, to_items| next_sym.number - end.map do |next_sym, new_items| - Shift.new(next_sym, new_items.flatten) end - self.shifts = shifts.freeze + + self._transitions = transitions.freeze self.reduces = reduces.freeze end + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void def set_items_to_state(items, next_state) @items_to_state[items] = next_state end + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void def set_look_ahead(rule, look_ahead) reduce = reduces.find do |r| r.rule == rule @@ -78,50 +147,78 @@ module Lrama reduce.look_ahead = look_ahead end - def nterm_transitions - @nterm_transitions ||= transitions.select {|shift, _| shift.next_sym.nterm? } + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } end - def term_transitions - @term_transitions ||= transitions.select {|shift, _| shift.next_sym.term? } + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } end + # @rbs () -> Array[transition] def transitions - @transitions ||= shifts.map {|shift| [shift, @items_to_state[shift.next_items]] } + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end end - def update_transition(shift, next_state) - set_items_to_state(shift.next_items, next_state) + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) next_state.append_predecessor(self) - clear_transitions_cache + update_transitions_caches(transition) end - def clear_transitions_cache + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition @nterm_transitions = nil @term_transitions = nil - @transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) end + # @rbs () -> Array[Action::Shift] def selected_term_transitions - term_transitions.reject do |shift, next_state| + term_transitions.reject do |shift| shift.not_selected end end # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State def transition(sym) result = nil if sym.term? - term_transitions.each do |shift, next_state| - term = shift.next_sym - result = next_state if term == sym - end + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state else - nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - result = next_state if nterm == sym - end + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state end raise "Can not transit by #{sym} #{self}" if result.nil? @@ -129,12 +226,14 @@ module Lrama result end + # @rbs (Item item) -> Action::Reduce def find_reduce_by_item!(item) reduces.find do |r| r.item == item end || (raise "reduce is not found. #{item}") end + # @rbs (Grammar::Rule default_reduction_rule) -> void def default_reduction_rule=(default_reduction_rule) @default_reduction_rule = default_reduction_rule @@ -145,200 +244,219 @@ module Lrama end end + # @rbs () -> bool def has_conflicts? !@conflicts.empty? end + # @rbs () -> Array[conflict] def sr_conflicts @conflicts.select do |conflict| conflict.type == :shift_reduce end end + # @rbs () -> Array[conflict] def rr_conflicts @conflicts.select do |conflict| conflict.type == :reduce_reduce end end + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set def propagate_lookaheads(next_state) - next_state.kernels.map {|item| + next_state.kernels.map {|next_kernel| lookahead_sets = - if item.position == 1 - goto_follow_set(item.lhs) - else - kernel = kernels.find {|k| k.predecessor_item_of?(item) } + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) end - [item, lookahead_sets & next_state.lookahead_set_filters[item]] + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] }.to_h end - def lookaheads_recomputed - !@item_lookahead_set.nil? - end - - def compatible_lookahead?(filtered_lookahead) + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) !lookaheads_recomputed || - @lalr_isocore.annotation_list.all? {|token, actions| - a = dominant_contribution(token, actions, item_lookahead_set) - b = dominant_contribution(token, actions, filtered_lookahead) + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) a.nil? || b.nil? || a == b } end + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set def lookahead_set_filters - kernels.map {|kernel| - [kernel, - @lalr_isocore.annotation_list.select {|token, actions| - token.term? && actions.any? {|action, contributions| - !contributions.nil? && contributions.key?(kernel) && contributions[kernel] - } - }.map {|token, _| token } - ] + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] }.to_h end - def dominant_contribution(token, actions, lookaheads) - a = actions.select {|action, contributions| - contributions.nil? || contributions.any? {|item, contributed| contributed && lookaheads[item].include?(token) } - }.map {|action, _| action } - return nil if a.empty? - a.reject {|action| - if action.is_a?(State::Shift) - action.not_selected - elsif action.is_a?(State::Reduce) - action.not_selected_symbols.include?(token) - end - } - end - + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] def inadequacy_list return @inadequacy_list if @inadequacy_list - shift_contributions = shifts.map {|shift| - [shift.next_sym, [shift]] - }.to_h - reduce_contributions = reduces.map {|reduce| - (reduce.look_ahead || []).map {|sym| - [sym, [reduce]] - }.to_h - }.reduce(Hash.new([])) {|hash, cont| - hash.merge(cont) {|_, a, b| a | b } - } + inadequacy_list = {} - list = shift_contributions.merge(reduce_contributions) {|_, a, b| a | b } - @inadequacy_list = list.select {|token, actions| token.term? && actions.size > 1 } - end - - def annotation_list - return @annotation_list if @annotation_list - - @annotation_list = annotate_manifestation - @annotation_list = @items_to_state.values.map {|next_state| next_state.annotate_predecessor(self) } - .reduce(@annotation_list) {|result, annotations| - result.merge(annotations) {|_, actions_a, actions_b| - if actions_a.nil? || actions_b.nil? - actions_a || actions_b - else - actions_a.merge(actions_b) {|_, contributions_a, contributions_b| - if contributions_a.nil? || contributions_b.nil? - next contributions_a || contributions_b - end - - contributions_a.merge(contributions_b) {|_, contributed_a, contributed_b| - contributed_a || contributed_b - } - } - end - } - } + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } end + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void def annotate_manifestation - inadequacy_list.transform_values {|actions| - actions.map {|action| - if action.is_a?(Shift) + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) [action, nil] - elsif action.is_a?(Reduce) - if action.rule.empty_rule? - [action, lhs_contributions(action.rule.lhs, inadequacy_list.key(actions))] - else - contributions = kernels.map {|kernel| [kernel, kernel.rule == action.rule && kernel.end_of_rule?] }.to_h - [action, contributions] - end + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] end }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) } end + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void def annotate_predecessor(predecessor) - annotation_list.transform_values {|actions| - token = annotation_list.key(actions) - actions.transform_values {|inadequacy| - next nil if inadequacy.nil? - lhs_adequacy = kernels.any? {|kernel| - inadequacy[kernel] && kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, token).nil? - } - if lhs_adequacy - next nil + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] else - predecessor.kernels.map {|pred_k| - [pred_k, kernels.any? {|k| - inadequacy[k] && ( - pred_k.predecessor_item_of?(k) && predecessor.item_lookahead_set[pred_k].include?(token) || - k.position == 1 && predecessor.lhs_contributions(k.lhs, token)[pred_k] - ) - }] + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] }.to_h + [action, cs] end - } - } + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) end - def lhs_contributions(sym, token) - shift, next_state = nterm_transitions.find {|sh, _| sh.next_sym == sym } - if always_follows(shift, next_state).include?(token) - nil - else - kernels.map {|kernel| [kernel, follow_kernel_items(shift, next_state, kernel) && item_lookahead_set[kernel].include?(token)] }.to_h - end + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } end - def follow_kernel_items(shift, next_state, kernel) - queue = [[self, shift, next_state]] - until queue.empty? - st, sh, next_st = queue.pop - return true if kernel.next_sym == sh.next_sym && kernel.symbols_after_transition.all?(&:nullable) - st.internal_dependencies(sh, next_st).each {|v| queue << v } + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list end - false + + @annotation_list += propagating_list end + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set def item_lookahead_set return @item_lookahead_set if @item_lookahead_set - kernels.map {|item| + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| value = - if item.lhs.accept_symbol? + if kernel.lhs.accept_symbol? [] - elsif item.position > 1 - prev_items = predecessors_with_item(item) + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } - elsif item.position == 1 - prev_state = @predecessors.find {|p| p.shifts.any? {|shift| shift.next_sym == item.lhs } } - shift, next_state = prev_state.nterm_transitions.find {|shift, _| shift.next_sym == item.lhs } - prev_state.goto_follows(shift, next_state) + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] end - [item, value] + [kernel, value] }.to_h end + # @rbs (lookahead_set k) -> void def item_lookahead_set=(k) @item_lookahead_set = k end + # @rbs (Item item) -> Array[[State, Item]] def predecessors_with_item(item) result = [] @predecessors.each do |pre| @@ -349,69 +467,53 @@ module Lrama result end + # @rbs (State prev_state) -> void def append_predecessor(prev_state) @predecessors << prev_state @predecessors.uniq! end + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] def goto_follow_set(nterm_token) return [] if nterm_token.accept_symbol? - shift, next_state = @lalr_isocore.nterm_transitions.find {|sh, _| sh.next_sym == nterm_token } + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } @kernels - .select {|kernel| follow_kernel_items(shift, next_state, kernel) } + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } .map {|kernel| item_lookahead_set[kernel] } - .reduce(always_follows(shift, next_state)) {|result, terms| result |= terms } - end - - def goto_follows(shift, next_state) - queue = internal_dependencies(shift, next_state) + predecessor_dependencies(shift, next_state) - terms = always_follows(shift, next_state) - until queue.empty? - st, sh, next_st = queue.pop - terms |= st.always_follows(sh, next_st) - st.internal_dependencies(sh, next_st).each {|v| queue << v } - st.predecessor_dependencies(sh, next_st).each {|v| queue << v } - end - terms - end - - def always_follows(shift, next_state) - return @always_follows[[shift, next_state]] if @always_follows[[shift, next_state]] - - queue = internal_dependencies(shift, next_state) + successor_dependencies(shift, next_state) - terms = [] - until queue.empty? - st, sh, next_st = queue.pop - terms |= next_st.term_transitions.map {|sh, _| sh.next_sym } - st.internal_dependencies(sh, next_st).each {|v| queue << v } - st.successor_dependencies(sh, next_st).each {|v| queue << v } - end - @always_follows[[shift, next_state]] = terms + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } end - def internal_dependencies(shift, next_state) - return @internal_dependencies[[shift, next_state]] if @internal_dependencies[[shift, next_state]] + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] syms = @items.select {|i| - i.next_sym == shift.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 }.map(&:lhs).uniq - @internal_dependencies[[shift, next_state]] = nterm_transitions.select {|sh, _| syms.include?(sh.next_sym) }.map {|goto| [self, *goto] } + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } end - def successor_dependencies(shift, next_state) - return @successor_dependencies[[shift, next_state]] if @successor_dependencies[[shift, next_state]] + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] - @successor_dependencies[[shift, next_state]] = - next_state.nterm_transitions - .select {|next_shift, _| next_shift.next_sym.nullable } - .map {|transition| [next_state, *transition] } + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } end - def predecessor_dependencies(shift, next_state) + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) state_items = [] @kernels.select {|kernel| - kernel.next_sym == shift.next_sym && kernel.symbols_after_transition.all?(&:nullable) + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.each do |item| queue = predecessors_with_item(item) until queue.empty? @@ -425,8 +527,7 @@ module Lrama end state_items.map {|state, item| - sh, next_st = state.nterm_transitions.find {|shi, _| shi.next_sym == item.lhs } - [state, sh, next_st] + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } } end end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/states/item.rb b/tool/lrama/lib/lrama/state/item.rb index e89cb9695b..3ecdd70b76 100644 --- a/tool/lrama/lib/lrama/states/item.rb +++ b/tool/lrama/lib/lrama/state/item.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true # TODO: Validate position is not over rule rhs @@ -5,84 +6,112 @@ require "forwardable" module Lrama - class States + class State class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + extend Forwardable def_delegators "rule", :lhs, :rhs # Optimization for States#setup_state + # + # @rbs () -> Integer def hash [rule_id, position].hash end + # @rbs () -> Integer def rule_id rule.id end + # @rbs () -> bool def empty_rule? rule.empty_rule? end + # @rbs () -> Integer def number_of_rest_symbols - rhs.count - position + @number_of_rest_symbols ||= rhs.count - position end + # @rbs () -> Grammar::Symbol def next_sym rhs[position] end + # @rbs () -> Grammar::Symbol def next_next_sym - rhs[position + 1] + @next_next_sym ||= rhs[position + 1] end + # @rbs () -> Grammar::Symbol def previous_sym rhs[position - 1] end + # @rbs () -> bool def end_of_rule? rhs.count == position end + # @rbs () -> bool def beginning_of_rule? position == 0 end + # @rbs () -> bool def start_item? rule.initial_rule? && beginning_of_rule? end + # @rbs () -> State::Item def new_by_next_position Item.new(rule: rule, position: position + 1) end + # @rbs () -> Array[Grammar::Symbol] def symbols_before_dot # steep:ignore rhs[0...position] end + # @rbs () -> Array[Grammar::Symbol] def symbols_after_dot # steep:ignore rhs[position..-1] end - def symbols_after_transition + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore rhs[position+1..-1] end + # @rbs () -> ::String def to_s "#{lhs.id.s_value}: #{display_name}" end + # @rbs () -> ::String def display_name r = rhs.map(&:display_name).insert(position, "•").join(" ") "#{r} (rule #{rule_id})" end # Right after position + # + # @rbs () -> ::String def display_rest r = symbols_after_dot.map(&:display_name).join(" ") ". #{r} (rule #{rule_id})" end + # @rbs (State::Item other_item) -> bool def predecessor_item_of?(other_item) rule == other_item.rule && position == other_item.position - 1 end diff --git a/tool/lrama/lib/lrama/state/reduce.rb b/tool/lrama/lib/lrama/state/reduce.rb deleted file mode 100644 index 54ab87b468..0000000000 --- a/tool/lrama/lib/lrama/state/reduce.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class State - class Reduce - # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html - attr_reader :item, :look_ahead, :not_selected_symbols - attr_accessor :default_reduction - - def initialize(item) - @item = item - @look_ahead = nil - @not_selected_symbols = [] - end - - def rule - @item.rule - end - - def look_ahead=(look_ahead) - @look_ahead = look_ahead.freeze - end - - def add_not_selected_symbol(sym) - @not_selected_symbols << sym - end - - def selected_look_ahead - if look_ahead - look_ahead - @not_selected_symbols - else - [] - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb index 736d08376a..55ecad40bd 100644 --- a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State - class ReduceReduceConflict < Struct.new(:symbols, :reduce1, :reduce2, keyword_init: true) + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce def type :reduce_reduce end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb index 3bb3d1446e..014533c233 100644 --- a/tool/lrama/lib/lrama/state/resolved_conflict.rb +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -1,20 +1,54 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State + # * state: A state on which the conflct is resolved # * symbol: A symbol under discussion # * reduce: A reduce under discussion # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) - class ResolvedConflict < Struct.new(:symbol, :reduce, :which, :same_prec, keyword_init: true) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved s = symbol.display_name r = reduce.rule.precedence_sym&.display_name case - when which == :shift && same_prec + when which == :shift && resolved_by_precedence msg = "resolved as #{which} (%right #{s})" when which == :shift msg = "resolved as #{which} (#{r} < #{s})" - when which == :reduce && same_prec + when which == :reduce && resolved_by_precedence msg = "resolved as #{which} (%left #{s})" when which == :reduce msg = "resolved as #{which} (#{s} < #{r})" @@ -24,7 +58,7 @@ module Lrama raise "Unknown direction. #{self}" end - "Conflict between rule #{reduce.rule.id} and token #{s} #{msg}." + msg end end end diff --git a/tool/lrama/lib/lrama/state/shift.rb b/tool/lrama/lib/lrama/state/shift.rb deleted file mode 100644 index 81ef013a17..0000000000 --- a/tool/lrama/lib/lrama/state/shift.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class State - class Shift - attr_reader :next_sym, :next_items - attr_accessor :not_selected - - def initialize(next_sym, next_items) - @next_sym = next_sym - @next_items = next_items - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb index fd66834539..548f2de614 100644 --- a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State - class ShiftReduceConflict < Struct.new(:symbols, :shift, :reduce, keyword_init: true) + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce def type :shift_reduce end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb index fd8ded905f..ddce627df4 100644 --- a/tool/lrama/lib/lrama/states.rb +++ b/tool/lrama/lib/lrama/states.rb @@ -1,8 +1,9 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" -require_relative "report/duration" -require_relative "states/item" +require_relative "tracer/duration" +require_relative "state/item" module Lrama # States is passed to a template file @@ -10,17 +11,42 @@ module Lrama # "Efficient Computation of LALR(1) Look-Ahead Sets" # https://dl.acm.org/doi/pdf/10.1145/69622.357187 class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + extend Forwardable - include Lrama::Report::Duration + include Lrama::Tracer::Duration - def_delegators "@grammar", :symbols, :terms, :nterms, :rules, - :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value! + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? - attr_reader :states, :reads_relation, :includes_relation, :lookback_relation + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] - def initialize(grammar, trace_state: false) + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) @grammar = grammar - @trace_state = trace_state + @tracer = tracer @states = [] @@ -28,7 +54,7 @@ module Lrama # where p is state, A is nterm, t is term. # # `@direct_read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @direct_read_sets = {} @@ -37,14 +63,14 @@ module Lrama # where p, r are state, A, C are nterm. # # `@reads_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @reads_relation = {} # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` # # `@read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @read_sets = {} @@ -52,112 +78,163 @@ module Lrama # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. # # `@includes_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @includes_relation = {} # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. # - # `@lookback_relation` is a hash whose - # key is [state.id, rule.id], - # value is array of [state.id, nterm.token_id]. + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. @lookback_relation = {} # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` # # `@follow_sets` is a hash whose - # key is [state.id, rule.id], + # key is goto, # value is bitmap of term. @follow_sets = {} # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` # - # `@la` is a hash whose - # key is [state.id, rule.id], + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, # value is bitmap of term. @la = {} end + # @rbs () -> void def compute - # Look Ahead Sets report_duration(:compute_lr0_states) { compute_lr0_states } - report_duration(:compute_direct_read_sets) { compute_direct_read_sets } - report_duration(:compute_reads_relation) { compute_reads_relation } - report_duration(:compute_read_sets) { compute_read_sets } - report_duration(:compute_includes_relation) { compute_includes_relation } - report_duration(:compute_lookback_relation) { compute_lookback_relation } - report_duration(:compute_follow_sets) { compute_follow_sets } + + # Look Ahead Sets report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } # Conflicts - report_duration(:compute_conflicts) { compute_conflicts } + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } report_duration(:compute_default_reduction) { compute_default_reduction } end + # @rbs () -> void def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 report_duration(:split_states) { split_states } - report_duration(:compute_direct_read_sets) { compute_direct_read_sets } - report_duration(:compute_reads_relation) { compute_reads_relation } - report_duration(:compute_read_sets) { compute_read_sets } - report_duration(:compute_includes_relation) { compute_includes_relation } - report_duration(:compute_lookback_relation) { compute_lookback_relation } - report_duration(:compute_follow_sets) { compute_follow_sets } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } - report_duration(:compute_conflicts) { compute_conflicts } - + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } report_duration(:compute_default_reduction) { compute_default_reduction } end - def reporter - StatesReporter.new(self) - end - + # @rbs () -> Integer def states_count @states.count end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def direct_read_sets - @direct_read_sets.transform_values do |v| + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def read_sets - @read_sets.transform_values do |v| + @_read_sets ||= @read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def follow_sets - @follow_sets.transform_values do |v| + @_follow_sets ||= @follow_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] def la - @la.transform_values do |v| - bitmap_to_terms(v) + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end end end + # @rbs () -> Integer def sr_conflicts_count @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count end + # @rbs () -> Integer def rr_conflicts_count @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count end - private + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end - def trace_state - if @trace_state - yield STDERR + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end end end + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] def create_state(accessing_symbol, kernels, states_created) # A item can appear in some states, # so need to use `kernels` (not `kernels.first`) as a key. @@ -204,27 +281,25 @@ module Lrama return [state, true] end + # @rbs (State state) -> void def setup_state(state) # closure closure = [] - visited = {} queued = {} items = state.kernels.dup items.each do |item| - queued[item] = true + queued[item.rule_id] = true if item.position == 0 end while (item = items.shift) do - visited[item] = true - if (sym = item.next_sym) && sym.nterm? @grammar.find_rules_by_symbol!(sym).each do |rule| - i = Item.new(rule: rule, position: 0) - next if queued[i] + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) closure << i items << i - queued[i] = true + queued[i.rule_id] = true end end end @@ -232,119 +307,107 @@ module Lrama state.closure = closure.sort_by {|i| i.rule.id } # Trace - trace_state do |out| - out << "Closure: input\n" - state.kernels.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - out << "Closure: output\n" - state.items.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - end + @tracer.trace_closure(state) # shift & reduce - state.compute_shifts_reduces + state.compute_transitions_and_reduces end + # @rbs (Array[State] states, State state) -> void def enqueue_state(states, state) # Trace - previous = state.kernels.first.previous_sym - trace_state do |out| - out << sprintf("state_list_append (state = %d, symbol = %d (%s))\n", - @states.count, previous.number, previous.display_name) - end + @tracer.trace_state_list_append(@states.count, state) states << state end + # @rbs () -> void def compute_lr0_states # State queue states = [] states_created = {} - state, _ = create_state(symbols.first, [Item.new(rule: @grammar.rules.first, position: 0)], states_created) + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) enqueue_state(states, state) while (state = states.shift) do # Trace - # - # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but - # I think it is not correct... - previous = state.kernels.first.previous_sym - trace_state do |out| - out << "Processing state #{state.id} (reached by #{previous.display_name})\n" - end + @tracer.trace_state(state) setup_state(state) - state.shifts.each do |shift| - new_state, created = create_state(shift.next_sym, shift.next_items, states_created) - state.set_items_to_state(shift.next_items, new_state) - if created - enqueue_state(states, new_state) - new_state.append_predecessor(state) - end + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) + enqueue_state(states, new_state) if created end end end + # @rbs () -> Array[State::Action::Goto] def nterm_transitions a = [] @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - a << [state, nterm, next_state] + state.nterm_transitions.each do |goto| + a << goto end end a end + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void def compute_direct_read_sets @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - - ary = next_state.term_transitions.map do |shift, _| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| shift.next_sym.number end - key = [state.id, nterm.token_id] - @direct_read_sets[key] = Bitmap.from_array(ary) + @direct_read_sets[goto] = Bitmap.from_array(ary) end end end + # @rbs () -> void def compute_reads_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - next_state.nterm_transitions.each do |shift2, _next_state2| - nterm2 = shift2.next_sym + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym if nterm2.nullable - key = [state.id, nterm.token_id] - @reads_relation[key] ||= [] - @reads_relation[key] << [next_state.id, nterm2.token_id] + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 end end end end end + # @rbs () -> void def compute_read_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @read_sets = Digraph.new(sets, @reads_relation, @direct_read_sets).compute + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute end # Execute transition of state by symbols # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State def transition(state, symbols) symbols.each do |sym| state = state.transition(sym) @@ -353,10 +416,11 @@ module Lrama state end + # @rbs () -> void def compute_includes_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| i = rule.rhs.count - 1 @@ -366,10 +430,12 @@ module Lrama break if sym.term? state2 = transition(state, rule.rhs[0...i]) # p' = state, B = nterm, p = state2, A = sym - key = [state2.id, sym.token_id] + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") # TODO: need to omit if state == state2 ? @includes_relation[key] ||= [] - @includes_relation[key] << [state.id, nterm.token_id] + @includes_relation[key] << goto break unless sym.nullable i -= 1 end @@ -378,45 +444,46 @@ module Lrama end end + # @rbs () -> void def compute_lookback_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| state2 = transition(state, rule.rhs) # p = state, A = nterm, q = state2, A -> ω = rule - key = [state2.id, rule.id] - @lookback_relation[key] ||= [] - @lookback_relation[key] << [state.id, nterm.token_id] + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto end end end end + # @rbs () -> void def compute_follow_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @follow_sets = Digraph.new(sets, @includes_relation, @read_sets).compute + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute end - def compute_look_ahead_sets + # @rbs () -> void + def compute_la @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state rules.each do |rule| - ary = @lookback_relation[[state.id, rule.id]] + ary = lookback_relation_on_state[rule.id] next unless ary - ary.each do |state2_id, nterm_token_id| + ary.each do |goto| # q = state, A -> ω = rule, p = state2, A = nterm - follows = @follow_sets[[state2_id, nterm_token_id]] + follows = @follow_sets[goto] next if follows == 0 - key = [state.id, rule.id] - @la[key] ||= 0 - look_ahead = @la[key] | follows - @la[key] |= look_ahead + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead # No risk of conflict when # * the state only has single reduce @@ -429,6 +496,7 @@ module Lrama end end + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] def bitmap_to_terms(bit) ary = Bitmap.to_array(bit) ary.map do |i| @@ -436,14 +504,16 @@ module Lrama end end - def compute_conflicts - compute_shift_reduce_conflicts + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) compute_reduce_reduce_conflicts end - def compute_shift_reduce_conflicts + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) states.each do |state| - state.shifts.each do |shift| + state.term_transitions.each do |shift| state.reduces.each do |reduce| sym = shift.next_sym @@ -463,43 +533,57 @@ module Lrama case when shift_prec < reduce_prec # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when shift_prec > reduce_prec # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next end # shift_prec == reduce_prec, then check associativity case sym.precedence.type when :precedence + # Can not resolve the conflict + # # %precedence only specifies precedence and not specify associativity # then a conflict is unresolved if precedence is same. state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) next when :right # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :left # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :nonassoc - # Can not resolve + # The conflict is resolved # - # nonassoc creates "run-time" error, precedence creates "compile-time" error. - # Then omit both the shift and reduce. + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. # # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :error) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) else raise "Unknown precedence type. #{sym}" end @@ -508,35 +592,41 @@ module Lrama end end + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void def compute_reduce_reduce_conflicts states.each do |state| - count = state.reduces.count - - (0...count).each do |i| - reduce1 = state.reduces[i] - next if reduce1.look_ahead.nil? + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? - ((i+1)...count).each do |j| - reduce2 = state.reduces[j] - next if reduce2.look_ahead.nil? + intersection = reduce1.look_ahead & reduce2.look_ahead - intersection = reduce1.look_ahead & reduce2.look_ahead - - unless intersection.empty? - state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) - end + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) end end end end + # @rbs () -> void def compute_default_reduction states.each do |state| next if state.reduces.empty? # Do not set, if conflict exist next unless state.conflicts.empty? # Do not set, if shift with `error` exists. - next if state.shifts.map(&:next_sym).include?(@grammar.error_symbol) + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) state.default_reduction_rule = state.reduces.map do |r| [r.rule, r.rule.id, (r.look_ahead || []).count] @@ -546,35 +636,171 @@ module Lrama end end + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void def split_states @states.each do |state| - state.transitions.each do |shift, next_state| - compute_state(state, shift, next_state) + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) end end end + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void def merge_lookaheads(state, filtered_lookaheads) return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } - state.transitions.each do |shift, next_state| - next if next_state.lookaheads_recomputed - compute_state(state, shift, next_state) + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) end end - def compute_state(state, shift, next_state) - filtered_lookaheads = state.propagate_lookaheads(next_state) - s = next_state.ielr_isocores.find {|st| st.compatible_lookahead?(filtered_lookaheads) } + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } if s.nil? - s = next_state.ielr_isocores.last + s = next_state.lalr_isocore new_state = State.new(@states.count, s.accessing_symbol, s.kernels) new_state.closure = s.closure - new_state.compute_shifts_reduces - s.transitions.each do |sh, next_state| - new_state.set_items_to_state(sh.next_items, next_state) + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) end @states << new_state new_state.lalr_isocore = s @@ -582,14 +808,60 @@ module Lrama s.ielr_isocores.each do |st| st.ielr_isocores = s.ielr_isocores end - new_state.item_lookahead_set = filtered_lookaheads - state.update_transition(shift, new_state) + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) elsif(!s.lookaheads_recomputed) - s.item_lookahead_set = filtered_lookaheads + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads else - state.update_transition(shift, s) - merge_lookaheads(s, filtered_lookaheads) + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id end end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end end end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb deleted file mode 100644 index 64ff4de100..0000000000 --- a/tool/lrama/lib/lrama/states_reporter.rb +++ /dev/null @@ -1,362 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class StatesReporter - include Lrama::Report::Duration - - def initialize(states) - @states = states - end - - def report(io, **options) - report_duration(:report) do - _report(io, **options) - end - end - - private - - def _report(io, grammar: false, rules: false, terms: false, states: false, itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false) - report_unused_rules(io) if rules - report_unused_terms(io) if terms - report_conflicts(io) - report_grammar(io) if grammar - report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - end - - def report_unused_terms(io) - look_aheads = @states.states.each do |state| - state.reduces.flat_map do |reduce| - reduce.look_ahead unless reduce.look_ahead.nil? - end - end - - next_terms = @states.states.flat_map do |state| - state.shifts.map(&:next_sym).select(&:term?) - end - - unused_symbols = @states.terms.select do |term| - !(look_aheads + next_terms).include?(term) - end - - unless unused_symbols.empty? - io << "#{unused_symbols.count} Unused Terms\n\n" - unused_symbols.each_with_index do |term, index| - io << sprintf("%5d %s\n", index, term.id.s_value) - end - io << "\n\n" - end - end - - def report_unused_rules(io) - used_rules = @states.rules.flat_map(&:rhs) - - unused_rules = @states.rules.map(&:lhs).select do |rule| - !used_rules.include?(rule) && rule.token_id != 0 - end - - unless unused_rules.empty? - io << "#{unused_rules.count} Unused Rules\n\n" - unused_rules.each_with_index do |rule, index| - io << sprintf("%5d %s\n", index, rule.display_name) - end - io << "\n\n" - end - end - - def report_conflicts(io) - has_conflict = false - - @states.states.each do |state| - messages = [] - cs = state.conflicts.group_by(&:type) - if cs[:shift_reduce] - messages << "#{cs[:shift_reduce].count} shift/reduce" - end - - if cs[:reduce_reduce] - messages << "#{cs[:reduce_reduce].count} reduce/reduce" - end - - unless messages.empty? - has_conflict = true - io << "State #{state.id} conflicts: #{messages.join(', ')}\n" - end - end - - if has_conflict - io << "\n\n" - end - end - - def report_grammar(io) - io << "Grammar\n" - last_lhs = nil - - @states.rules.each do |rule| - if rule.empty_rule? - r = "ε" - else - r = rule.rhs.map(&:display_name).join(" ") - end - - if rule.lhs == last_lhs - io << sprintf("%5d %s| %s\n", rule.id, " " * rule.lhs.display_name.length, r) - else - io << "\n" - io << sprintf("%5d %s: %s\n", rule.id, rule.lhs.display_name, r) - end - - last_lhs = rule.lhs - end - io << "\n\n" - end - - def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - if counterexamples - cex = Counterexamples.new(@states) - end - - @states.states.each do |state| - # Report State - io << "State #{state.id}\n\n" - - # Report item - last_lhs = nil - list = itemsets ? state.items : state.kernels - list.sort_by {|i| [i.rule_id, i.position] }.each do |item| - if item.empty_rule? - r = "ε •" - else - r = item.rhs.map(&:display_name).insert(item.position, "•").join(" ") - end - if item.lhs == last_lhs - l = " " * item.lhs.id.s_value.length + "|" - else - l = item.lhs.id.s_value + ":" - end - la = "" - if lookaheads && item.end_of_rule? - reduce = state.find_reduce_by_item!(item) - look_ahead = reduce.selected_look_ahead - unless look_ahead.empty? - la = " [#{look_ahead.map(&:display_name).join(", ")}]" - end - end - last_lhs = item.lhs - - io << sprintf("%5i %s %s%s\n", item.rule_id, l, r, la) - end - io << "\n" - - # Report shifts - tmp = state.term_transitions.reject do |shift, _| - shift.not_selected - end.map do |shift, next_state| - [shift.next_sym, next_state.id] - end - max_len = tmp.map(&:first).map(&:display_name).map(&:length).max - tmp.each do |term, state_id| - io << " #{term.display_name.ljust(max_len)} shift, and go to state #{state_id}\n" - end - io << "\n" unless tmp.empty? - - # Report error caused by %nonassoc - nl = false - tmp = state.resolved_conflicts.select do |resolved| - resolved.which == :error - end.map do |error| - error.symbol.display_name - end - max_len = tmp.map(&:length).max - tmp.each do |name| - nl = true - io << " #{name.ljust(max_len)} error (nonassociative)\n" - end - io << "\n" unless tmp.empty? - - # Report reduces - nl = false - max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 - max_len = [max_len, "$default".length].max if state.default_reduction_rule - ary = [] - - state.non_default_reduces.each do |reduce| - reduce.look_ahead.each do |term| - ary << [term, reduce] - end - end - - ary.sort_by do |term, reduce| - term.number - end.each do |term, reduce| - rule = reduce.item.rule - io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" - nl = true - end - - if (r = state.default_reduction_rule) - nl = true - s = "$default".ljust(max_len) - - if r.initial_rule? - io << " #{s} accept\n" - else - io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" - end - end - io << "\n" if nl - - # Report nonterminal transitions - tmp = [] - max_len = 0 - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - tmp << [nterm, next_state.id] - max_len = [max_len, nterm.id.s_value.length].max - end - tmp.uniq! - tmp.sort_by! do |nterm, state_id| - nterm.number - end - tmp.each do |nterm, state_id| - io << " #{nterm.id.s_value.ljust(max_len)} go to state #{state_id}\n" - end - io << "\n" unless tmp.empty? - - if solved - # Report conflict resolutions - state.resolved_conflicts.each do |resolved| - io << " #{resolved.report_message}\n" - end - io << "\n" unless state.resolved_conflicts.empty? - end - - if counterexamples && state.has_conflicts? - # Report counterexamples - examples = cex.compute(state) - examples.each do |example| - label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" - label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" - label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" - - io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" - io << " #{example.path1_item}\n" - io << " #{example.path2_item}\n" - io << " #{label1}\n" - example.derivations1.render_strings_for_report.each do |str| - io << " #{str}\n" - end - io << " #{label2}\n" - example.derivations2.render_strings_for_report.each do |str| - io << " #{str}\n" - end - end - end - - if verbose - # Report direct_read_sets - io << " [Direct Read sets]\n" - direct_read_sets = @states.direct_read_sets - @states.nterms.each do |nterm| - terms = direct_read_sets[[state.id, nterm.token_id]] - next unless terms - next if terms.empty? - - str = terms.map {|sym| sym.id.s_value }.join(", ") - io << " read #{nterm.id.s_value} shift #{str}\n" - end - io << "\n" - - # Report reads_relation - io << " [Reads Relation]\n" - @states.nterms.each do |nterm| - a = @states.reads_relation[[state.id, nterm.token_id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report read_sets - io << " [Read sets]\n" - read_sets = @states.read_sets - @states.nterms.each do |nterm| - terms = read_sets[[state.id, nterm.token_id]] - next unless terms - next if terms.empty? - - terms.each do |sym| - io << " #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report includes_relation - io << " [Includes Relation]\n" - @states.nterms.each do |nterm| - a = @states.includes_relation[[state.id, nterm.token_id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state.id}, #{nterm.id.s_value}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report lookback_relation - io << " [Lookback Relation]\n" - @states.rules.each do |rule| - a = @states.lookback_relation[[state.id, rule.id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (Rule: #{rule.display_name}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report follow_sets - io << " [Follow sets]\n" - follow_sets = @states.follow_sets - @states.nterms.each do |nterm| - terms = follow_sets[[state.id, nterm.token_id]] - - next unless terms - - terms.each do |sym| - io << " #{nterm.id.s_value} -> #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report LA - io << " [Look-Ahead Sets]\n" - tmp = [] - max_len = 0 - @states.rules.each do |rule| - syms = @states.la[[state.id, rule.id]] - next unless syms - - tmp << [rule, syms] - max_len = ([max_len] + syms.map {|s| s.id.s_value.length }).max - end - tmp.each do |rule, syms| - syms.each do |sym| - io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" - end - end - io << "\n" unless tmp.empty? - end - - # End of Report State - io << "\n" - end - end - end -end diff --git a/tool/lrama/lib/lrama/trace_reporter.rb b/tool/lrama/lib/lrama/trace_reporter.rb deleted file mode 100644 index bcf1ef1e50..0000000000 --- a/tool/lrama/lib/lrama/trace_reporter.rb +++ /dev/null @@ -1,45 +0,0 @@ -# rbs_inline: enabled -# frozen_string_literal: true - -module Lrama - class TraceReporter - # @rbs (Lrama::Grammar grammar) -> void - def initialize(grammar) - @grammar = grammar - end - - # @rbs (**Hash[Symbol, bool] options) -> void - def report(**options) - _report(**options) - end - - private - - # @rbs rules: (bool rules, bool actions, bool only_explicit_rules, **untyped _) -> void - def _report(rules: false, actions: false, only_explicit_rules: false, **_) - report_rules if rules && !only_explicit_rules - report_only_explicit_rules if only_explicit_rules - report_actions if actions - end - - # @rbs () -> void - def report_rules - puts "Grammar rules:" - @grammar.rules.each { |rule| puts rule.display_name } - end - - # @rbs () -> void - def report_only_explicit_rules - puts "Grammar rules:" - @grammar.rules.each do |rule| - puts rule.display_name_without_action if rule.lhs.first_set.any? - end - end - - # @rbs () -> void - def report_actions - puts "Grammar rules with actions:" - @grammar.rules.each { |rule| puts rule.with_actions } - end - end -end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 12ece5a8f2..eb1d1b46c7 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,5 +1,6 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama - VERSION = "0.7.0".freeze + VERSION = "0.8.0".freeze #: String end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h index 34ed6d81f5..3e270c9171 100644 --- a/tool/lrama/template/bison/_yacc.h +++ b/tool/lrama/template/bison/_yacc.h @@ -28,6 +28,7 @@ extern int yydebug; <%-# b4_declare_yylstype -%> <%-# b4_value_type_define -%> /* Value type. */ +<% if output.grammar.union %> #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { @@ -40,6 +41,13 @@ typedef union YYSTYPE YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define YYSTYPE_IS_DECLARED 1 #endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> <%-# b4_location_type_define -%> /* Location type. */ diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index 98359fa1f9..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -4,6 +4,6 @@ AC_DEFUN([RUBY_APPEND_OPTION], AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl AC_DEFUN([RUBY_PREPEND_OPTION], - [# RUBY_APPEND_OPTION($1) + [# RUBY_PREPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e9ed68e5b8..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/make-snapshot b/tool/make-snapshot index c7ccc468d4..dff636d601 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -10,6 +10,7 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) @@ -53,7 +54,7 @@ PACKAGES = { "xz" => %w".tar.xz xz -c", "zip" => %w".zip zip -Xqr", } -DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] +DEFAULT_PACKAGES = PACKAGES.keys - ["tar", "bzip"] if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} @@ -252,7 +253,6 @@ end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd - patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` @@ -269,22 +269,23 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return @@ -334,7 +335,7 @@ def package(vcs, rev, destdir, tmp = nil) FileUtils.rm(file, verbose: $VERBOSE) end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -346,26 +347,20 @@ def package(vcs, rev, destdir, tmp = nil) File.open("#{v}/revision.h", "wb") {|f| f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") + if prerelease + versionhdr ||= File.read("#{v}/version.h") versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" - IO.write("#{v}/version.h", versionhdr) + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end @@ -430,7 +425,7 @@ def package(vcs, rev, destdir, tmp = nil) puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -439,11 +434,11 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end @@ -456,7 +451,7 @@ def package(vcs, rev, destdir, tmp = nil) File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -472,6 +467,7 @@ def package(vcs, rev, destdir, tmp = nil) "VPATH"=>(ENV["VPATH"] || "include/ruby"), "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), + "DUMP_AST"=>"build-tool/dump_ast#{RbConfig::CONFIG['EXEEXT']} ", } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 @@ -480,6 +476,13 @@ def package(vcs, rev, destdir, tmp = nil) vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' @@ -508,6 +511,7 @@ touch-unicode-files: File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"])) return unless make.run("prepare-package") return unless make.run("clean-cache") + return unless make.run("clean") if modified new_time = modified + 2 touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat| @@ -634,7 +638,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version - info[key]['date'] = release_date.strftime('%Y-%m-%d') + info[key]['date'] = release_date.to_date if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" diff --git a/tool/merger.rb b/tool/merger.rb index 8b12334b73..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -65,7 +65,8 @@ class << Merger = Object.new if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -113,7 +114,13 @@ class << Merger = Object.new abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" + else + tagname = "v#{v.join('_')}" + end unless execute('git', 'diff', '--exit-code') abort 'uncommitted changes' @@ -135,10 +142,12 @@ class << Merger = Object.new unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" + else + tagname = "v#{relname.gsub(/[.-]/, '_')}" end execute('git', 'tag', '-d', tagname) @@ -263,7 +272,7 @@ else end # Merge revision from Git patch - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" resp = Net::HTTP.get_response(URI(git_uri)) if resp.code != '200' abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index fcc75ea902..d39568fe86 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -18,6 +18,13 @@ : ; abort () { exit 1; } call :warn "executable host ruby is required. use --with-baseruby option." -call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." call :abort -: || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 6e1f5c666a..a84f322e84 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -1,12 +1,13 @@ # Parse built-in script and make rbinc file -require 'ripper' +require 'json' +require 'open3' require 'stringio' require_relative 'ruby_vm/helpers/c_escape' SUBLIBS = {} REQUIRED = {} -BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace without_interrupts] module CompileWarning @@warnings = 0 @@ -24,231 +25,204 @@ end Warning.extend CompileWarning -def string_literal(lit, str = []) - while lit - case lit.first - when :string_concat, :string_embexpr, :string_content - _, *lit = lit - lit.each {|s| string_literal(s, str)} - return str - when :string_literal - _, lit = lit - when :@tstring_content - str << lit[1] - return str - else - raise "unexpected #{lit.first}" - end - end -end +# ruby mk_builtin_loader.rb path/to/dump_ast TARGET_FILE.rb +# #=> generate TARGET_FILE.rbinc +# +# dump_ast is a standalone C program (tool/dump_ast.c) that parses Ruby files +# with prism and dumps the AST as JSON. It must be compiled with CC before this +# script can run, which means rbinc generation is skipped during `make up` +# (where CC=false). The rbinc files are gitignored build artifacts, so they do +# not need to be present in srcdir after `make up` — they will be generated in +# the build directory during `make all` once dump_ast has been compiled. + +LOCALS_DB = {} # [method_name, first_line] = locals -# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] -def symbol_literal(lit) - symbol_literal, symbol_lit = lit - raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal - symbol, ident_lit = symbol_lit - raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol - ident, symbol_name, = ident_lit - raise "#{ident.inspect} was not :@ident" if ident != :@ident - symbol_name +# Extract the contents of the given string node. +def extract_string_literal(node) + case node["type"] + when "StringNode" + node["unescaped"] + when "InterpolatedStringNode" + node["parts"].map { |part| extract_string_literal(part) }.join + else + raise "unexpected #{node["type"]}" + end end -def inline_text argc, arg1 - raise "argc (#{argc}) of inline! should be 1" unless argc == 1 - arg1 = string_literal(arg1) - raise "1st argument should be string literal" unless arg1 - arg1.join("").rstrip +# Retrieve the line number of the given node in the source. +def line_number(source, node) + source.b.byteslice(0, node["location"]["start"]).count("\n") + 1 end -def inline_attrs(args) - raise "args was empty" if args.empty? - args.each do |arg| - attr = symbol_literal(arg) - unless BUILTIN_ATTRS.include?(attr) - raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" - end +def visit_call_node(source, node, name, locals, requires, bs, inlines) + # If this is a call to require or require relative with a single string node + # argument, then we will attempt to find the file that is being required and + # add it to the files that should be processed. + if %w[require require_relative].include?(node["name"]) && !node["arguments"].nil? && (argument = node["arguments"]["arguments"][0])["type"] == "StringNode" + requires << argument["unescaped"] + return true end -end -def make_cfunc_name inlines, name, lineno - case name - when /\[\]/ - name = '_GETTER' - when /\[\]=/ - name = '_SETTER' + primitive_name = nil + + receiver = node["receiver"] + + if (!receiver.nil? && receiver["type"] == "ConstantReadNode" && receiver["name"] == "Primitive") || + (!receiver.nil? && receiver["type"] == "CallNode" && receiver["flags"].include?("VARIABLE_CALL") && receiver["name"] == "__builtin") + primitive_name = node["name"] + elsif node["name"].start_with?("__builtin_") + primitive_name = node["name"][10..-1] else - name = name.tr('!?', 'EP') + # If we get here, then this isn't a primitive function call and we can + # continue the visit. + return true end - base = "builtin_inline_#{name}_#{lineno}" - if inlines[base] - 1000.times{|i| - name = "#{base}_#{i}" - return name unless inlines[name] - } - raise "too many functions in same line..." - else - base + # The name of the C function that we will be calling for this call node. It + # may change later in this method depending on the type of primitive. + cfunction_name = primitive_name + + args = node["arguments"].nil? ? [] : node["arguments"]["arguments"] + argc = args.size + + if primitive_name.match?(/[\!\?]$/) + case (primitive_macro = primitive_name[0...-1]) + when "arg" + # This is a call to Primitive.arg!, which expects a single symbol argument + # detailing the name of the argument. + raise "unexpected argument number #{argc}" if argc != 1 + raise "symbol literal expected, got #{args[0]["type"]}" if args[0]["type"] != "SymbolNode" + return true + when "attr" + # This is a call to Primitive.attr!, which expects a list of known + # symbols. We will check that each of the arguments is a symbol and that + # the symbol is one of the known symbols. + raise "args was empty" if argc == 0 + + args.each do |arg| + raise "#{arg["type"]} was not a SymbolNode" if arg["type"] != "SymbolNode" + raise "attr (#{arg["unescaped"]}) was not in: leaf, inline_block, use_block" unless BUILTIN_ATTRS.include?(arg["unescaped"]) + end + + return true + when "mandatory_only" + # This is a call to Primitive.mandatory_only?. This method does not + # require any further processing. + return true + when "cstmt", "cexpr", "cconst", "cinit" + # This is a call to Primitive.cstmt!, Primitive.cexpr!, Primitive.cconst!, + # or Primitive.cinit!. These methods expect a single string argument that + # is the C code that should be executed. We will extract the string, emit + # an inline function, and then continue the visit. + raise "argc (#{argc}) of inline! should be 1" if argc != 1 + + text = extract_string_literal(args[0]).rstrip + lineno = line_number(source, node) + + case primitive_macro + when "cstmt", "cexpr", "cconst" + cfunction_name = "builtin_inline_#{name}_#{lineno}" + primitive_name = "_bi#{lineno}" + + if primitive_macro == "cstmt" + inlines << [cfunction_name, lineno, text, locals, primitive_name] + else + inlines << [cfunction_name, lineno, "return #{text};", primitive_macro == "cexpr" ? locals : nil, primitive_name] + end + when "cinit" + inlines << [inlines.size, lineno, text, nil, nil] + return true + end + + argc -= 1 + else + # This is a call to Primitive that is not a known method, so it must be a + # regular C function. In this case we do not need any special processing. + end end + + bs << [primitive_name, argc, cfunction_name] + return true end -def collect_locals tree - _type, name, (line, _cols) = tree - if locals = LOCALS_DB[[name, line]] - locals - else - if false # for debugging - pp LOCALS_DB - raise "not found: [#{name}, #{line}]" +def each_node(root, &blk) + return unless yield root + + root.each do |key, value| + next if key == "type" || key == "location" + + if value.is_a?(Hash) + each_node(value, &blk) if value.key?("type") + elsif value.is_a?(Array) && value[0].is_a?(Hash) + value.each { |node| each_node(node, &blk) } end end end -def collect_builtin base, tree, name, bs, inlines, locals = nil - while tree - recv = sep = mid = args = nil - case tree.first - when :def - locals = collect_locals(tree[1]) - tree = tree[3] - next - when :defs - locals = collect_locals(tree[3]) - tree = tree[5] - next - when :class - name = 'class' - tree = tree[3] - next - when :sclass, :module - name = 'class' - tree = tree[2] - next - when :method_add_arg - _method_add_arg, mid, (_arg_paren, args) = tree - case mid.first - when :call - _, recv, sep, mid = mid - when :fcall - _, mid = mid - else - mid = nil - end - # w/ trailing comma: [[:method_add_arg, ...]] - # w/o trailing comma: [:args_add_block, [[:method_add_arg, ...]], false] - if args && args.first == :args_add_block - args = args[1] - end - when :vcall - _, mid = tree - when :command # FCALL - _, mid, (_, args) = tree - when :call, :command_call # CALL - _, recv, sep, mid, (_, args) = tree +def visit_node(source, root, name, locals, requires, bs, inlines) + each_node(root) do |node| + case node["type"] + when "CallNode" + visit_call_node(source, node, name, locals, requires, bs, inlines) + when "DefNode" + lineno = line_number(source, node) + visit_node(source, node["body"], name, LOCALS_DB[[node["name"], lineno]], requires, bs, inlines) if node["body"] + false + when "ClassNode", "ModuleNode", "SingletonClassNode" + visit_node(source, node["body"], "class", nil, requires, bs, inlines) if node["body"] + false + else + true end + end +end - if mid - raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) - _, mid, (lineno,) = mid - if recv - func_name = nil - case recv.first - when :var_ref - _, recv = recv - if recv.first == :@const and recv[1] == "Primitive" - func_name = mid.to_s - end - when :vcall - _, recv = recv - if recv.first == :@ident and recv[1] == "__builtin" - func_name = mid.to_s - end - end - collect_builtin(base, recv, name, bs, inlines) unless func_name - else - func_name = mid[/\A__builtin_(.+)/, 1] - end - if func_name - cfunc_name = func_name - args.pop unless (args ||= []).last - argc = args.size - - if /(.+)[\!\?]\z/ =~ func_name - case $1 - when 'attr' - # Compile-time validation only. compile.c will parse them. - inline_attrs(args) - break - when 'cstmt' - text = inline_text argc, args.first - - func_name = "_bi#{lineno}" - cfunc_name = make_cfunc_name(inlines, name, lineno) - inlines[cfunc_name] = [lineno, text, locals, func_name] - argc -= 1 - when 'cexpr', 'cconst' - text = inline_text argc, args.first - code = "return #{text};" - - func_name = "_bi#{lineno}" - cfunc_name = make_cfunc_name(inlines, name, lineno) - - locals = [] if $1 == 'cconst' - inlines[cfunc_name] = [lineno, code, locals, func_name] - argc -= 1 - when 'cinit' - text = inline_text argc, args.first - func_name = nil # required - inlines[inlines.size] = [lineno, text, nil, nil] - argc -= 1 - when 'mandatory_only' - func_name = nil - when 'arg' - argc == 1 or raise "unexpected argument number #{argc}" - (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" - (arg = arg[1])[0] == :symbol or raise "symbol expected #{arg}" - (var = arg[1] and var = var[1]) or raise "argument name expected #{arg}" - func_name = nil - end - end +def collect_builtins(dump_ast, file) + stdout, stderr, status = Open3.capture3(dump_ast, file) + unless status.success? + warn(stderr) + exit(1) + end - if bs[func_name] && - bs[func_name] != [argc, cfunc_name] - raise "same builtin function \"#{func_name}\", but different arity (was #{bs[func_name]} but #{argc})" - end + source = File.read(file) + root = JSON.parse(stdout) + visit_node(source, root, "top", nil, requires = [], builtins = [], inlines = []) - bs[func_name] = [argc, cfunc_name] if func_name - elsif /\Arequire(?:_relative)\z/ =~ mid and args.size == 1 and - (arg1 = args[0])[0] == :string_literal and - (arg1 = arg1[1])[0] == :string_content and - (arg1 = arg1[1])[0] == :@tstring_content and - sublib = arg1[1] - if File.exist?(f = File.join(@dir, sublib)+".rb") - puts "- #{@base}.rb requires #{sublib}" - if REQUIRED[sublib] - warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" - else - REQUIRED[sublib] = @base - (SUBLIBS[@base] ||= []) << sublib - end - ARGV.push(f) - end + requires.each do |sublib| + if File.exist?(f = File.join(@dir, sublib)+".rb") + puts "- #{@base}.rb requires #{sublib}" + if REQUIRED[sublib] + warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" + else + REQUIRED[sublib] = @base + (SUBLIBS[@base] ||= []) << sublib end - break unless tree = args + ARGV.push(f) end + end - tree.each do |t| - collect_builtin base, t, name, bs, inlines, locals if Array === t + processed_builtins = {} + builtins.each do |(primitive_name, argc, cfunction_name)| + if processed_builtins.key?(primitive_name) && processed_builtins[primitive_name] != [argc, cfunction_name] + raise "same builtin function \"#{primitive_name}\", but different arity (was #{processed_builtins[primitive_name]} but #{argc})" end - break + + processed_builtins[primitive_name] = [argc, cfunction_name] end -end -# ruby mk_builtin_loader.rb TARGET_FILE.rb -# #=> generate TARGET_FILE.rbinc -# + processed_inlines = {} + inlines.each do |(cfunction_name, lineno, text, locals, primitive_name)| + if processed_inlines.key?(cfunction_name) + found = 1000.times.find { |i| !processed_inlines.key?("#{cfunction_name}_#{i}") } + raise "too many functions in same line..." unless found + cfunction_name = "#{cfunction_name}_#{found}" + end -LOCALS_DB = {} # [method_name, first_line] = locals + processed_inlines[cfunction_name] = [lineno, text, locals, primitive_name] + end + + [processed_builtins, processed_inlines] +end def collect_iseq iseq_ary # iseq_ary.each_with_index{|e, i| p [i, e]} @@ -282,17 +256,22 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam # Avoid generating fetches of lvars we don't need. This is imperfect as it # will match text inside strings or other false positives. - local_candidates = text.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) f.puts '{' lineno += 1 # locals is nil outside methods locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - next unless local_candidates.include?(param.to_s) + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" - f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" - lineno += 1 + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" lineno += 1 @@ -308,24 +287,24 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam return lineno, f.string end -def mk_builtin_header file +def mk_builtin_header dump_ast, file @dir = File.dirname(file) base = File.basename(file, '.rb') @base = base ofile = "#{file}inc" - # bs = { func_name => argc } - code = File.read(file) begin verbose, $VERBOSE = $VERBOSE, true - collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + collect_iseq RubyVM::InstructionSequence.compile_file(file).to_a ensure $VERBOSE = verbose end if warnings = CompileWarning.reset raise "#{warnings} warnings in #{file}" end - collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + # bs = { func_name => argc } + bs, inlines = collect_builtins(dump_ast, file) StringIO.open do |f| if File::ALT_SEPARATOR @@ -418,7 +397,9 @@ def mk_builtin_header file end end +dump_ast = ARGV.shift + ARGV.each{|file| # feature.rb => load_feature.inc - mk_builtin_header file + mk_builtin_header dump_ast, file } diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index ffa4c1c0b2..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -394,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb index c82d31d743..b272c448c6 100755 --- a/tool/outdate-bundled-gems.rb +++ b/tool/outdate-bundled-gems.rb @@ -60,6 +60,7 @@ class Removal def initialize(base = nil) @base = (File.join(base, "/") if base) @remove = {} + @defaults = nil end def prefixed(name) @@ -92,10 +93,22 @@ class Removal @remove[slash(stripped(name))] = :rm_rf end - def glob(pattern, *rest) - Dir.glob(prefixed(pattern), *rest) {|n| - yield stripped(n) - } + def glob(pattern, *rest, &block) + Dir.glob(pattern, *rest, base: @base, &block) + end + + def default_gem?(spec) + (@defaults ||= {}).fetch(spec) do + File.open(prefixed(spec)) do |f| + if /^# default: (\S+) (\d+\.\d+)/ =~ f.gets("") + File.mtime(prefixed($1)) <= Time.at(Rational($2)) + else + false + end + rescue + false + end + end end def sorted @@ -115,7 +128,7 @@ srcdir = Removal.new(ARGV.shift) curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). - grep(/^(\w\S+)\s+\S+(?:\s+\S+\s+(\S+))?/) {$~.captures}.to_h rescue nil + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil srcdir.glob(".bundle/gems/*/") do |dir| base = File.basename(dir) @@ -133,12 +146,14 @@ end srcdir.glob(".bundle/specifications/*.gemspec") do |spec| unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") + next if srcdir.default_gem?(spec) srcdir.unlink(spec) end end curdir.glob(".bundle/specifications/*.gemspec") do |spec| unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}") + next if curdir.default_gem?(spec) curdir.unlink(spec) end end diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..44c0718a2d 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -9,6 +9,7 @@ s,@CC@,false,g s,@CFLAGS@,,g s,@CHDIR@,cd,g s,@CONFIGURE@,configure,g +s,@COUTFLAG@,-o ,g s,@CP@,cp,g s,@CPPFLAGS@,,g s,@CXXFLAGS@,,g @@ -24,6 +25,7 @@ s,@LIBRUBY_A@,libruby.a,g s,@MINIRUBY@,$(BASERUBY),g s,@MKDIR_P@,mkdir -p,g s,@OBJEXT@,o,g +s,@OUTFLAG@,-o ,g s,@PATH_SEPARATOR@,:,g s,@PWD@,.,g s,@RM@,rm -f,g @@ -32,6 +34,9 @@ s,@RMDIR@,rmdir,g s,@RMDIRS@,$(RMDIR) -p,g s,@RUBY@,$(BASERUBY),g s,@RUNRUBY@,$(MINIRUBY),g +s,@X_BUILD_EXEEXT@,,g +s,@X_DUMP_AST@,build-tool/dump_ast$(BUILD_EXEEXT),g +s,@X_DUMP_AST_TARGET@,$(DUMP_AST),g s,@arch@,noarch,g s,@bindir@,,g s,@configure_args@,,g @@ -40,5 +45,9 @@ s,@rubyarchdir@,,g s,@rubylibprefix@,,g s,@srcdir@,.,g +# for comipling dump_ast on build-os +/^CC *=/d + s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index d00d3ff69c..047fd0a571 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -527,6 +527,8 @@ module RbInstall const_set(:FileUtils, fu::NoWrite) fu end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. dir_mode = options.delete(:dir_mode) if options end yield @@ -659,6 +661,17 @@ module RbInstall "#{srcdir}/lib" end end + + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end + end end end @@ -696,6 +709,77 @@ module RbInstall end class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last + end + def write_cache_file end @@ -741,7 +825,7 @@ module RbInstall def install spec.post_install_message = nil dir_creating(without_destdir(gem_dir)) - RbInstall.no_write(options) {super} + RbInstall.no_write(options) { install_with_default_gem } end # Now build-ext builds all extensions including bundled gems. @@ -770,32 +854,57 @@ module RbInstall $installed_list.puts(d+"/") if $installed_list end end + + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + + def regenerate_plugins_for(spec, plugins_dir) + plugins = spec.plugins + return if plugins.empty? + dir = without_destdir(plugins_dir) + plugins.each do |plugin| + $installed_list.puts(File.join(dir, "#{spec.name}_plugin#{File.extname(plugin)}")) + end + unless $dryrun + super + end + end end end -def load_gemspec(file, base = nil) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - files = [] - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end if base + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do - "[" + files.join(", ") + "]" - end + "[" + spec_files + "]" + end \ + or code.gsub!(/IO\.popen\(.*git.*?\)/) do - "[" + files.join(", ") + "] || itself" + "[" + spec_files + "] || itself" end spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end spec.loaded_from = base ? File.join(base, File.basename(file)) : file - spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} spec.date = RUBY_RELEASE_DATE spec @@ -806,6 +915,7 @@ def install_default_gem(dir, srcdir, bindir) install_dir = with_destdir(gem_dir) prepare "default gems from #{dir}", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -824,14 +934,11 @@ def install_default_gem(dir, srcdir, bindir) base = "#{srcdir}/#{dir}" gems = Dir.glob("**/*.gemspec", base: base).map {|src| - spec = load_gemspec("#{base}/#{src}") - file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) - files = file_collector.collect + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect if files.empty? next end - spec.files = files - spec + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] @@ -1006,7 +1113,6 @@ install?(:local, :comm, :man) do prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} mantype, suffix, compress = Compressors.for($mantype) - mandir = File.join(mandir, "man") has_goruby = File.exist?(goruby_install_name+exeext) require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype mdocs.each do |mdoc| @@ -1016,8 +1122,8 @@ install?(:local, :comm, :man) do next unless has_goruby end - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) destfile = File.join(destdir, "#{destname}.#{section}") if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) @@ -1103,6 +1209,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do install_dir = with_destdir(gem_dir) prepare "bundled gems", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -1131,6 +1238,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do # the newly installed ruby. ENV.delete('RUBYOPT') + collector = RbInstall::Specs::FileCollector::UnpackedGem File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name @@ -1149,7 +1257,11 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped[gem_name] = "gemspec not found" next end - spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) unless spec.platform == Gem::Platform::RUBY skipped[gem_name] = "not ruby platform (#{spec.platform})" next @@ -1164,6 +1276,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do next end spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" @@ -1183,7 +1296,8 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped.default = "not found in bundled_gems" puts "skipped bundled gems:" gems.each do |gem| - printf " %-32s%s\n", File.basename(gem), skipped[gem] + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] end end end diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index 94a52dcb87..39ac16cb8f 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -37,6 +37,9 @@ TestInstanceNetHTTPResponse depending on external resources test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + ## Unknown failures # NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt @@ -44,7 +47,6 @@ test_compile(RegexpSingletonTest) test_linear_time?(RegexpSingletonTest) test_new(RegexpSingletonTest) -## Failed tests caused by unreleased version of Ruby - -# https://github.com/ruby/openssl/pull/774 -test_params(OpenSSLDHTest) +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir index f830fdc302..417a057d7f 100755 --- a/tool/rdoc-srcdir +++ b/tool/rdoc-srcdir @@ -1,7 +1,6 @@ #!ruby -W0 -rdoc_path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/rdoc-*").first -$LOAD_PATH.unshift("#{rdoc_path}/lib") +require 'rubygems' require 'rdoc/rdoc' # Make only the output directory relative to the invoked directory. @@ -17,7 +16,7 @@ options.title = options.title.sub(/Ruby \K.*version/) { .sort # "MAJOR" < "MINOR", fortunately .to_h.values.join(".") } -options.parse ARGV +options.parse ARGV + ["#{invoked}/rbconfig.rb"] options.singleton_class.define_method(:finish) do super() diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index 7f08eb8d1a..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -190,7 +190,7 @@ def backport_command_string next false if c.match(/\A\d{1,6}\z/) # skip SVN revision # check if the Git revision is included in master - has_commit(c, "master") + has_commit(c, "origin/master") end.sort_by do |changeset| Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb index 6dc0e4cec1..17fa499d69 100755 --- a/tool/releng/gen-mail.rb +++ b/tool/releng/gen-mail.rb @@ -10,7 +10,7 @@ end # Confirm current directory is www.ruby-lang.org's working directory def confirm_w_r_l_o_wd File.foreach('.git/config') do |line| - return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + return true if line.include?('ruby/www.ruby-lang.org.git') end abort "Run this script in www.ruby-lang.org's working directory" end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb index 8a5651dcd0..0dd5b25631 100755 --- a/tool/releng/update-www-meta.rb +++ b/tool/releng/update-www-meta.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "open-uri" require "yaml" +require_relative "../ruby-version" class Tarball attr_reader :version, :size, :sha1, :sha256, :sha512 @@ -41,22 +42,7 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" info = YAML.load(URI(uri).read) @@ -74,9 +60,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) rubydir = File.expand_path(File.join(__FILE__, '../../../')) puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` @@ -155,7 +142,7 @@ eom date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} - tag: v#{ver.tr('-.', '_')} + tag: #{RubyVersion.tag(ver)} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ stats: diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index e03e54e397..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 2a99e408da..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index 8a04041da9..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index ef6de8120e..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index 034905f74e..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index ac4122f3ac..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index a810d89f3c..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,7 +14,7 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions +class RubyVM::BareInstruction attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @@ -108,14 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - - def leaf_without_check_ints? - @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -149,6 +140,10 @@ class RubyVM::BareInstructions @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } end + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private def check_attribute_consistency @@ -187,6 +182,7 @@ class RubyVM::BareInstructions generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp @@ -228,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 4b5aec58dd..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index 10e5742897..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index d762dd3321..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 47d8da5513..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index cb895815ce..8bb28db1c1 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,6 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); @@ -13,15 +23,14 @@ rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, % end +#endif }; signed char c = t[insn]; diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index 569dca5845..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,6 +5,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ @@ -13,9 +16,14 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const uint8_t rb_vm_insn_len_info[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, % end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index e7ded75e65..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,10 +6,14 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); @@ -20,18 +24,29 @@ extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; const char rb_vm_insn_name_base[] = -% a.each do |i| - <%=cstr i%> "\0" +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end +#endif ; const unsigned short rb_vm_insn_name_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 996c33e960..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,10 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); @@ -21,15 +27,25 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const char rb_vm_insn_op_base[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end +#endif ; const unsigned short rb_vm_insn_op_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index 6dae554a51..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,10 +10,6 @@ #include "iseq.h" -// This is used to tell JIT that this insn would be leaf if CHECK_INTS didn't exist. -// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. -static bool leafness_of_check_ints = false; - static bool leafness_of_defined(rb_num_t op_type) { @@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type) case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index 2ca5aca7cf..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> +<%= render 'insn_leaf_info' %> <%= render 'comptime_insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..793528af5d --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index de7bb210ea..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,7 +23,7 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| @@ -56,7 +56,7 @@ rb_insn_unified_local_var_level(VALUE insn) switch (insn) { default: return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 9863c6bbe9..db64e20274 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -4,6 +4,8 @@ require 'fileutils' require "rbconfig" +require "find" +require "tempfile" module SyncDefaultGems include FileUtils @@ -11,80 +13,331 @@ module SyncDefaultGems module_function + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end + + CLASSICAL_DEFAULT_BRANCH = "master" + + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end + + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] REPOSITORIES = { - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - "net-http": "ruby/net-http", - "net-protocol": "ruby/net-protocol", - "open-uri": "ruby/open-uri", - "win32-registry": "ruby/win32-registry", - English: "ruby/English", - cgi: "ruby/cgi", - date: 'ruby/date', - delegate: "ruby/delegate", - did_you_mean: "ruby/did_you_mean", - digest: "ruby/digest", - erb: "ruby/erb", - error_highlight: "ruby/error_highlight", - etc: 'ruby/etc', - fcntl: 'ruby/fcntl', - fileutils: 'ruby/fileutils', - find: "ruby/find", - forwardable: "ruby/forwardable", - ipaddr: 'ruby/ipaddr', - json: 'ruby/json', - mmtk: ['ruby/mmtk', "main"], - open3: "ruby/open3", - openssl: "ruby/openssl", - optparse: "ruby/optparse", - pathname: "ruby/pathname", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - prism: ["ruby/prism", "main"], - psych: 'ruby/psych', - resolv: "ruby/resolv", - rubygems: 'rubygems/rubygems', - securerandom: "ruby/securerandom", - set: "ruby/set", - shellwords: "ruby/shellwords", - singleton: "ruby/singleton", - stringio: 'ruby/stringio', - strscan: 'ruby/strscan', - syntax_suggest: ["ruby/syntax_suggest", "main"], - tempfile: "ruby/tempfile", - time: "ruby/time", - timeout: "ruby/timeout", - tmpdir: "ruby/tmpdir", - tsort: "ruby/tsort", - un: "ruby/un", - uri: "ruby/uri", - weakref: "ruby/weakref", - yaml: "ruby/yaml", - zlib: 'ruby/zlib', + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + English: lib("ruby/English"), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + "ext/strscan/lib/strscan/truffleruby.rb", + ]), + syntax_suggest: repo(["ruby/syntax_suggest", "main"], [ + ["lib/syntax_suggest.rb", "lib/syntax_suggest.rb"], + ["lib/syntax_suggest", "lib/syntax_suggest"], + ["syntax_suggest.gemspec", "lib/syntax_suggest/syntax_suggest.gemspec"], + ["exe/syntax_suggest", "libexec/syntax_suggest"], + ["spec", "spec/syntax_suggest"], + ]), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), }.transform_keys(&:to_s) - CLASSICAL_DEFAULT_BRANCH = "master" + def REPOSITORIES.[](gem) + fetch(gem) {raise "unknown repository - #{gem}"} + end - class << REPOSITORIES - def [](gem) - repo, branch = super(gem) - return repo, branch || CLASSICAL_DEFAULT_BRANCH + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end + end end - def each_pair - super do |gem, (repo, branch)| - yield gem, [repo, branch || CLASSICAL_DEFAULT_BRANCH] - end + def group(files) + files.group_by {|file| find_upstream(file)} end end + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + def pipe_readlines(args, rs: "\0", chomp: true) IO.popen(args) do |f| f.readlines(rs, chomp: chomp) end end + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + def replace_rdoc_ref(file) src = File.binread(file) changed = false @@ -103,7 +356,7 @@ module SyncDefaultGems end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") + result = porcelain_status("*.c", "*.rb", "*.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! return if result.empty? @@ -111,247 +364,69 @@ module SyncDefaultGems result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| - next if line =~ /LICENSE\.md/ - - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") - end.compact.join - File.write("lib/bundler/bundler.gemspec", gemspec_content) - - cp_r("#{upstream}/bundler/spec", "spec/bundler") - %w[dev_gems test_gems rubocop_gems standard_gems].each do |gemfile| - ["rb.lock", "rb"].each do |ext| - cp_r("#{upstream}/tool/bundler/#{gemfile}.#{ext}", "tool/bundler") + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + unless File.exist?(upstream) + abort %[Expected '#{upstream}' (#{File.expand_path("#{upstream}")}) to be a directory, but it didn't exist.] + end + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) end end - rm_rf Dir.glob("spec/bundler/support/artifice/{vcr_cassettes,used_cassettes.txt}") - rm_rf Dir.glob("lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}") - when "json" - rm_rf(%w[ext/json lib/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/test/json", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - rm_rf(%w[ext/json/lib/json/pure.rb ext/json/lib/json/pure ext/json/lib/json/truffle_ruby/]) - json_files = Dir.glob("ext/json/lib/json/ext/**/*", File::FNM_DOTMATCH).select { |f| File.file?(f) } - rm_rf(json_files - Dir.glob("ext/json/lib/json/ext/**/*.rb") - Dir.glob("ext/json/lib/json/ext/**/depend")) - `git checkout ext/json/extconf.rb ext/json/generator/depend ext/json/parser/depend ext/json/depend benchmark/` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend ext/psych/.gitignore` - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/doc/date", "doc") - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/lib", "ext/strscan") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - begin - cp_r("#{upstream}/doc/strscan", "doc") - rescue Errno::ENOENT + end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) end - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib/cgi", "lib") - cp_r("#{upstream}/lib/cgi.rb", "lib") - rm_rf("lib/cgi/escape.jar") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf("test/did_you_mean/lib") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib/digest") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") - mkdir_p("ext/digest/sha2/lib") - cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") - move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r(Dir.glob("#{upstream}/test/*"), "test/set") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - when "open3" - sync_lib gem, upstream - rm_rf("lib/open3/jruby_windows.rb") - when "syntax_suggest" - sync_lib gem, upstream - rm_rf(%w[spec/syntax_suggest libexec/syntax_suggest]) - cp_r("#{upstream}/spec", "spec/syntax_suggest") - cp_r("#{upstream}/exe/syntax_suggest", "libexec/syntax_suggest") - when "prism" - rm_rf(%w[test/prism prism]) - - cp_r("#{upstream}/ext/prism", "prism") - cp_r("#{upstream}/lib/.", "lib") - cp_r("#{upstream}/test/prism", "test") - cp_r("#{upstream}/src/.", "prism") - - cp_r("#{upstream}/prism.gemspec", "lib/prism") - cp_r("#{upstream}/include/prism/.", "prism") - cp_r("#{upstream}/include/prism.h", "prism") - - cp_r("#{upstream}/config.yml", "prism/") - cp_r("#{upstream}/templates", "prism/") - rm_rf("prism/templates/javascript") - rm_rf("prism/templates/java") - rm_rf("prism/templates/rbi") - rm_rf("prism/templates/sig") - - rm("test/prism/snapshots_test.rb") - rm_rf("test/prism/snapshots") - - rm("prism/extconf.rb") - when "resolv" - rm_rf(%w[lib/resolv.* ext/win32/resolv test/resolv ext/win32/lib/win32/resolv.rb]) - cp_r("#{upstream}/lib/resolv.rb", "lib") - cp_r("#{upstream}/resolv.gemspec", "lib") - cp_r("#{upstream}/ext/win32/resolv", "ext/win32") - move("ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32") - rm_rf("ext/win32/resolv/lib") # Clean up empty directory - cp_r("#{upstream}/test/resolv", "test") - `git checkout ext/win32/resolv/depend` - when "win32-registry" - rm_rf(%w[ext/win32/lib/win32/registry.rb test/win32/test_registry.rb]) - cp_r("#{upstream}/lib/win32/registry.rb", "ext/win32/lib/win32") - cp_r("#{upstream}/test/win32/test_registry.rb", "test/win32") - cp_r("#{upstream}/win32-registry.gemspec", "ext/win32") - when "mmtk" - rm_rf("gc/mmtk") - cp_r("#{upstream}/gc/mmtk", "gc") - else - sync_lib gem, upstream + end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup end check_prerelease_version(gem) @@ -362,16 +437,13 @@ module SyncDefaultGems end def check_prerelease_version(gem) - return if gem == "rubygems" - return if gem == "mmtk" - - gem = gem.downcase + return if ["rubygems", "mmtk", "Onigmo"].include?(gem) require "net/https" require "json" require "uri" - uri = URI("https://rubygems.org/api/v1/versions/#{gem}/latest.json") + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") response = Net::HTTP.get(uri) latest_version = JSON.parse(response)["version"] @@ -388,42 +460,19 @@ module SyncDefaultGems puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version end - def ignore_file_pattern_for(gem) - patterns = [] - - # Common patterns - patterns << %r[\A(?: - [^/]+ # top-level entries - |\.git.* - |bin/.* - |ext/.*\.java - |rakelib/.* - |test/(?:lib|fixtures)/.* - |tool/(?!bundler/).* - )\z]mx - - # Gem-specific patterns - case gem - when nil - end&.tap do |pattern| - patterns << pattern - end - - Regexp.union(*patterns) - end - - def message_filter(repo, sha, input: ARGF) + def message_filter(repo, sha, log, context: nil) unless repo.count("/") == 1 and /\A\S+\z/ =~ repo raise ArgumentError, "invalid repository: #{repo}" end unless /\A\h{10,40}\z/ =~ sha raise ArgumentError, "invalid commit-hash: #{sha}" end - log = input.read - log.delete!("\r") - log << "\n" if !log.end_with?("\n") repo_url = "https://github.com/#{repo}" + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + # Split the subject from the log message according to git conventions. # SPECIAL TREAT: when the first line ends with a dot `.` (which is not # obeying the conventions too), takes only that line. @@ -445,41 +494,69 @@ module SyncDefaultGems end end commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url if log and !log.empty? log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last conv[log] log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { - ($~.begin(1) ? "" : "\n\n") + commit_url + ($~.begin(2) ? "\n" : "") + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") } else - log = commit_url + log = sync_note end - puts subject, "\n", log + "#{subject}\n\n#{log}" end - # Returns commit list as array of [commit_hash, subject]. - def commits_in_ranges(gem, repo, default_branch, ranges) - # If -a is given, discover all commits since the last picked commit - if ranges == true - # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running - # tests on Windows) - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", "rb", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] - end + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end - # Parse a given range with git log - ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" - end + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end - IO.popen(%W"git log --format=%H,%s #{range} --", "rb") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} - end + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } end end + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + #-- # Following methods used by sync_default_gems_with_commits return # true: success @@ -488,27 +565,9 @@ module SyncDefaultGems #++ def resolve_conflicts(gem, sha, edit) - # Skip this commit if everything has been removed as `ignored_paths`. - changes = pipe_readlines(%W"git status --porcelain -z") - if changes.empty? - puts "Skip empty commit #{sha}" - return false - end - - # We want to skip DD: deleted by both. - deleted = changes.grep(/^DD /) {$'} - system(*%W"git rm -f --", *deleted) unless deleted.empty? - - # Import UA: added by them - added = changes.grep(/^UA /) {$'} - system(*%W"git add --", *added) unless added.empty? - - # Discover unmerged files - # AU: unmerged, added by us - # DU: unmerged, deleted by us - # UU: unmerged, both modified - # AA: unmerged, both added - conflict = changes.grep(/\A(?:.U|AA) /) {$'} + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} # If -e option is given, open each conflicted file with an editor unless conflict.empty? if edit @@ -529,123 +588,170 @@ module SyncDefaultGems return true end - def preexisting?(base, file) - system(*%w"git cat-file -e", "#{base}:#{file}", err: File::NULL) + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end end - def filter_pickup_files(changed, ignore_file_pattern, base) - toplevels = {} - remove = [] - ignore = [] - changed = changed.reject do |f| - case - when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { - remove << top if toplevels[top] = !preexisting?(base, top) - } - # Remove any new top-level directories. - true - when ignore_file_pattern.match?(f) - # Forcibly reset any changes matching ignore_file_pattern. - (preexisting?(base, f) ? ignore : remove) << f - end + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] end - return changed, remove, ignore + [rewritten, ignored] end - def pickup_files(gem, changed, picked) - # Forcibly remove any files that we don't want to copy to this - # repository. - - ignore_file_pattern = ignore_file_pattern_for(gem) + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end - base = picked ? "HEAD~" : "HEAD" - changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? - unless remove.empty? - puts "Remove added files: #{remove.join(', ')}" - system(*%w"git rm -fr --", *remove) - if picked - system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup end + replace_rdoc_ref_all_full end - unless ignore.empty? - puts "Reset ignored files: #{ignore.join(', ')}" - system(*%W"git rm -r --", *ignore) - ignore.each {|f| system(*%W"git checkout -f", base, "--", f)} - end + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? - if changed.empty? - return nil + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp end - return changed + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit end - def pickup_commit(gem, sha, edit) - # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", "rb", &:read) - picked = $?.success? - if result =~ /nothing\ to\ commit/ - `git reset` - puts "Skip empty commit #{sha}" + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) + end + + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" return false end - # Skip empty commits - if result.empty? - return false - end + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end - if picked - changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") - else - changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") - end + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) - # Pick up files to merge. - unless changed = pickup_files(gem, changed, picked) - puts "Skip commit #{sha} only for tools or toplevel" - if picked - `git reset --hard HEAD~` - else - `git cherry-pick --abort` - end - return false - end + # No changes remaining after rewriting + return false unless rewritten - # If the cherry-pick attempt failed, try to resolve conflicts. - # Skip the commit, if it contains unresolved conflicts or no files to pick up. - unless picked or resolve_conflicts(gem, sha, edit) - `git reset` && `git checkout .` && `git clean -fd` - return picked || nil # Fail unless cherry-picked - end + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end - # Commit cherry-picked commit - if picked - system(*%w"git commit --amend --no-edit") - else - system(*%w"git cherry-pick --continue --no-edit") - end or return nil + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end - # Amend the commit if RDoc references need to be replaced - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end end + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" return true end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem] + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch puts "Sync #{repo} with commit history." # Fetch the repository to be synchronized @@ -654,86 +760,46 @@ module SyncDefaultGems `git remote add #{gem} https://github.com/#{repo}.git` end end - system(*%W"git fetch --no-tags #{gem}") - - commits = commits_in_ranges(gem, repo, default_branch, ranges) + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") - # Ignore Merge commits and already-merged commits. - commits.delete_if do |sha, subject| - subject.start_with?("Merge", "Auto Merge") + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] end - + commits = commits_in_ranges(ranges) if commits.empty? puts "No commits to pick" return true end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" - failed_commits = [] - - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." + puts "----" + puts "Pick #{sha} #{subject}" case pickup_commit(gem, sha, edit) when false - next + # skipped when nil - failed_commits << sha - next - end - - puts "Update commit message: #{sha}" - - # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter - IO.popen({"FILTER_BRANCH_SQUELCH_WARNING" => "1"}, - %W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], - &:read) - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + failed_commits << [sha, subject] end end unless failed_commits.empty? puts "---- failed commits ----" - puts failed_commits + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end return false end return true end - def sync_lib(repo, upstream = nil) - unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") - abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] - end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" - else - "lib/#{repo}.gemspec" - end - cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") - end - def update_default_gems(gem, release: false) - - repository, default_branch = REPOSITORIES[gem] - author, repository = repository.split('/') + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch puts "Update #{author}/#{repository}" @@ -773,28 +839,18 @@ module SyncDefaultGems REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" - if ARGV[1] == "release" - REPOSITORIES.each_key do |gem| - update_default_gems(gem, release: true) - sync_default_gems(gem) - end - else - REPOSITORIES.each_key {|gem| sync_default_gems(gem)} + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) end when "list" ARGV.shift pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, (gem)| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem - end - when "--message-filter" - ARGV.shift - if ARGV.size < 2 - abort "usage: #{$0} --message-filter repository commit-hash [input...]" + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream end - message_filter(*ARGV.shift(2)) - exit when "rdoc-ref" ARGV.shift pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV @@ -810,7 +866,13 @@ module SyncDefaultGems puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m -\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m ruby #$0 rubygems \e[1mPick a single commit from the upstream repository\e[0m @@ -822,6 +884,9 @@ module SyncDefaultGems \e[1mPick all commits since the last picked commit\e[0m ruby #$0 -a rubygems +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + \e[1mList known libraries\e[0m ruby #$0 list diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 535d101d48..b603cc09d7 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,37 +1,57 @@ require 'rbconfig' require 'timeout' require 'fileutils' +require 'shellwords' +require 'etc' require_relative 'lib/colorize' require_relative 'lib/gem_env' +require_relative 'lib/test/jobserver' ENV.delete("GNUMAKEFLAGS") github_actions = ENV["GITHUB_ACTIONS"] == "true" +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' -if RUBY_PLATFORM =~ /mswin|mingw/ - allowed_failures = [allowed_failures, "rbs,debug,irb"].join(',') -end -allowed_failures = allowed_failures.split(',').uniq.reject(&:empty?) +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) # make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 -bundled_gems = ARGV.first || '' +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] + +max = ENV['TEST_BUNDLED_GEMS_NPROCS']&.to_i || [Etc.nprocessors, 8].min +nprocs = Test::JobServer.max_jobs(max) || max +nprocs = 1 if nprocs < 1 + +if /mingw|mswin/ =~ RUBY_PLATFORM + spawn_group = :new_pgroup + signal_prefix = "" +else + spawn_group = :pgroup + signal_prefix = "-" +end + +jobs = [] File.foreach("#{gem_dir}/bundled_gems") do |line| - next if /^\s*(?:#|$)/ =~ line - gem = line.split.first - next unless bundled_gems.empty? || bundled_gems.split(",").include?(gem) + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} next unless File.directory?("#{gem_dir}/src/#{gem}/test") - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min + env_rubylib = rubylib toplib = gem unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") @@ -50,68 +70,174 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| File.unlink(path) if File.exist?(path) end - test_command << " stdlib_test validate RBS_SKIP_TESTS=#{__dir__}/rbs_skip_tests SKIP_RBS_VALIDATION=true" + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] first_timeout *= 3 when "debug" # Since debug gem requires debug.so in child processes without # activating the gem, we preset necessary paths in RUBYLIB # environment variable. - load_path = true + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + env_rubylib = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) when "test-unit" - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} test/run-test.rb" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 when "win32ole" next unless /mswin|mingw/ =~ RUBY_PLATFORM end - if load_path - libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) - next unless $?.success? - ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) - else - ENV["RUBYLIB"] = rubylib + jobs << { + gem: gem, + test_command: test_command, + first_timeout: first_timeout, + rubylib: env_rubylib, + } +end + +running_pids = [] +interrupted = false + +trap(:INT) do + interrupted = true + running_pids.each do |pid| + Process.kill("#{signal_prefix}INT", pid) rescue nil end +end - # 93(bright yellow) is copied from .github/workflows/mingw.yml - puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" - print "[command]" if github_actions - puts test_command - pid = Process.spawn(test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) - {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| - if sig - puts "Sending #{sig} signal" - Process.kill("-#{sig}", pid) - end - begin - break Timeout.timeout(sec) {Process.wait(pid)} - rescue Timeout::Error +results = Array.new(jobs.size) +queue = Queue.new +jobs.each_with_index { |j, i| queue << [j, i] } +nprocs.times { queue << nil } +print_queue = Queue.new + +puts "Running #{jobs.size} gem tests with #{nprocs} workers..." + +printer = Thread.new do + printed = 0 + while printed < jobs.size + result = print_queue.pop + break if result.nil? + + gem = result[:gem] + elapsed = result[:elapsed] + status = result[:status] + t = " in %.6f sec" % elapsed + + print (github_actions ? "::group::" : "\n") + puts colorize.decorate("Testing the #{gem} gem", "note") + print "[command]" if github_actions + p result[:test_command] + result[:log_lines].each { |l| puts l } + print result[:output] + print "::endgroup::\n" if github_actions + + if status&.success? + puts colorize.decorate("Test passed#{t}", "pass") + else + mesg = "Tests failed " + + (status&.signaled? ? "by SIG#{Signal.signame(status.termsig)}" : + "with exit code #{status&.exitstatus}") + t + puts colorize.decorate(mesg, "fail") + if allowed_failures.include?(gem) + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" + puts colorize.decorate(mesg, "skip") + else + failed << gem + exit_code = 1 + end end - rescue Interrupt - exit_code = Signal.list["INT"] - Process.kill("-KILL", pid) - Process.wait(pid) - break - end - print "::endgroup::\n" if github_actions - unless $?.success? + printed += 1 + end +end - mesg = "Tests failed " + - ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : - "with exit code #{$?.exitstatus}") - puts colorize.decorate(mesg, "fail") - if allowed_failures.include?(gem) - mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" - puts colorize.decorate(mesg, "skip") - else - failed << gem - exit_code = $?.exitstatus if $?.exitstatus +threads = nprocs.times.map do + Thread.new do + while (item = queue.pop) + break if interrupted + job, index = item + + start_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + rd, wr = IO.pipe + env = { "RUBYLIB" => job[:rubylib] } + pid = Process.spawn(env, *job[:test_command], spawn_group => true, [:out, :err] => wr) + wr.close + running_pids << pid + output_thread = Thread.new { rd.read } + + timeouts = { nil => job[:first_timeout], INT: 30, TERM: 10, KILL: nil } + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) + end + + log_lines = [] + status = nil + timeouts.each do |sig, sec| + if sig + log_lines << "Sending #{sig} signal" + begin + Process.kill("#{signal_prefix}#{sig}", pid) + rescue Errno::ESRCH + _, status = Process.wait2(pid) unless status + break + end + end + begin + break Timeout.timeout(sec) { _, status = Process.wait2(pid) } + rescue Timeout::Error + end + end + + captured = output_thread.value + rd.close + running_pids.delete(pid) + + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_at + + result = { + gem: job[:gem], + test_command: job[:test_command], + status: status, + elapsed: elapsed, + output: captured, + log_lines: log_lines, + } + results[index] = result + print_queue << result end end end -puts "Failed gems: #{failed.join(', ')}" unless failed.empty? +threads.each(&:join) +print_queue << nil +printer.join + +if interrupted + exit Signal.list["INT"] +end + +unless failed.empty? + puts "\n#{colorize.decorate("Failed gems: #{failed.join(', ')}", "fail")}" + results.compact.each do |result| + next if result[:status]&.success? + next if allowed_failures.include?(result[:gem]) + puts colorize.decorate("\nTesting the #{result[:gem]} gem", "note") + print result[:output] + end +end exit exit_code diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index 055577feea..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -114,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb index 3a1143d01d..3fd1419a9c 100644 --- a/tool/test/init.rb +++ b/tool/test/init.rb @@ -1,7 +1,15 @@ -# This file includes the settings for "make test-all". +# This file includes the settings for "make test-all" and "make test-tool". # Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. -ENV["GEM_SKIP"] = ENV["GEM_HOME"] = ENV["GEM_PATH"] = "".freeze +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + ENV.delete("RUBY_CODESIGN") Warning[:experimental] = false diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index e64c6c6fda..7fb39f010e 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -2,6 +2,7 @@ require 'test/unit' require 'stringio' require 'tmpdir' +require 'rubygems/version' require_relative '../sync_default_gems' module Test_SyncDefaultGems @@ -19,14 +20,8 @@ module Test_SyncDefaultGems expected.concat(trailers.map {_1+"\n"}) end - out, err = capture_output do - SyncDefaultGems.message_filter(repo, sha, input: StringIO.new(input, "r")) - end - - all_assertions do |a| - a.for("error") {assert_empty err} - a.for("result") {assert_pattern_list(expected, out)} - end + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) end def test_subject_only @@ -90,14 +85,41 @@ module Test_SyncDefaultGems @target = nil pend "No git" unless system("git --version", out: IO::NULL) @testdir = Dir.mktmpdir("sync") - @git_config = %W"HOME GIT_CONFIG_GLOBAL".each_with_object({}) {|k, c| c[k] = ENV[k]} + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" - git(*%W"config --global user.email test@ruby-lang.org") - git(*%W"config --global user.name", "Ruby") + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end @target = "sync-test" - SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) @sha = {} @origdir = Dir.pwd Dir.chdir(@testdir) @@ -129,6 +151,9 @@ module Test_SyncDefaultGems def teardown if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end Dir.chdir(@origdir) SyncDefaultGems::REPOSITORIES.delete(@target) ENV.update(@git_config) @@ -168,7 +193,7 @@ module Test_SyncDefaultGems end def top_commit(dir, format: "%H") - IO.popen(%W[git log --format=#{format} -1], chdir: dir, &:read)&.chomp + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp end def assert_sync(commits = true, success: true, editor: nil) @@ -200,6 +225,12 @@ module Test_SyncDefaultGems assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) end + def test_unknown_repository + assert_raise_with_message(RuntimeError, /unknown/) do + SyncDefaultGems::REPOSITORIES["not-exist"] + end + end + def test_skip_tool git(*%W"rm -q tool/ok", chdir: @target) git(*%W"commit -q -m", "Remove tool", chdir: @target) @@ -293,5 +324,56 @@ module Test_SyncDefaultGems assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) assert_not_operator(File, :exist?, "src/lib/bad.rb", out) end - end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index b0c2267b31..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb index 84b6cf688c..7f53e4b7dd 100644 --- a/tool/test/testunit/test_minitest_unit.rb +++ b/tool/test/testunit/test_minitest_unit.rb @@ -646,7 +646,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase def test_assert_in_delta_triggered x = "1.0e-06" - util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}." do + util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}.", strip: /\s+\(\/proc\/loadavg=.*\)/ do @tc.assert_in_delta 0.0, 1.0 / 1000, 0.000001 end end @@ -678,7 +678,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase end def test_assert_in_epsilon_triggered - util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.' do + util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.', strip: /\s+\(\/proc\/loadavg=.*\)/ do @tc.assert_in_epsilon 10000, 9990 end end @@ -686,7 +686,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase def test_assert_in_epsilon_triggered_negative_case x = "0.100000xxx" y = "0.1" - util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}." do + util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}.", strip: /\s+\(\/proc\/loadavg=.*\)/ do @tc.assert_in_epsilon(-1.1, -1, 0.1) end end @@ -1352,7 +1352,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase assert_equal expected, sample_test_case.test_methods.sort end - def assert_triggered expected, klass = Test::Unit::AssertionFailedError + def assert_triggered expected, klass = Test::Unit::AssertionFailedError, strip: nil e = assert_raise klass do yield end @@ -1360,6 +1360,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase msg = e.message.sub(/(---Backtrace---).*/m, '\1') msg.gsub!(/\(oid=[-0-9]+\)/, '(oid=N)') msg.gsub!(/(\d\.\d{6})\d+/, '\1xxx') # normalize: ruby version, impl, platform + msg.gsub!(strip, '') if strip assert_equal expected, msg end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 66c5390e1b..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -6,7 +6,15 @@ module TestParallel PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup @@ -25,7 +33,7 @@ module TestParallel @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,7 +53,7 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -58,7 +66,7 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) @@ -75,7 +83,7 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -99,7 +107,7 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf @@ -110,7 +118,7 @@ module TestParallel end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf @@ -118,24 +126,24 @@ module TestParallel assert_not_nil($1, "'done' was not found") result = Marshal.load($1.chomp.unpack1("m")) - assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do # TODO: misusage of pend (pend doens't use given block) - assert_equal(12, result[1]) - end - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) - assert_kind_of(Test::Unit::PendedError,result[2][1][2]) - assert_kind_of(Test::Unit::PendedError,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end @@ -143,9 +151,9 @@ module TestParallel end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args, jobs: "t1") + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@__runner_options__[:ruby], TESTS+"/runner.rb", + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", "--ruby", @__runner_options__[:ruby].join(" "), "-j", jobs, *opt_args, out: o, err: o) o.close @@ -154,7 +162,7 @@ module TestParallel def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -167,54 +175,47 @@ module TestParallel def test_ignore_jzero spawn_runner(jobs: "0") - Timeout.timeout(TIMEOUT) { + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end def test_hungup - spawn_runner "--worker-timeout=1", "test4test_hungup.rb" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} - assert_match(/^Retrying hung up testcases\.+$/, buf) - assert_match(/^2 tests,.* 0 failures,/, buf) - end - - def test_retry_workers - spawn_runner "--worker-timeout=1", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying hung up testcases\.+$/, buf) assert_match(/^2 tests,.* 0 failures,/, buf) end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb deleted file mode 100644 index a749b0e1d3..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV0 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb deleted file mode 100644 index 924a3b11fa..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV1 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb index e1535eb400..68284ab76a 100755 --- a/tool/update-NEWS-gemlist.rb +++ b/tool/update-NEWS-gemlist.rb @@ -5,13 +5,29 @@ prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] prevs = [prev, prev.sub(/\.\d+\z/, '')] update = ->(list, type, desc = "updated") do - item = ->(mark = "* ") do - "The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + - list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" + item = ->(mark = "* ", sub_bullets = {}) do + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + list.map {|g, v| + s = "#{mark}#{g} #{v}\n" + s += sub_bullets[g].join("") if sub_bullets[g] + s + }.join("") + "\n" end - news.sub!(/^(?:\*( +))?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do - item["#{$1&.<< " "}*#{$2 || ' '}"] - end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?:(?(1) \1)\*( *).*\n)(?:[ \t]+\*.*\n)*)*\n*/) do + mark = "#{$1&.dup&.<< " "}*#{$2 || ' '}" + # Parse existing sub-bullets from matched section + sb = {}; cg = nil + $~.to_s.each_line do |l| + if l =~ /^\* ([A-Za-z0-9_\-]+)\s/ + cg = $1 + elsif cg && l =~ /^\s+\*/ + (sb[cg] ||= []) << l + else + cg = nil + end + end + item[mark, sb] + end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+(?:( *\* *).*|[ \t]+\*.*))*)* *\n+\K/) do item[$1 || "* "] end end diff --git a/tool/update-NEWS-github-release.rb b/tool/update-NEWS-github-release.rb new file mode 100755 index 0000000000..f346209a7d --- /dev/null +++ b/tool/update-NEWS-github-release.rb @@ -0,0 +1,395 @@ +#!/usr/bin/env ruby + +require "bundler/inline" +require "json" +require "net/http" +require "uri" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" +end + +Octokit.configure do |c| + c.access_token = ENV["GITHUB_TOKEN"] + c.auto_paginate = true + c.per_page = 100 +end + +# Build a gem=>version map from stdgems.org stdgems.json for a given Ruby version (e.g., "3.4") +def fetch_default_gems_versions(ruby_version) + uri = URI.parse("https://stdgems.org/stdgems.json") + body = http_get(uri) + json = JSON.parse(body) + gems = json["gems"] || [] + + # Prefer the initial release key (e.g. "4.0.0") over the rolling + # major.minor key (e.g. "4.0") so the diff baseline reflects the original + # X.Y.0 release rather than the latest patch level. + initial_release_key = (ruby_version =~ /\A\d+\.\d+\z/) ? "#{ruby_version}.0" : nil + + map = {} + gems.each do |g| + # Only include default gems (skip ones marked removed) + next if g["removed"] + versions = g["versions"] || {} + + # versions has "default" and "bundled" keys, each containing Ruby version => version mappings + selected_version = nil + + # Try both "default" and "bundled" categories + ["default", "bundled"].each do |category| + category_versions = versions[category] || {} + next if selected_version + + if initial_release_key && category_versions.key?(initial_release_key) + selected_version = category_versions[initial_release_key] + elsif category_versions.key?(ruby_version) + selected_version = category_versions[ruby_version] + else + # Fall back to the highest patch version matching the given major.minor + major_minor = /^#{Regexp.escape(ruby_version)}\./ + candidates = category_versions.select { |k, _| k.match?(major_minor) } + if !candidates.empty? + # Sort keys as Gem::Version to pick the highest patch + selected_version = candidates.sort_by { |k, _| Gem::Version.new(k) }.last[1] + end + end + end + + next unless selected_version + + name = g["gem"] + # Normalize name to match existing special cases + name = "RubyGems" if name == "rubygems" + map[name] = selected_version + end + + map +end + +def previous_ruby_version + version_h = File.join(__dir__, "..", "include", "ruby", "version.h") + major = minor = nil + File.foreach(version_h) do |l| + major = $1.to_i if l =~ /^\s*#\s*define\s+RUBY_API_VERSION_MAJOR\s+(\d+)/ + minor = $1.to_i if l =~ /^\s*#\s*define\s+RUBY_API_VERSION_MINOR\s+(\d+)/ + end + abort "Cannot detect Ruby version from #{version_h}" unless major && minor + minor > 0 ? "#{major}.#{minor - 1}" : "#{major - 1}.0" +end + +# Load gem=>version map from a file or from stdgems.org if a Ruby version is given. +def load_versions(arg) + arg ||= previous_ruby_version + if File.exist?(arg) + File.readlines(arg).map(&:split).to_h + elsif arg.match?(/^\d+\.\d+(?:\.\d+)?$/) + fetch_default_gems_versions(arg) + elsif arg.downcase == "news" || arg =~ %r{https?://.*/NEWS\.md} + fetch_versions_from_news(arg) + else + abort "Invalid argument: #{arg}. Provide a file path or a Ruby version (e.g., 3.4)." + end +end + +# Build a gem=>version map by parsing the "## Stdlib updates" section from Ruby's NEWS.md +def fetch_versions_from_news(arg) + if arg.downcase == "news" + body = read_local_news_md + else + body = http_get(URI.parse(arg)) + end + + parse_stdlib_versions_from_news(body) +end + +# Fetch a URL with a clear abort message on network or HTTP failures. +# Used for sources whose absence makes the rest of the script meaningless. +def http_get(uri) + res = Net::HTTP.get_response(uri) + unless res.is_a?(Net::HTTPSuccess) + abort "error: #{uri} returned HTTP #{res.code} #{res.message}" + end + res.body +rescue SystemCallError, SocketError, IOError, Net::HTTPError => e + abort "error: failed to fetch #{uri}: #{e.class}: #{e.message}" +end + +def read_local_news_md + news_path = File.join(__dir__, "..", "NEWS.md") + unless File.exist?(news_path) + abort "NEWS.md not found at #{news_path}" + end + File.read(news_path) +end + +# Build a gem=>version map from the current repository state. Default gems +# come from {ext,lib}/**/*.gemspec (mirroring default_gems_list.yml) and +# bundled gems come from gems/bundled_gems. This avoids reading NEWS.md as +# the source of "current versions", which would create a circular dependency +# with update-NEWS-gemlist.rb. +def load_current_versions + require "rubygems" + root = File.expand_path("..", __dir__) + map = {} + + rg_path = File.join(root, "lib", "rubygems.rb") + if File.exist?(rg_path) + File.foreach(rg_path) do |line| + if /^\s*VERSION\s*=\s*"([^"]+)"/ =~ line + map["RubyGems"] = $1 + break + end + end + end + + Dir.glob(File.join(root, "{ext,lib}/**/*.gemspec")).each do |path| + spec = Gem::Specification.load(path) + next unless spec + map[spec.name] = spec.version.to_s + end + + bundled_path = File.join(root, "gems", "bundled_gems") + if File.exist?(bundled_path) + File.foreach(bundled_path) do |line| + next if line.start_with?("#") + name, version = line.split(" ", 3) + map[name] = version if name && version + end + end + + map +end + +def parse_stdlib_versions_from_news(body) + # Extract the Stdlib updates section + start_idx = body.index(/^## Stdlib updates$/) + unless start_idx + # Try a more lenient search if anchors differ + start_idx = body.index("## Stdlib\nupdates") || body.index("## Stdlib updates") + end + abort "Stdlib updates section not found in NEWS.md" unless start_idx + + section = body[start_idx..-1] + # Stop at the next top-level section header (skip the current header line) + first_line_len = section.lines.first ? section.lines.first.length : 0 + stop_idx = section.index(/^##\s+/, first_line_len) + section = stop_idx ? section[0...stop_idx] : section + + map = {} + + # Normalize lines and collect bullet entries like: "* gemname x.y.z" + section.each_line do |line| + line = line.strip + next unless line.start_with?("*") + # Remove leading bullet + entry = line.sub(/^\*\s+/, "") + + # Some lines can include descriptions or links; we only take simple "name version" + # Accept names with hyphens/underscores and versions like 1.2.3 or 1.2.3.4 + if entry =~ /^([A-Za-z0-9_\-]+)\s+(\d+(?:\.\d+){0,3})\b/ + name = $1 + ver = $2 + name = "RubyGems" if name.downcase == "rubygems" + map[name] = ver + end + end + + map +end + +def resolve_repo(name) + case name + when "minitest" + { repo: name, org: "minitest" } + when "test-unit" + { repo: name, org: "test-unit" } + when "RubyGems" + { repo: "rubygems", org: "rubygems" } + when "bundler" + { repo: "rubygems", org: "rubygems", tag_prefix: "bundler-" } + else + { repo: name, org: "ruby" } + end +end + +def fetch_release_range(name, from_version, to_version, org, repo, tag_prefix: "") + releases = [] + begin + Octokit.releases("#{org}/#{repo}").each do |release| + releases << release.tag_name + end + rescue Octokit::Error, Faraday::Error => e + warn "warning: skipping #{name} (#{org}/#{repo}): #{e.class}: #{e.message}" + return nil + end + + # Keep only this gem's version-like tags and sort ascending by semantic version + prefix = Regexp.escape(tag_prefix) + releases = releases.select { |t| t =~ /\A#{prefix}v?\d/ } + releases = releases.sort_by { |t| Gem::Version.new(t.sub(/\A#{prefix}/, "").sub(/^v/, "").tr("_", ".")) } + + start_index = releases.index("#{tag_prefix}v#{from_version}") || releases.index("#{tag_prefix}#{from_version}") + end_index = releases.index("#{tag_prefix}v#{to_version}") || releases.index("#{tag_prefix}#{to_version}") + + # If the "to" version is unreleased (e.g. 4.1.0.dev), include every released + # tag after the baseline up to the latest one available. + end_index ||= releases.length - 1 if to_version =~ /(?:\.|-)(?:dev|beta|alpha|rc|pre)/i + + return nil unless start_index && end_index + + range = releases[start_index + 1..end_index] + return nil if range.nil? || range.empty? + + range +end + +def collect_gem_updates(versions_from, versions_to) + results = [] + + versions_to.each do |name, version| + # Skip items which do not exist in the FROM map to reduce API calls + next unless versions_from.key?(name) + + info = resolve_repo(name) + org = info[:org] + repo = info[:repo] + tag_prefix = info[:tag_prefix] || "" + + release_range = fetch_release_range(name, versions_from[name], version, org, repo, tag_prefix: tag_prefix) + next unless release_range + + footnote_links = release_range.map do |rel| + tag = rel.sub(/\A#{Regexp.escape(tag_prefix)}/, "") + { + ref: "#{name}-#{tag}", + url: "https://github.com/#{org}/#{repo}/releases/tag/#{rel}", + } + end + + results << { + name: name, + version: version, + from_version: versions_from[name], + release_range: release_range, + footnote_links: footnote_links, + tag_prefix: tag_prefix, + } + end + + results +end + +def format_release_diff(result) + prefix = Regexp.escape(result[:tag_prefix] || "") + links = result[:release_range].map do |rel| + tag = rel.sub(/\A#{prefix}/, "") + "[#{tag}][#{result[:name]}-#{tag}]" + end + " * #{result[:from_version]} to #{links.join(', ')}" +end + +def print_results(results) + footnote_lines = [] + + results.each do |r| + puts "* #{r[:name]} #{r[:version]}" + puts format_release_diff(r) + r[:footnote_links].each do |fl| + footnote_lines << "[#{fl[:ref]}]: #{fl[:url]}" + end + end + + puts footnote_lines.join("\n") +end + +def update_news_md(results) + news_path = File.join(__dir__, "..", "NEWS.md") + unless File.exist?(news_path) + abort "NEWS.md not found at #{news_path}" + end + content = File.read(news_path) + lines = content.lines + + result_by_name = results.to_h { |r| [r[:name], r] } + + new_lines = [] + i = 0 + while i < lines.length + line = lines[i] + + if line =~ /^\* ([A-Za-z0-9_\-]+)\s+(\d+(?:\.\d+){0,3})\b/ + gem_name = $1 + + new_lines << line + + if (r = result_by_name[gem_name]) + # Skip any existing sub-bullet lines that follow + while i + 1 < lines.length && lines[i + 1] =~ /^\s+\*/ + i += 1 + end + + new_lines << "#{format_release_diff(r)}\n" + end + else + new_lines << line + end + i += 1 + end + + # All footnote definitions we can emit, indexed by ref name. Seed from existing + # release-tag defs in the file so gems skipped this run (e.g. transient API + # failures) keep their URLs, then overlay freshly fetched URLs. + release_ref_pattern = %r{^\[([^\]]+)\]:\s+(https://github\.com/[^/]+/[^/]+/releases/tag/.*)} + available_footnotes = {} + new_lines.each do |line| + if (m = line.match(release_ref_pattern)) + available_footnotes[m[1]] = "[#{m[1]}]: #{m[2]}" + end + end + results.each do |r| + r[:footnote_links].each do |fl| + available_footnotes[fl[:ref]] = "[#{fl[:ref]}]: #{fl[:url]}" + end + end + + # Refs the regenerated body actually uses (e.g. `][gem-vX.Y.Z]`) + used_refs = new_lines.join.scan(/\]\[([^\]]+)\]/).flatten.uniq + + # Drop all existing GitHub release-tag link defs; the used subset is + # re-emitted below in body-ref order so the footer is deterministic. + new_lines.reject! { |line| line.match?(release_ref_pattern) } + + # Trim trailing blank lines so the appended footer block is clean + new_lines.pop while new_lines.last == "\n" + new_lines << "\n" unless new_lines.last&.end_with?("\n") + + # Append footnote defs only for refs the body still references + emitted = 0 + used_refs.each do |ref| + if (footnote = available_footnotes[ref]) + new_lines << "#{footnote}\n" + emitted += 1 + end + end + + File.write(news_path, new_lines.join) + puts "Updated #{news_path} with #{results.length} gem update entries and #{emitted} footnote links." +end + +# --- Main --- + +update_mode = ARGV.delete("--update") + +versions_from = load_versions(ARGV[0]) +versions_to = load_current_versions + +results = collect_gem_updates(versions_from, versions_to) + +print_results(results) + +if update_mode + update_news_md(results) +end diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index 2842516cac..565a522aa0 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,38 +1,45 @@ -#!ruby -pla +#!ruby -alpF\s+|#.* BEGIN { require 'rubygems' date = nil # STDOUT is not usable in inplace edit mode output = $-i ? STDOUT : STDERR + # Gems to skip auto-updating (e.g. when a new major version breaks CI) + pinned = %w[rbs] } output = STDERR if ARGF.file == STDIN END { output.print date.strftime("latest_date=%F") if date } -unless /^[^#]/ !~ (gem = $F[0]) - ver = Gem::Version.new($F[1]) - (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| - s.platform == "ruby" && s.name == gem - } - if gem.version > ver - gem = src.fetch_spec(gem) - if ENV["UPDATE_BUNDLED_GEMS_ALL"] - uri = gem.metadata["source_code_uri"] || gem.homepage - uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") - else - uri = $F[2] - end - date = gem.date if !date or gem.date && gem.date > date - if $F[3] - if $F[3].include?($F[1]) - $F[3][$F[1]] = gem.version.to_s - elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] - $F[3..-1] = [] +if gem = $F[0] + unless pinned.include?(gem) + ver = Gem::Version.new($F[1]) + (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| + s.platform == "ruby" && s.name == gem + } + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! end - f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] - $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} - $_ = [$_, *f].join(" ") unless f.empty? - $_.rstrip! end end diff --git a/tool/update-deps b/tool/update-deps index 0b90876cd2..0b73228b88 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,6 +17,14 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] @@ -88,6 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -149,16 +166,6 @@ FILES_NEED_VPATH = %w[ enc/trans/single_byte.c enc/trans/utf8_mac.c enc/trans/utf_16_32.c - - prism/api_node.c - prism/ast.h - prism/diagnostic.c - prism/diagnostic.h - prism/node.c - prism/prettyprint.c - prism/serialize.c - prism/token_type.c - prism/version.h ] # Multiple files with same filename. @@ -206,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -319,6 +326,9 @@ def read_make_deps(cwd) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) + next if /\.bundle\// =~ curdir.to_s next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..a265a3c01f --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,165 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +required_ruby_version = Gem::Version.new("3.4.0") +raise "Ruby version #{required_ruby_version} or higher is required" if Gem::Version.new(RUBY_VERSION) < required_ruby_version + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index && specopts_index + raise "Expected only one of RUN_OPTS or SPECOPTS to be present in make command, but both were found" + end + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + # SPECOPTS needs -T before each option to pass it through mspec to Ruby + zjit_opts.each { |opt| specopts.concat(["-T", opt]) } + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = add_zjit_options([RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS]).shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_diff.rb b/tool/zjit_diff.rb new file mode 100755 index 0000000000..4f8f74d20f --- /dev/null +++ b/tool/zjit_diff.rb @@ -0,0 +1,272 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'fileutils' +require 'optparse' +require 'tmpdir' +require 'logger' +require 'digest' +require 'shellwords' + +GitRef = Struct.new(:ref, :commit_hash) + +RUBIES_DIR = File.join(Dir.home, '.zjit-diff') +BEFORE_NAME = 'ruby-zjit-before' +AFTER_NAME = 'ruby-zjit-after' + +LOG = Logger.new($stderr) + +def macos? + Gem::Platform.local == 'darwin' +end + +class CommandRunner + def initialize(quiet: false) + @quiet = quiet + end + + def cmd(*args, **options) + options[:out] ||= @quiet ? File::NULL : $stderr + options = options.merge(exception: true) + system(*args, **options) + end +end + +class ZJITDiff + DATA_FILENAME = File.join('data', 'zjit_diff') + RUBY_BENCH_REPO_URL = 'https://github.com/ruby/ruby-bench.git' + + def initialize(before_hash:, after_hash:, runner:, options:) + @before_hash = before_hash + @after_hash = after_hash + @runner = runner + @options = options + end + + def bench! + LOG.info('Running benchmarks') + ruby_bench_path = @options[:bench_path] || setup_ruby_bench + run_benchmarks(ruby_bench_path) + end + + private + + def run_benchmarks(ruby_bench_path) + Dir.chdir(ruby_bench_path) do + @runner.cmd({ 'RUBIES_DIR' => RUBIES_DIR }, + './run_benchmarks.rb', + '--chruby', + "before::#{@before_hash} --zjit-stats;after::#{@after_hash} --zjit-stats", + '--out-name', + DATA_FILENAME, + *@options[:bench_args], + *@options[:name_filters]) + + @runner.cmd('./misc/zjit_diff.rb', "#{DATA_FILENAME}.json", out: $stdout) + end + end + + def setup_ruby_bench + path = File.join(Dir.tmpdir, 'ruby-bench') + if Dir.exist?(path) + LOG.info('ruby-bench already cloned, pulling from upstream') + Dir.chdir(path) do + @runner.cmd('git', 'pull') + end + else + LOG.info("ruby-bench not cloned yet, cloning repository to #{path}") + @runner.cmd('git', 'clone', RUBY_BENCH_REPO_URL, path) + end + path + end +end + +class RubyWorktree + attr_reader :hash + + BREW_REQUIRED_PACKAGES = %w[openssl readline libyaml].freeze + + def initialize(name:, ref:, runner:, force_rebuild: false) + @path = File.join(Dir.tmpdir, name) + @ref = ref + @force_rebuild = force_rebuild + @runner = runner + @hash = nil + + setup_worktree + end + + def build! + Dir.chdir(@path) do + configure_cmd_args = ['--enable-zjit=dev', '--disable-install-doc'] + if macos? + brew_prefixes = BREW_REQUIRED_PACKAGES.map do |pkg| + `brew --prefix #{pkg}`.strip + end + configure_cmd_args << "--with-opt-dir=#{brew_prefixes.join(':')}" + end + configure_cmd_hash = Digest::MD5.hexdigest(configure_cmd_args.join('')) + + build_cmd_args = ['-j', 'miniruby'] + build_cmd_hash = Digest::MD5.hexdigest(build_cmd_args.join('')) + + @hash = "#{configure_cmd_hash}-#{build_cmd_hash}-#{@ref.commit_hash}" + prefix = File.join(RUBIES_DIR, @hash) + + if Dir.exist?(prefix) && !@force_rebuild + LOG.info("Found existing build for #{@ref.ref}, skipping build") + return + end + + @runner.cmd('./autogen.sh') + + cmd = [ + './configure', + *configure_cmd_args, + "--prefix=#{prefix}" + ] + + @runner.cmd(*cmd) + @runner.cmd('make', *build_cmd_args) + @runner.cmd('make', 'install') + end + end + + private + + def setup_worktree + if Dir.exist?(@path) + LOG.info("Existing worktree found at #{@path}") + Dir.chdir(@path) do + @runner.cmd('git', 'checkout', @ref.commit_hash) + end + else + LOG.info("Creating worktree for ref '#{@ref.ref}' at #{@path}") + @runner.cmd('git', 'worktree', 'add', '--detach', @path, @ref.commit_hash) + end + end +end + +def clean! + [BEFORE_NAME, AFTER_NAME].each do |name| + path = File.join(Dir.tmpdir, name) + if Dir.exist?(path) + LOG.info("Removing worktree at #{path}") + system('git', 'worktree', 'remove', '--force', path) + end + end + + if Dir.exist?(RUBIES_DIR) + LOG.info("Removing ruby installations from #{RUBIES_DIR}") + FileUtils.rm_rf(RUBIES_DIR) + end + + bench_path = File.join(Dir.tmpdir, 'ruby-bench') + return unless Dir.exist?(bench_path) + + LOG.info("Removing ruby-bench clone at #{bench_path}") + FileUtils.rm_rf(bench_path) +end + +def parse_ref(ref) + out = `git rev-parse --verify #{ref}` + return nil unless $?.success? + + GitRef.new(ref: ref, commit_hash: out.strip) +end + +DEFAULT_BENCHMARKS = %w[lobsters railsbench].freeze + +options = {} + +subtext = <<~HELP + Subcommands: + bench : Run benchmarks + clean : Clean temporary files created by benchmarks + See '#{$PROGRAM_NAME} COMMAND --help' for more information on a specific command. +HELP + +top_level = OptionParser.new do |opts| + opts.banner = "Usage: #{$PROGRAM_NAME} [options]" + opts.separator('') + opts.separator(subtext) +end + +subcommands = { + 'bench' => OptionParser.new do |opts| + opts.banner = "Usage: #{$PROGRAM_NAME} [options] <benchmarks to run>" + + opts.on('--before REF', 'Git ref for ruby (before)') do |ref| + git_ref = parse_ref ref + if git_ref.nil? + warn "Error: '#{ref}' is not a valid git ref" + exit 1 + end + + options[:before] = git_ref + end + + opts.on('--after REF', 'Git ref for ruby (after)') do |ref| + git_ref = parse_ref ref + if git_ref.nil? + warn "Error: '#{ref}' is not a valid git ref" + exit 1 + end + + options[:after] = git_ref + end + + opts.on('--bench-path PATH', + 'Path to an existing ruby-bench repository clone ' \ + '(if not specified, ruby-bench will be cloned automatically to a temporary directory)') do |path| + options[:bench_path] = path + end + + opts.on('--bench-args ARGS', 'Args to pass to ruby-bench') do |bench_args| + options[:bench_args] = bench_args.shellsplit + end + + opts.on('--force-rebuild', + 'Force building ruby again instead of using even if existing builds exist in the cache at ~/.diffs') do + options[:force_rebuild] = true + end + + opts.on('--quiet', 'Silence output of commands except for benchmark result') do + options[:quiet] = true + end + + opts.separator('') + opts.separator('If no benchmarks are specified, the benchmarks that will be run are:') + opts.separator(DEFAULT_BENCHMARKS.join(', ')) + end, + 'clean' => OptionParser.new do |opts| + end +} + +top_level.order! +command = ARGV.shift +subcommands[command].order! + +case command +when 'bench' + options[:name_filters] = ARGV.empty? ? DEFAULT_BENCHMARKS : ARGV + options[:after] ||= parse_ref('HEAD') + + runner = CommandRunner.new(quiet: options[:quiet]) + + before = RubyWorktree.new(name: BEFORE_NAME, + ref: options[:before], + runner: runner, + force_rebuild: options[:force_rebuild]) + before.build! + after = RubyWorktree.new(name: AFTER_NAME, + ref: options[:after], + runner: runner, + force_rebuild: options[:force_rebuild]) + after.build! + + zjit_diff = ZJITDiff.new(runner: runner, before_hash: before.hash, after_hash: after.hash, options: options) + zjit_diff.bench! +when 'clean' + clean! +end diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |